From 8e878f6871bdd09c68d86d3cc0477106d72ecb5b Mon Sep 17 00:00:00 2001 From: Felipe Coury Date: Sun, 5 Jul 2026 20:50:23 -0300 Subject: [PATCH] feat(terminal-browser): import hardened carbonyl runtime --- codex-rs/Cargo.lock | 27 + codex-rs/Cargo.toml | 2 + codex-rs/features/src/lib.rs | 16 + codex-rs/terminal-browser/BUILD.bazel | 6 + codex-rs/terminal-browser/Cargo.toml | 37 ++ .../terminal-browser/src/accessibility.rs | 267 ++++++++++ codex-rs/terminal-browser/src/actions.rs | 268 ++++++++++ .../terminal-browser/src/actions_tests.rs | 311 +++++++++++ codex-rs/terminal-browser/src/cdp.rs | 254 +++++++++ codex-rs/terminal-browser/src/cdp_tests.rs | 277 ++++++++++ codex-rs/terminal-browser/src/devtools.rs | 141 +++++ codex-rs/terminal-browser/src/diagnostics.rs | 140 +++++ .../terminal-browser/src/diagnostics_tests.rs | 36 ++ codex-rs/terminal-browser/src/error.rs | 132 +++++ codex-rs/terminal-browser/src/error_tests.rs | 38 ++ codex-rs/terminal-browser/src/handles.rs | 57 ++ .../terminal-browser/src/handles_tests.rs | 17 + .../terminal-browser/src/human_control.rs | 248 +++++++++ codex-rs/terminal-browser/src/input.rs | 96 ++++ codex-rs/terminal-browser/src/lib.rs | 46 ++ codex-rs/terminal-browser/src/navigation.rs | 423 +++++++++++++++ codex-rs/terminal-browser/src/network.rs | 11 + codex-rs/terminal-browser/src/process.rs | 493 ++++++++++++++++++ .../terminal-browser/src/process_tests.rs | 98 ++++ codex-rs/terminal-browser/src/profile.rs | 228 ++++++++ .../terminal-browser/src/profile_control.rs | 199 +++++++ .../terminal-browser/src/profile_tests.rs | 74 +++ codex-rs/terminal-browser/src/runtime.rs | 115 ++++ .../terminal-browser/src/runtime_tests.rs | 20 + codex-rs/terminal-browser/src/sandbox.rs | 259 +++++++++ .../terminal-browser/src/sandbox_tests.rs | 119 +++++ codex-rs/terminal-browser/src/screen.rs | 236 +++++++++ codex-rs/terminal-browser/src/scripts.rs | 22 + codex-rs/terminal-browser/src/session.rs | 278 ++++++++++ .../src/terminal_browser_tests.rs | 349 +++++++++++++ .../terminal-browser/src/tool_dispatch.rs | 318 +++++++++++ codex-rs/terminal-browser/src/url_policy.rs | 56 ++ 37 files changed, 5714 insertions(+) create mode 100644 codex-rs/terminal-browser/BUILD.bazel create mode 100644 codex-rs/terminal-browser/Cargo.toml create mode 100644 codex-rs/terminal-browser/src/accessibility.rs create mode 100644 codex-rs/terminal-browser/src/actions.rs create mode 100644 codex-rs/terminal-browser/src/actions_tests.rs create mode 100644 codex-rs/terminal-browser/src/cdp.rs create mode 100644 codex-rs/terminal-browser/src/cdp_tests.rs create mode 100644 codex-rs/terminal-browser/src/devtools.rs create mode 100644 codex-rs/terminal-browser/src/diagnostics.rs create mode 100644 codex-rs/terminal-browser/src/diagnostics_tests.rs create mode 100644 codex-rs/terminal-browser/src/error.rs create mode 100644 codex-rs/terminal-browser/src/error_tests.rs create mode 100644 codex-rs/terminal-browser/src/handles.rs create mode 100644 codex-rs/terminal-browser/src/handles_tests.rs create mode 100644 codex-rs/terminal-browser/src/human_control.rs create mode 100644 codex-rs/terminal-browser/src/input.rs create mode 100644 codex-rs/terminal-browser/src/lib.rs create mode 100644 codex-rs/terminal-browser/src/navigation.rs create mode 100644 codex-rs/terminal-browser/src/network.rs create mode 100644 codex-rs/terminal-browser/src/process.rs create mode 100644 codex-rs/terminal-browser/src/process_tests.rs create mode 100644 codex-rs/terminal-browser/src/profile.rs create mode 100644 codex-rs/terminal-browser/src/profile_control.rs create mode 100644 codex-rs/terminal-browser/src/profile_tests.rs create mode 100644 codex-rs/terminal-browser/src/runtime.rs create mode 100644 codex-rs/terminal-browser/src/runtime_tests.rs create mode 100644 codex-rs/terminal-browser/src/sandbox.rs create mode 100644 codex-rs/terminal-browser/src/sandbox_tests.rs create mode 100644 codex-rs/terminal-browser/src/screen.rs create mode 100644 codex-rs/terminal-browser/src/scripts.rs create mode 100644 codex-rs/terminal-browser/src/session.rs create mode 100644 codex-rs/terminal-browser/src/terminal_browser_tests.rs create mode 100644 codex-rs/terminal-browser/src/tool_dispatch.rs create mode 100644 codex-rs/terminal-browser/src/url_policy.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 88601e9384..339fc68867 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -4001,6 +4001,32 @@ dependencies = [ "tokio", ] +[[package]] +name = "codex-terminal-browser" +version = "0.0.0" +dependencies = [ + "anyhow", + "codex-protocol", + "codex-sandboxing", + "codex-utils-absolute-path", + "codex-utils-path-uri", + "codex-utils-pty", + "futures", + "libc", + "pretty_assertions", + "reqwest 0.12.28", + "serde", + "serde_json", + "sha2 0.10.9", + "tempfile", + "tokio", + "tokio-tungstenite", + "tracing", + "url", + "vt100", + "which 8.0.0", +] + [[package]] name = "codex-terminal-detection" version = "0.0.0" @@ -4114,6 +4140,7 @@ dependencies = [ "codex-sandboxing", "codex-shell-command", "codex-state", + "codex-terminal-browser", "codex-terminal-detection", "codex-utils-absolute-path", "codex-utils-approval-presets", diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index 34097b56aa..72bde4fb92 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -115,6 +115,7 @@ members = [ "codex-client", "codex-api", "state", + "terminal-browser", "terminal-detection", "test-binary-support", "thread-manager-sample", @@ -226,6 +227,7 @@ codex-skills-extension = { path = "ext/skills" } codex-skills = { path = "skills" } codex-state = { path = "state" } codex-stdio-to-uds = { path = "stdio-to-uds" } +codex-terminal-browser = { path = "terminal-browser" } codex-terminal-detection = { path = "terminal-detection" } codex-test-binary-support = { path = "test-binary-support" } codex-thread-store = { path = "thread-store" } diff --git a/codex-rs/features/src/lib.rs b/codex-rs/features/src/lib.rs index 7ab7df18d1..f01546d031 100644 --- a/codex-rs/features/src/lib.rs +++ b/codex-rs/features/src/lib.rs @@ -186,6 +186,8 @@ pub enum Feature { /// /// Requirements-only gate: this should be set from requirements, not user config. BrowserUseExternal, + /// Allow the agent to control a Carbonyl browser rendered in the terminal. + TerminalBrowser, /// Allow Codex Computer Use. /// /// Requirements-only gate: this should be set from requirements, not user config. @@ -1136,6 +1138,20 @@ pub const FEATURES: &[FeatureSpec] = &[ stage: Stage::Stable, default_enabled: true, }, + FeatureSpec { + id: Feature::TerminalBrowser, + key: "terminal_browser", + stage: if cfg!(any(target_os = "macos", target_os = "linux")) { + Stage::Experimental { + name: "Terminal browser", + menu_description: "Let Codex browse with Carbonyl in the right terminal panel.", + announcement: "", + } + } else { + Stage::UnderDevelopment + }, + default_enabled: false, + }, FeatureSpec { id: Feature::ComputerUse, key: "computer_use", diff --git a/codex-rs/terminal-browser/BUILD.bazel b/codex-rs/terminal-browser/BUILD.bazel new file mode 100644 index 0000000000..d9bca76f4f --- /dev/null +++ b/codex-rs/terminal-browser/BUILD.bazel @@ -0,0 +1,6 @@ +load("//:defs.bzl", "codex_rust_crate") + +codex_rust_crate( + name = "terminal-browser", + crate_name = "codex_terminal_browser", +) diff --git a/codex-rs/terminal-browser/Cargo.toml b/codex-rs/terminal-browser/Cargo.toml new file mode 100644 index 0000000000..88a8c1a1d2 --- /dev/null +++ b/codex-rs/terminal-browser/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "codex-terminal-browser" +version.workspace = true +edition.workspace = true +license.workspace = true + +[lib] +name = "codex_terminal_browser" +path = "src/lib.rs" +doctest = false + +[lints] +workspace = true + +[dependencies] +anyhow = { workspace = true } +codex-protocol = { workspace = true } +codex-sandboxing = { workspace = true } +codex-utils-absolute-path = { workspace = true } +codex-utils-path-uri = { workspace = true } +codex-utils-pty = { workspace = true } +futures = { workspace = true } +libc = { workspace = true } +reqwest = { workspace = true, features = ["json"] } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +sha2 = { workspace = true } +tempfile = { workspace = true } +tokio = { workspace = true, features = ["macros", "net", "process", "rt", "sync", "time"] } +tokio-tungstenite = { workspace = true } +tracing = { workspace = true } +url = { workspace = true } +vt100 = { workspace = true } +which = { workspace = true } + +[dev-dependencies] +pretty_assertions = { workspace = true } diff --git a/codex-rs/terminal-browser/src/accessibility.rs b/codex-rs/terminal-browser/src/accessibility.rs new file mode 100644 index 0000000000..7011797f83 --- /dev/null +++ b/codex-rs/terminal-browser/src/accessibility.rs @@ -0,0 +1,267 @@ +use anyhow::Context; +use anyhow::Result; +use serde_json::Value; +use serde_json::json; + +use crate::actions::BrowserToolOutput; +use crate::actions::bounded_snapshot_json; +use crate::actions::page_metadata; +use crate::cdp::CdpClient; +use crate::handles::BrowserHandles; + +const MAX_AX_NODES: usize = 300; +const MAX_SNAPSHOT_TEXT_CHARS: usize = 6_000; + +pub(crate) async fn snapshot( + client: &CdpClient, + handles: &mut BrowserHandles, +) -> Result { + let tree = client + .call("Accessibility.getFullAXTree", json!({})) + .await?; + let metadata = page_metadata(client).await?; + let ax_nodes = tree + .get("nodes") + .and_then(Value::as_array) + .context("accessibility tree omitted nodes")?; + handles.begin_snapshot(); + let mut actionable_nodes = Vec::new(); + let mut contextual_nodes = Vec::new(); + let mut page_text = String::new(); + for (index, node) in ax_nodes.iter().enumerate().filter(|(_, node)| { + !node + .get("ignored") + .and_then(Value::as_bool) + .unwrap_or(/*default*/ false) + }) { + let role = ax_value(node, "role"); + let name = ax_value(node, "name"); + if role == "StaticText" && !name.is_empty() && page_text.len() < MAX_SNAPSHOT_TEXT_CHARS { + if !page_text.is_empty() { + page_text.push('\n'); + } + let remaining = MAX_SNAPSHOT_TEXT_CHARS.saturating_sub(page_text.len()); + page_text.extend(name.chars().take(remaining)); + } + let actionable = is_actionable_role(&role); + if (role.is_empty() && name.is_empty()) + || (actionable && actionable_nodes.len() >= MAX_AX_NODES) + { + continue; + } + let mut output = serde_json::Map::new(); + output.insert("role".to_string(), Value::String(role.clone())); + output.insert( + "name".to_string(), + Value::String(clipped(&name, /*max_chars*/ 200)), + ); + if actionable + && let Some(backend_node_id) = node.get("backendDOMNodeId").and_then(Value::as_u64) + { + output.insert( + "nodeId".to_string(), + Value::String(handles.insert(backend_node_id)), + ); + } + if let Some(disabled) = ax_property(node, "disabled").and_then(Value::as_bool) { + output.insert("disabled".to_string(), Value::Bool(disabled)); + } + if let Some(checked) = ax_property(node, "checked") { + output.insert("checked".to_string(), checked.clone()); + } + let value = ax_value(node, "value"); + if !value.is_empty() { + output.insert( + "value".to_string(), + Value::String(if is_editable_role(&role) { + "".to_string() + } else { + clipped(&value, /*max_chars*/ 200) + }), + ); + } + if actionable { + actionable_nodes.push((index, Value::Object(output))); + } else if contextual_nodes.len() < MAX_AX_NODES { + contextual_nodes.push((index, Value::Object(output))); + } + } + contextual_nodes.truncate(MAX_AX_NODES.saturating_sub(actionable_nodes.len())); + actionable_nodes.extend(contextual_nodes); + actionable_nodes.sort_by_key(|(index, _)| *index); + let nodes = actionable_nodes + .into_iter() + .map(|(_, node)| node) + .collect::>(); + let snapshot = json!({ + "url": metadata.url, + "title": metadata.title, + "nodes": nodes, + "text": clipped(&page_text, MAX_SNAPSHOT_TEXT_CHARS), + }); + Ok(BrowserToolOutput::Text(bounded_snapshot_json(snapshot)?)) +} + +pub(crate) async fn click( + client: &CdpClient, + handles: &BrowserHandles, + node_id: &str, +) -> Result { + let backend_node_id = handles.resolve(node_id)?; + client + .call( + "DOM.scrollIntoViewIfNeeded", + json!({ "backendNodeId": backend_node_id }), + ) + .await + .context("scroll target node into view")?; + let (x, y) = node_center(client, backend_node_id).await?; + client + .call( + "Input.dispatchMouseEvent", + json!({ "type": "mouseMoved", "x": x, "y": y, "button": "none" }), + ) + .await?; + client + .call( + "Input.dispatchMouseEvent", + json!({ "type": "mousePressed", "x": x, "y": y, "button": "left", "clickCount": 1 }), + ) + .await?; + client + .call( + "Input.dispatchMouseEvent", + json!({ "type": "mouseReleased", "x": x, "y": y, "button": "left", "clickCount": 1 }), + ) + .await?; + Ok(BrowserToolOutput::Text(format!("clicked {node_id}"))) +} + +pub(crate) async fn fill( + client: &CdpClient, + handles: &BrowserHandles, + node_id: &str, + text: &str, +) -> Result { + let backend_node_id = handles.resolve(node_id)?; + client + .call("DOM.focus", json!({ "backendNodeId": backend_node_id })) + .await?; + let select_modifier = if cfg!(target_os = "macos") { 4 } else { 2 }; + client + .call( + "Input.dispatchKeyEvent", + json!({ "type": "rawKeyDown", "key": "a", "code": "KeyA", "modifiers": select_modifier }), + ) + .await?; + client + .call( + "Input.dispatchKeyEvent", + json!({ "type": "keyUp", "key": "a", "code": "KeyA", "modifiers": select_modifier }), + ) + .await?; + client + .call( + "Input.dispatchKeyEvent", + json!({ "type": "rawKeyDown", "key": "Backspace", "code": "Backspace" }), + ) + .await?; + client + .call( + "Input.dispatchKeyEvent", + json!({ "type": "keyUp", "key": "Backspace", "code": "Backspace" }), + ) + .await?; + client + .call("Input.insertText", json!({ "text": text })) + .await?; + Ok(BrowserToolOutput::Text(format!("filled {node_id}"))) +} + +pub(crate) async fn node_center(client: &CdpClient, backend_node_id: u64) -> Result<(f64, f64)> { + let model = client + .call( + "DOM.getBoxModel", + json!({ "backendNodeId": backend_node_id }), + ) + .await?; + let quad = model + .pointer("/model/content") + .or_else(|| model.pointer("/model/border")) + .and_then(Value::as_array) + .context("node has no visible box model")?; + anyhow::ensure!(quad.len() == 8, "node box model has an invalid quad"); + let coordinates = quad + .iter() + .map(|coordinate| { + coordinate + .as_f64() + .context("node box model coordinate is not numeric") + }) + .collect::>>()?; + let x = coordinates.iter().step_by(2).sum::() / 4.0; + let y = coordinates.iter().skip(1).step_by(2).sum::() / 4.0; + Ok((x, y)) +} + +pub(crate) async fn node_is_attached(client: &CdpClient, backend_node_id: u64) -> Result { + match client + .call( + "DOM.describeNode", + json!({ "backendNodeId": backend_node_id, "depth": 0 }), + ) + .await + { + Ok(_) => Ok(true), + Err(error) + if error.to_string().contains("Could not find node") + || error.to_string().contains("No node with given id") => + { + Ok(false) + } + Err(error) => Err(error), + } +} + +fn ax_value(node: &Value, field: &str) -> String { + node.pointer(&format!("/{field}/value")) + .and_then(Value::as_str) + .unwrap_or_default() + .to_string() +} + +fn ax_property<'a>(node: &'a Value, name: &str) -> Option<&'a Value> { + node.get("properties") + .and_then(Value::as_array)? + .iter() + .find(|property| property.get("name").and_then(Value::as_str) == Some(name))? + .pointer("/value/value") +} + +fn is_actionable_role(role: &str) -> bool { + matches!( + role, + "button" + | "checkbox" + | "combobox" + | "link" + | "menuitem" + | "option" + | "radio" + | "searchbox" + | "slider" + | "spinbutton" + | "switch" + | "tab" + | "textbox" + | "treeitem" + ) +} + +fn is_editable_role(role: &str) -> bool { + matches!(role, "combobox" | "searchbox" | "spinbutton" | "textbox") +} + +fn clipped(text: &str, max_chars: usize) -> String { + text.chars().take(max_chars).collect() +} diff --git a/codex-rs/terminal-browser/src/actions.rs b/codex-rs/terminal-browser/src/actions.rs new file mode 100644 index 0000000000..2eb3309070 --- /dev/null +++ b/codex-rs/terminal-browser/src/actions.rs @@ -0,0 +1,268 @@ +use anyhow::Context; +use anyhow::Result; +use serde_json::Value; +use serde_json::json; + +use crate::cdp::CdpClient; +use crate::input::BrowserKeyInput; +use crate::input::BrowserMouseInput; +use crate::input::BrowserMouseKind; +use crate::scripts; + +const MAX_SCREENSHOT_BYTES: usize = 4 * 1024 * 1024; +const MAX_SNAPSHOT_OUTPUT_BYTES: usize = 8 * 1024; +const MAX_SNAPSHOT_URL_CHARS: usize = 2_048; +const MAX_SNAPSHOT_TITLE_CHARS: usize = 512; +const MAX_SNAPSHOT_TEXT_CHARS: usize = 6_000; + +#[derive(Default)] +pub(crate) struct HumanMouseDispatchState { + viewport: Option<(u16, u16, f64, f64)>, + buttons: u8, +} + +pub(crate) struct PageMetadata { + pub(crate) url: Option, + pub(crate) title: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum BrowserToolOutput { + Text(String), + ImageDataUrl(String), +} + +pub(crate) async fn page_metadata(client: &CdpClient) -> Result { + let metadata = client + .evaluate("({ url: location.href, title: document.title })") + .await?; + Ok(PageMetadata { + url: metadata + .get("url") + .and_then(Value::as_str) + .map(|url| clipped(url, MAX_SNAPSHOT_URL_CHARS)), + title: metadata + .get("title") + .and_then(Value::as_str) + .map(|title| clipped(title, MAX_SNAPSHOT_TITLE_CHARS)), + }) +} + +pub(crate) async fn press(client: &CdpClient, key: &str) -> Result { + anyhow::ensure!(!key.is_empty(), "key must not be empty"); + anyhow::ensure!(key.chars().count() <= 32, "key is too long"); + let code = scripts::key_code(key); + let text = if key.chars().count() == 1 { key } else { "" }; + client + .call( + "Input.dispatchKeyEvent", + json!({ + "type": "rawKeyDown", + "key": key, + "code": code, + "text": text, + "unmodifiedText": text, + }), + ) + .await?; + client + .call( + "Input.dispatchKeyEvent", + json!({ "type": "keyUp", "key": key, "code": code }), + ) + .await?; + Ok(BrowserToolOutput::Text(format!("pressed {key}"))) +} + +pub(crate) async fn scroll( + client: &CdpClient, + delta_x: i64, + delta_y: i64, +) -> Result { + let result = client + .evaluate(&scripts::scroll_expression(delta_x, delta_y)) + .await?; + Ok(BrowserToolOutput::Text(serde_json::to_string(&result)?)) +} + +pub(crate) async fn screenshot(client: &CdpClient) -> Result { + let result = client + .call( + "Page.captureScreenshot", + json!({ "format": "png", "captureBeyondViewport": false }), + ) + .await?; + let data = result + .get("data") + .and_then(Value::as_str) + .context("screenshot response did not include image data")?; + let decoded_size_estimate = data.len().saturating_mul(/*rhs*/ 3) / 4; + anyhow::ensure!( + decoded_size_estimate <= MAX_SCREENSHOT_BYTES, + "screenshot exceeds the 4 MiB tool-output limit" + ); + Ok(BrowserToolOutput::ImageDataUrl(format!( + "data:image/png;base64,{data}" + ))) +} + +pub(crate) async fn dispatch_human_key(client: &CdpClient, input: &BrowserKeyInput) -> Result<()> { + let text = input.text.as_deref().unwrap_or_default(); + let modifiers = input.modifiers.cdp_mask(); + client + .call( + "Input.dispatchKeyEvent", + json!({ + "type": "rawKeyDown", + "key": input.key, + "code": input.code, + "text": text, + "unmodifiedText": text, + "modifiers": modifiers, + }), + ) + .await?; + client + .call( + "Input.dispatchKeyEvent", + json!({ + "type": "keyUp", + "key": input.key, + "code": input.code, + "modifiers": modifiers, + }), + ) + .await?; + Ok(()) +} + +pub(crate) async fn insert_human_text(client: &CdpClient, text: &str) -> Result<()> { + client + .call("Input.insertText", json!({ "text": text })) + .await?; + Ok(()) +} + +pub(crate) async fn dispatch_human_mouse( + client: &CdpClient, + input: BrowserMouseInput, + state: &mut HumanMouseDispatchState, +) -> Result<()> { + anyhow::ensure!( + input.viewport_cols > 0 && input.viewport_rows > 0, + "browser viewport must be non-zero" + ); + let (width, height) = if let Some((cols, rows, width, height)) = state.viewport + && cols == input.viewport_cols + && rows == input.viewport_rows + { + (width, height) + } else { + let metrics = client.call("Page.getLayoutMetrics", json!({})).await?; + let width = metrics + .pointer("/cssLayoutViewport/clientWidth") + .or_else(|| metrics.pointer("/layoutViewport/clientWidth")) + .and_then(Value::as_f64) + .context("browser layout metrics omitted viewport width")?; + let height = metrics + .pointer("/cssLayoutViewport/clientHeight") + .or_else(|| metrics.pointer("/layoutViewport/clientHeight")) + .and_then(Value::as_f64) + .context("browser layout metrics omitted viewport height")?; + state.viewport = Some((input.viewport_cols, input.viewport_rows, width, height)); + (width, height) + }; + let x = f64::from(input.column) * width / f64::from(input.viewport_cols); + let y = f64::from(input.row) * height / f64::from(input.viewport_rows); + let modifiers = input.modifiers.cdp_mask(); + let (event_type, delta_x, delta_y) = match input.kind { + BrowserMouseKind::Move => ("mouseMoved", 0.0, 0.0), + BrowserMouseKind::Down => ("mousePressed", 0.0, 0.0), + BrowserMouseKind::Up => ("mouseReleased", 0.0, 0.0), + BrowserMouseKind::Wheel { delta_x, delta_y } => ("mouseWheel", delta_x, delta_y), + }; + match input.kind { + BrowserMouseKind::Down => state.buttons |= input.button.cdp_buttons_mask(), + BrowserMouseKind::Up => state.buttons &= !input.button.cdp_buttons_mask(), + BrowserMouseKind::Move | BrowserMouseKind::Wheel { .. } => {} + } + client + .call( + "Input.dispatchMouseEvent", + json!({ + "type": event_type, + "x": x, + "y": y, + "button": input.button.as_cdp(), + "buttons": state.buttons, + "modifiers": modifiers, + "clickCount": if matches!(input.kind, BrowserMouseKind::Down | BrowserMouseKind::Up) { 1 } else { 0 }, + "deltaX": delta_x, + "deltaY": delta_y, + }), + ) + .await?; + Ok(()) +} + +pub(crate) fn bounded_snapshot_json(mut snapshot: Value) -> Result { + truncate_string_field(&mut snapshot, "url", MAX_SNAPSHOT_URL_CHARS); + truncate_string_field(&mut snapshot, "title", MAX_SNAPSHOT_TITLE_CHARS); + truncate_string_field(&mut snapshot, "text", MAX_SNAPSHOT_TEXT_CHARS); + let mut output = serde_json::to_string_pretty(&snapshot)?; + if output.len() <= MAX_SNAPSHOT_OUTPUT_BYTES { + return Ok(output); + } + + if let Some(object) = snapshot.as_object_mut() { + object.insert("truncated".to_string(), Value::Bool(true)); + if let Some(text) = object.get("text").and_then(Value::as_str) { + object.insert( + "text".to_string(), + Value::String(text.chars().take(/*n*/ 8_000).collect()), + ); + } + } + loop { + output = serde_json::to_string_pretty(&snapshot)?; + if output.len() <= MAX_SNAPSHOT_OUTPUT_BYTES { + return Ok(output); + } + let removed = snapshot + .get_mut("nodes") + .and_then(Value::as_array_mut) + .and_then(Vec::pop) + .is_some(); + if !removed { + break; + } + } + + Ok(serde_json::to_string_pretty(&json!({ + "url": snapshot.get("url").cloned().unwrap_or(Value::Null), + "title": snapshot.get("title").cloned().unwrap_or(Value::Null), + "truncated": true, + "text": "Snapshot exceeded the output limit. Narrow the page state and take another snapshot." + }))?) +} + +fn truncate_string_field(value: &mut Value, field: &str, max_chars: usize) { + let Some(text) = value + .get(field) + .and_then(Value::as_str) + .map(ToOwned::to_owned) + else { + return; + }; + if let Some(object) = value.as_object_mut() { + object.insert(field.to_string(), Value::String(clipped(&text, max_chars))); + } +} + +fn clipped(text: &str, max_chars: usize) -> String { + text.chars().take(max_chars).collect() +} + +#[cfg(test)] +#[path = "actions_tests.rs"] +mod tests; diff --git a/codex-rs/terminal-browser/src/actions_tests.rs b/codex-rs/terminal-browser/src/actions_tests.rs new file mode 100644 index 0000000000..c50ca376a3 --- /dev/null +++ b/codex-rs/terminal-browser/src/actions_tests.rs @@ -0,0 +1,311 @@ +use futures::SinkExt; +use futures::StreamExt; +use pretty_assertions::assert_eq; +use serde_json::Value; +use serde_json::json; +use tokio::net::TcpListener; +use tokio::task::JoinHandle; +use tokio_tungstenite::accept_async; +use tokio_tungstenite::tungstenite::Message; + +use super::BrowserToolOutput; +use super::HumanMouseDispatchState; +use super::dispatch_human_mouse; +use crate::accessibility::click; +use crate::accessibility::snapshot; +use crate::cdp::CdpClient; +use crate::handles::BrowserHandles; +use crate::input::BrowserInputModifiers; +use crate::input::BrowserMouseButton; +use crate::input::BrowserMouseInput; +use crate::input::BrowserMouseKind; +use crate::navigation::LoadState; +use crate::navigation::NavigateRequest; +use crate::navigation::NavigationAction; +use crate::navigation::navigate; +use crate::navigation::navigate_request; + +type TestSocket = tokio_tungstenite::WebSocketStream; + +async fn test_server( + handler: impl FnOnce(TestSocket) -> JoinHandle<()> + Send + 'static, +) -> (String, JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind test websocket"); + let address = listener.local_addr().expect("test websocket address"); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.expect("accept test websocket"); + let socket = accept_async(stream).await.expect("upgrade test websocket"); + handler(socket).await.expect("test handler task"); + }); + (format!("ws://{address}"), server) +} + +async fn request(socket: &mut TestSocket) -> Value { + let Message::Text(text) = socket + .next() + .await + .expect("request message") + .expect("valid request message") + else { + panic!("expected text request"); + }; + serde_json::from_str(text.as_str()).expect("decode request") +} + +async fn respond(socket: &mut TestSocket, request: &Value, result: Value) { + socket + .send(Message::Text( + json!({ "id": request["id"], "result": result }) + .to_string() + .into(), + )) + .await + .expect("send response"); +} + +async fn initialize(socket: &mut TestSocket) { + for method in [ + "Page.enable", + "Runtime.enable", + "DOM.enable", + "Accessibility.enable", + "Page.setLifecycleEventsEnabled", + ] { + let request = request(socket).await; + assert_eq!(request["method"], method); + respond(socket, &request, json!({})).await; + } +} + +#[tokio::test] +async fn accessibility_snapshot_creates_host_owned_handles_and_redacts_editable_values() { + let (url, server) = test_server(|mut socket| { + tokio::spawn(async move { + initialize(&mut socket).await; + let tree = request(&mut socket).await; + assert_eq!(tree["method"], "Accessibility.getFullAXTree"); + respond( + &mut socket, + &tree, + json!({ + "nodes": [ + { + "ignored": false, + "role": { "value": "textbox" }, + "name": { "value": "Password" }, + "value": { "value": "super-secret" }, + "backendDOMNodeId": 7 + }, + { + "ignored": false, + "role": { "value": "button" }, + "name": { "value": "Submit" }, + "backendDOMNodeId": 8 + }, + { + "ignored": false, + "role": { "value": "StaticText" }, + "name": { "value": "Hello" } + } + ] + }), + ) + .await; + let metadata = request(&mut socket).await; + assert_eq!(metadata["method"], "Runtime.evaluate"); + respond( + &mut socket, + &metadata, + json!({ "result": { "value": { "url": "https://example.test", "title": "Example" } } }), + ) + .await; + }) + }) + .await; + let client = CdpClient::connect(&url).await.expect("connect client"); + let mut handles = BrowserHandles::default(); + + let BrowserToolOutput::Text(output) = snapshot(&client, &mut handles).await.expect("snapshot") + else { + panic!("expected text snapshot"); + }; + let output: Value = serde_json::from_str(&output).expect("snapshot JSON"); + + assert_eq!(output["nodes"][0]["value"], ""); + assert_eq!(output["nodes"][1]["name"], "Submit"); + assert_eq!(output["text"], "Hello"); + let node_id = output["nodes"][1]["nodeId"] + .as_str() + .expect("button node id"); + assert_eq!(handles.resolve(node_id).expect("resolve button"), 8); + server.await.expect("server task"); +} + +#[tokio::test] +async fn native_click_uses_the_accessibility_handle_box_center() { + let (url, server) = test_server(|mut socket| { + tokio::spawn(async move { + initialize(&mut socket).await; + let scroll = request(&mut socket).await; + assert_eq!(scroll["method"], "DOM.scrollIntoViewIfNeeded"); + respond(&mut socket, &scroll, json!({})).await; + let box_model = request(&mut socket).await; + assert_eq!(box_model["method"], "DOM.getBoxModel"); + respond( + &mut socket, + &box_model, + json!({ "model": { "content": [10, 20, 30, 20, 30, 40, 10, 40] } }), + ) + .await; + for expected_type in ["mouseMoved", "mousePressed", "mouseReleased"] { + let input = request(&mut socket).await; + assert_eq!(input["method"], "Input.dispatchMouseEvent"); + assert_eq!(input["params"]["type"], expected_type); + assert_eq!(input["params"]["x"], 20.0); + assert_eq!(input["params"]["y"], 30.0); + respond(&mut socket, &input, json!({})).await; + } + }) + }) + .await; + let client = CdpClient::connect(&url).await.expect("connect client"); + let mut handles = BrowserHandles::default(); + let node_id = handles.insert(/*backend_node_id*/ 9); + + let output = click(&client, &handles, &node_id) + .await + .expect("native click"); + + assert_eq!( + output, + BrowserToolOutput::Text(format!("clicked {node_id}")) + ); + server.await.expect("server task"); +} + +#[tokio::test] +async fn human_mouse_caches_metrics_and_preserves_drag_buttons() { + let (url, server) = test_server(|mut socket| { + tokio::spawn(async move { + initialize(&mut socket).await; + let metrics = request(&mut socket).await; + assert_eq!(metrics["method"], "Page.getLayoutMetrics"); + respond( + &mut socket, + &metrics, + json!({ "cssLayoutViewport": { "clientWidth": 800, "clientHeight": 600 } }), + ) + .await; + for (expected_type, expected_buttons) in + [("mousePressed", 1), ("mouseMoved", 1), ("mouseReleased", 0)] + { + let input = request(&mut socket).await; + assert_eq!(input["method"], "Input.dispatchMouseEvent"); + assert_eq!(input["params"]["type"], expected_type); + assert_eq!(input["params"]["buttons"], expected_buttons); + respond(&mut socket, &input, json!({})).await; + } + }) + }) + .await; + let client = CdpClient::connect(&url).await.expect("connect client"); + let mut state = HumanMouseDispatchState::default(); + for kind in [ + BrowserMouseKind::Down, + BrowserMouseKind::Move, + BrowserMouseKind::Up, + ] { + dispatch_human_mouse( + &client, + BrowserMouseInput { + kind, + button: BrowserMouseButton::Left, + column: 10, + row: 5, + viewport_cols: 80, + viewport_rows: 24, + modifiers: BrowserInputModifiers::default(), + }, + &mut state, + ) + .await + .expect("dispatch mouse input"); + } + server.await.expect("server task"); +} + +#[tokio::test] +async fn navigation_waits_for_the_cdp_lifecycle_event() { + let (url, server) = test_server(|mut socket| { + tokio::spawn(async move { + initialize(&mut socket).await; + let navigation = request(&mut socket).await; + assert_eq!(navigation["method"], "Page.navigate"); + respond( + &mut socket, + &navigation, + json!({ "frameId": "frame", "loaderId": "loader" }), + ) + .await; + socket + .send(Message::Text( + json!({ + "method": "Page.lifecycleEvent", + "params": { + "frameId": "frame", + "loaderId": "loader", + "name": "DOMContentLoaded", + "timestamp": 1, + } + }) + .to_string() + .into(), + )) + .await + .expect("send lifecycle event"); + }) + }) + .await; + let client = CdpClient::connect(&url).await.expect("connect client"); + + navigate(&client, "https://example.test") + .await + .expect("navigation"); + + server.await.expect("server task"); +} + +#[tokio::test] +async fn navigation_timeout_bounds_the_cdp_command_and_wait() { + let (release_tx, release_rx) = tokio::sync::oneshot::channel(); + let (url, server) = test_server(|mut socket| { + tokio::spawn(async move { + initialize(&mut socket).await; + let navigation = request(&mut socket).await; + assert_eq!(navigation["method"], "Page.navigate"); + release_rx.await.expect("release server"); + }) + }) + .await; + let client = CdpClient::connect(&url).await.expect("connect client"); + let started = std::time::Instant::now(); + + let error = navigate_request( + &client, + &NavigateRequest { + action: NavigationAction::Goto, + url: Some("https://example.test".to_string()), + wait_until: LoadState::Load, + timeout_ms: Some(/*value*/ 10), + }, + ) + .await + .expect_err("navigation must time out"); + + assert!(error.to_string().contains("navigation_timeout")); + assert!(started.elapsed() < std::time::Duration::from_millis(/*millis*/ 500)); + release_tx.send(()).expect("release server"); + server.await.expect("server task"); +} diff --git a/codex-rs/terminal-browser/src/cdp.rs b/codex-rs/terminal-browser/src/cdp.rs new file mode 100644 index 0000000000..2e14fc5402 --- /dev/null +++ b/codex-rs/terminal-browser/src/cdp.rs @@ -0,0 +1,254 @@ +use std::collections::HashMap; +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::PoisonError; +use std::sync::atomic::AtomicU64; +use std::sync::atomic::Ordering; +use std::time::Duration; + +use anyhow::Context; +use anyhow::Result; +use anyhow::bail; +use futures::SinkExt; +use futures::StreamExt; +use serde_json::Value; +use serde_json::json; +use tokio::net::TcpStream; +use tokio::sync::broadcast; +use tokio::sync::mpsc; +use tokio::sync::oneshot; +use tokio::time::timeout; +use tokio_tungstenite::MaybeTlsStream; +use tokio_tungstenite::WebSocketStream; +use tokio_tungstenite::connect_async; +use tokio_tungstenite::tungstenite::Message; + +type Socket = WebSocketStream>; +type PendingCalls = Arc>>>>; + +const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); +const CALL_TIMEOUT: Duration = Duration::from_secs(15); +const OUTBOUND_CAPACITY: usize = 64; +const EVENT_CAPACITY: usize = 256; +const MAX_EVENT_BYTES: usize = 16 * 1024; + +#[derive(Clone, Debug, PartialEq)] +pub(crate) enum CdpEvent { + Message(Value), + Disconnected(String), +} + +#[derive(Clone)] +pub(crate) struct CdpClient { + outbound: mpsc::Sender, + pending: PendingCalls, + events: broadcast::Sender, + next_id: Arc, + call_timeout: Duration, +} + +impl CdpClient { + pub(crate) async fn connect(websocket_url: &str) -> Result { + Self::connect_with_call_timeout(websocket_url, CALL_TIMEOUT).await + } + + async fn connect_with_call_timeout( + websocket_url: &str, + call_timeout: Duration, + ) -> Result { + let (socket, _) = timeout(CONNECT_TIMEOUT, connect_async(websocket_url)) + .await + .context("timed out connecting to Carbonyl DevTools")??; + let (outbound, outbound_rx) = mpsc::channel(OUTBOUND_CAPACITY); + let (events, _) = broadcast::channel(EVENT_CAPACITY); + let client = Self { + outbound, + pending: Arc::new(Mutex::new(HashMap::new())), + events, + next_id: Arc::new(AtomicU64::new(/*v*/ 1)), + call_timeout, + }; + tokio::spawn(run_pump( + socket, + outbound_rx, + client.pending.clone(), + client.events.clone(), + )); + client.call("Page.enable", json!({})).await?; + client.call("Runtime.enable", json!({})).await?; + client.call("DOM.enable", json!({})).await?; + client.call("Accessibility.enable", json!({})).await?; + client + .call("Page.setLifecycleEventsEnabled", json!({ "enabled": true })) + .await?; + Ok(client) + } + + pub(crate) fn subscribe_events(&self) -> broadcast::Receiver { + self.events.subscribe() + } + + pub(crate) async fn call(&self, method: &str, params: Value) -> Result { + let id = self.next_id.fetch_add(/*val*/ 1, Ordering::Relaxed); + let request = json!({ + "id": id, + "method": method, + "params": params, + }); + let (response_tx, response_rx) = oneshot::channel(); + self.pending + .lock() + .unwrap_or_else(PoisonError::into_inner) + .insert(id, response_tx); + let _pending_call = PendingCallGuard { + pending: self.pending.clone(), + id, + }; + + let result = timeout(self.call_timeout, async { + if self + .outbound + .send(Message::Text(request.to_string().into())) + .await + .is_err() + { + bail!("Carbonyl closed the DevTools connection"); + } + response_rx + .await + .context("Carbonyl closed the DevTools connection")? + .map_err(anyhow::Error::msg) + }) + .await; + + let response = match result { + Ok(response) => response?, + Err(_) => { + bail!("timed out waiting for CDP {method}"); + } + }; + if let Some(error) = response.get("error") { + bail!("CDP {method} failed: {error}"); + } + Ok(response.get("result").cloned().unwrap_or(Value::Null)) + } + + pub(crate) async fn evaluate(&self, expression: &str) -> Result { + let result = self + .call( + "Runtime.evaluate", + json!({ + "expression": expression, + "returnByValue": true, + "awaitPromise": true, + }), + ) + .await?; + if let Some(details) = result.get("exceptionDetails") { + bail!("browser script failed: {details}"); + } + Ok(result + .pointer("/result/value") + .cloned() + .unwrap_or(Value::Null)) + } +} + +struct PendingCallGuard { + pending: PendingCalls, + id: u64, +} + +impl Drop for PendingCallGuard { + fn drop(&mut self) { + self.pending + .lock() + .unwrap_or_else(PoisonError::into_inner) + .remove(&self.id); + } +} + +async fn run_pump( + mut socket: Socket, + mut outbound: mpsc::Receiver, + pending: PendingCalls, + events: broadcast::Sender, +) { + let disconnect_reason = loop { + tokio::select! { + outgoing = outbound.recv() => match outgoing { + Some(message) => { + if let Err(error) = socket.send(message).await { + break format!("failed to send DevTools message: {error}"); + } + } + None => { + let _ = socket.close(None).await; + break "DevTools client closed".to_string(); + } + }, + incoming = socket.next() => match incoming { + Some(Ok(Message::Text(text))) => { + let response = match serde_json::from_str::(text.as_str()) { + Ok(response) => response, + Err(error) => break format!("failed to decode DevTools message: {error}"), + }; + dispatch_message(response, &pending, &events); + } + Some(Ok(Message::Ping(bytes))) => { + if let Err(error) = socket.send(Message::Pong(bytes)).await { + break format!("failed to reply to DevTools ping: {error}"); + } + } + Some(Ok(Message::Close(frame))) => { + break format!("Carbonyl closed the DevTools connection: {frame:?}"); + } + Some(Ok(Message::Binary(_) | Message::Pong(_) | Message::Frame(_))) => {} + Some(Err(error)) => break format!("DevTools connection failed: {error}"), + None => break "Carbonyl closed the DevTools connection".to_string(), + } + } + }; + + fail_pending(&pending, &disconnect_reason); + let _ = events.send(CdpEvent::Disconnected(disconnect_reason)); +} + +fn dispatch_message(response: Value, pending: &PendingCalls, events: &broadcast::Sender) { + let response_tx = response.get("id").and_then(Value::as_u64).and_then(|id| { + pending + .lock() + .unwrap_or_else(PoisonError::into_inner) + .remove(&id) + }); + if let Some(response_tx) = response_tx { + let _ = response_tx.send(Ok(response)); + } else if should_broadcast_event(&response) { + let _ = events.send(CdpEvent::Message(response)); + } +} + +fn should_broadcast_event(message: &Value) -> bool { + let Some(method) = message.get("method").and_then(Value::as_str) else { + return false; + }; + matches!( + method, + "Page.domContentEventFired" + | "Page.frameNavigated" + | "Page.lifecycleEvent" + | "Page.loadEventFired" + | "Page.navigatedWithinDocument" + ) && serde_json::to_vec(message).is_ok_and(|encoded| encoded.len() <= MAX_EVENT_BYTES) +} + +fn fail_pending(pending: &PendingCalls, reason: &str) { + let pending = std::mem::take(&mut *pending.lock().unwrap_or_else(PoisonError::into_inner)); + for response_tx in pending.into_values() { + let _ = response_tx.send(Err(reason.to_string())); + } +} + +#[cfg(test)] +#[path = "cdp_tests.rs"] +mod tests; diff --git a/codex-rs/terminal-browser/src/cdp_tests.rs b/codex-rs/terminal-browser/src/cdp_tests.rs new file mode 100644 index 0000000000..19c16ca70d --- /dev/null +++ b/codex-rs/terminal-browser/src/cdp_tests.rs @@ -0,0 +1,277 @@ +use std::time::Duration; + +use futures::SinkExt; +use futures::StreamExt; +use pretty_assertions::assert_eq; +use serde_json::Value; +use serde_json::json; +use tokio::net::TcpListener; +use tokio::task::JoinHandle; +use tokio_tungstenite::accept_async; +use tokio_tungstenite::tungstenite::Message; + +use super::CdpClient; +use super::CdpEvent; + +async fn test_server( + handler: impl FnOnce(tokio_tungstenite::WebSocketStream) -> JoinHandle<()> + + Send + + 'static, +) -> (String, JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind test websocket"); + let address = listener.local_addr().expect("test websocket address"); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.expect("accept test websocket"); + let socket = accept_async(stream).await.expect("upgrade test websocket"); + handler(socket).await.expect("test handler task"); + }); + (format!("ws://{address}"), server) +} + +async fn receive_request( + socket: &mut tokio_tungstenite::WebSocketStream, +) -> Value { + let Message::Text(text) = socket + .next() + .await + .expect("request message") + .expect("valid request message") + else { + panic!("expected text request"); + }; + serde_json::from_str(text.as_str()).expect("decode request") +} + +async fn respond( + socket: &mut tokio_tungstenite::WebSocketStream, + request: &Value, + result: Value, +) { + socket + .send(Message::Text( + json!({ "id": request["id"], "result": result }) + .to_string() + .into(), + )) + .await + .expect("send response"); +} + +async fn complete_initialization( + socket: &mut tokio_tungstenite::WebSocketStream, +) { + for expected_method in [ + "Page.enable", + "Runtime.enable", + "DOM.enable", + "Accessibility.enable", + "Page.setLifecycleEventsEnabled", + ] { + let request = receive_request(socket).await; + assert_eq!(request["method"], expected_method); + respond(socket, &request, json!({})).await; + } +} + +#[tokio::test] +async fn lifecycle_messages_are_broadcast_without_leaking_unknown_responses() { + let (url, server) = test_server(|mut socket| { + tokio::spawn(async move { + complete_initialization(&mut socket).await; + let request = receive_request(&mut socket).await; + socket + .send(Message::Text( + json!({ "method": "Page.loadEventFired", "params": { "timestamp": 1 } }) + .to_string() + .into(), + )) + .await + .expect("send notification"); + socket + .send(Message::Text( + json!({ "id": 999, "result": { "late": true } }) + .to_string() + .into(), + )) + .await + .expect("send unknown response"); + respond(&mut socket, &request, json!({ "ok": true })).await; + }) + }) + .await; + let client = CdpClient::connect(&url).await.expect("connect client"); + let mut events = client.subscribe_events(); + + let result = client + .call("Test.command", json!({})) + .await + .expect("command response"); + + assert_eq!(result, json!({ "ok": true })); + assert_eq!( + events.recv().await.expect("notification"), + CdpEvent::Message(json!({ "method": "Page.loadEventFired", "params": { "timestamp": 1 } })) + ); + assert!(matches!( + events.recv().await.expect("disconnect event"), + CdpEvent::Disconnected(_) + )); + server.await.expect("server task"); +} + +#[tokio::test] +async fn cdp_errors_keep_the_method_name() { + let (url, server) = test_server(|mut socket| { + tokio::spawn(async move { + complete_initialization(&mut socket).await; + let request = receive_request(&mut socket).await; + socket + .send(Message::Text( + json!({ "id": request["id"], "error": { "code": -1, "message": "nope" } }) + .to_string() + .into(), + )) + .await + .expect("send error response"); + }) + }) + .await; + let client = CdpClient::connect(&url).await.expect("connect client"); + + let error = client + .call("Test.failure", json!({})) + .await + .expect_err("command should fail"); + + assert!(error.to_string().contains("CDP Test.failure failed")); + server.await.expect("server task"); +} + +#[tokio::test] +async fn disconnect_fails_pending_calls_and_emits_an_event() { + let (url, server) = test_server(|mut socket| { + tokio::spawn(async move { + complete_initialization(&mut socket).await; + let _request = receive_request(&mut socket).await; + socket.close(None).await.expect("close websocket"); + }) + }) + .await; + let client = CdpClient::connect(&url).await.expect("connect client"); + let mut events = client.subscribe_events(); + + let error = client + .call("Test.disconnect", json!({})) + .await + .expect_err("disconnect should fail the call"); + + assert!(error.to_string().contains("closed the DevTools connection")); + assert!(matches!( + events.recv().await.expect("disconnect event"), + CdpEvent::Disconnected(_) + )); + server.await.expect("server task"); +} + +#[tokio::test] +async fn late_responses_after_timeout_are_discarded() { + let (release_tx, release_rx) = tokio::sync::oneshot::channel(); + let (url, server) = test_server(|mut socket| { + tokio::spawn(async move { + complete_initialization(&mut socket).await; + let request = receive_request(&mut socket).await; + release_rx.await.expect("release late response"); + respond(&mut socket, &request, json!({ "late": true })).await; + }) + }) + .await; + let client = CdpClient::connect_with_call_timeout(&url, Duration::from_millis(20)) + .await + .expect("connect client"); + let mut events = client.subscribe_events(); + + let error = client + .call("Test.timeout", json!({})) + .await + .expect_err("command should time out"); + assert!(error.to_string().contains("timed out waiting for CDP")); + release_tx.send(()).expect("release server response"); + assert!(matches!( + events.recv().await.expect("disconnect event"), + CdpEvent::Disconnected(_) + )); + server.await.expect("server task"); +} + +#[tokio::test] +async fn cancelling_a_call_removes_its_pending_entry() { + let (seen_tx, seen_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = tokio::sync::oneshot::channel(); + let (url, server) = test_server(|mut socket| { + tokio::spawn(async move { + complete_initialization(&mut socket).await; + let _request = receive_request(&mut socket).await; + seen_tx.send(()).expect("mark request seen"); + release_rx.await.expect("release server"); + }) + }) + .await; + let client = CdpClient::connect(&url).await.expect("connect client"); + let caller = { + let client = client.clone(); + tokio::spawn(async move { client.call("Test.cancel", json!({})).await }) + }; + seen_rx.await.expect("request seen"); + + caller.abort(); + let _ = caller.await; + + assert!( + client + .pending + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .is_empty() + ); + release_tx.send(()).expect("release server"); + server.await.expect("server task"); +} + +#[tokio::test] +async fn concurrent_calls_match_reverse_order_responses() { + let (url, server) = test_server(|mut socket| { + tokio::spawn(async move { + complete_initialization(&mut socket).await; + let first = receive_request(&mut socket).await; + let second = receive_request(&mut socket).await; + respond(&mut socket, &second, json!({ "order": 2 })).await; + socket + .send(Message::Text( + json!({ "method": "Page.loadEventFired", "params": { "timestamp": 1 } }) + .to_string() + .into(), + )) + .await + .expect("send lifecycle event"); + respond(&mut socket, &first, json!({ "order": 1 })).await; + }) + }) + .await; + let client = CdpClient::connect(&url).await.expect("connect client"); + let mut events = client.subscribe_events(); + + let (first, second) = tokio::join!( + client.call("Test.first", json!({})), + client.call("Test.second", json!({})) + ); + + assert_eq!(first.expect("first response"), json!({ "order": 1 })); + assert_eq!(second.expect("second response"), json!({ "order": 2 })); + assert!(matches!( + events.recv().await.expect("lifecycle event"), + CdpEvent::Message(_) + )); + server.await.expect("server task"); +} diff --git a/codex-rs/terminal-browser/src/devtools.rs b/codex-rs/terminal-browser/src/devtools.rs new file mode 100644 index 0000000000..ec960d5a70 --- /dev/null +++ b/codex-rs/terminal-browser/src/devtools.rs @@ -0,0 +1,141 @@ +use std::path::Path; +use std::time::Duration; +use std::time::Instant; + +use anyhow::Context; +use anyhow::Result; +use reqwest::Client; +use serde::Deserialize; +use url::Host; +use url::Url; + +use crate::cdp::CdpClient; + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct DevtoolsTarget { + #[serde(rename = "type")] + kind: String, + #[serde(default)] + title: String, + web_socket_debugger_url: Option, +} + +pub(crate) struct PageTarget { + pub(crate) title: String, + pub(crate) websocket_url: String, +} + +pub(crate) fn prepare_devtools_active_port(profile: &Path) -> Result<()> { + let active_port_path = profile.join("DevToolsActivePort"); + let metadata = match std::fs::symlink_metadata(&active_port_path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => { + return Err(error).context("inspect Carbonyl DevToolsActivePort before startup"); + } + }; + anyhow::ensure!( + !metadata.file_type().is_symlink(), + "refusing symbolic-link Carbonyl DevToolsActivePort" + ); + anyhow::ensure!( + metadata.is_file(), + "refusing non-file Carbonyl DevToolsActivePort" + ); + std::fs::remove_file(&active_port_path) + .context("remove stale Carbonyl DevToolsActivePort before startup") +} + +pub(crate) async fn discover_page_target(profile: &Path) -> Result { + let client = Client::builder() + .no_proxy() + .timeout(Duration::from_secs(/*secs*/ 2)) + .build()?; + let deadline = Instant::now() + Duration::from_secs(/*secs*/ 12); + let active_port_path = profile.join("DevToolsActivePort"); + let port = loop { + match std::fs::read_to_string(&active_port_path) { + Ok(contents) => { + let port = contents + .lines() + .next() + .context("DevToolsActivePort did not contain a port")? + .parse::() + .context("DevToolsActivePort contained an invalid port")?; + anyhow::ensure!(port != 0, "DevToolsActivePort contained port zero"); + break port; + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(error).context("read Carbonyl DevToolsActivePort"); + } + } + anyhow::ensure!( + Instant::now() < deadline, + "timed out waiting for Carbonyl DevToolsActivePort" + ); + tokio::time::sleep(Duration::from_millis(/*millis*/ 100)).await; + }; + let endpoint = format!("http://127.0.0.1:{port}/json/list"); + loop { + if let Ok(response) = client.get(&endpoint).send().await + && let Ok(targets) = response.json::>().await + { + for target in targets { + if target.kind != "page" { + continue; + } + let Some(websocket_url) = target.web_socket_debugger_url else { + continue; + }; + return Ok(PageTarget { + title: target.title, + websocket_url: validated_websocket_url(&websocket_url, port)?, + }); + } + } + anyhow::ensure!( + Instant::now() < deadline, + "timed out waiting for Carbonyl DevTools on {endpoint}" + ); + tokio::time::sleep(Duration::from_millis(/*millis*/ 100)).await; + } +} + +pub(crate) fn validated_websocket_url(websocket_url: &str, expected_port: u16) -> Result { + let parsed = Url::parse(websocket_url).context("parse Carbonyl DevTools WebSocket URL")?; + anyhow::ensure!(parsed.scheme() == "ws", "Carbonyl DevTools URL must use ws"); + let loopback = match parsed.host() { + Some(Host::Ipv4(address)) => address.is_loopback(), + Some(Host::Ipv6(address)) => address.is_loopback(), + Some(Host::Domain(_)) | None => false, + }; + anyhow::ensure!(loopback, "Carbonyl DevTools URL must use a loopback host"); + anyhow::ensure!( + parsed.port() == Some(expected_port), + "Carbonyl DevTools URL used an unexpected port" + ); + Ok(parsed.to_string()) +} + +pub(crate) async fn deny_downloads(client: &CdpClient) -> Result<()> { + if client + .call( + "Browser.setDownloadBehavior", + serde_json::json!({ "behavior": "deny" }), + ) + .await + .is_ok() + { + return Ok(()); + } + client + .call( + "Page.setDownloadBehavior", + serde_json::json!({ "behavior": "deny" }), + ) + .await + .context("disable Carbonyl downloads")?; + Ok(()) +} diff --git a/codex-rs/terminal-browser/src/diagnostics.rs b/codex-rs/terminal-browser/src/diagnostics.rs new file mode 100644 index 0000000000..775f8fa0c0 --- /dev/null +++ b/codex-rs/terminal-browser/src/diagnostics.rs @@ -0,0 +1,140 @@ +use std::path::Path; +use std::path::PathBuf; +use std::process::Stdio; +use std::time::Duration; + +use anyhow::Context; +use anyhow::Result; + +use crate::network::BrowserNetworkPolicy; +use crate::runtime::BrowserRuntime; +use crate::sandbox::BrowserLaunchContext; +use crate::sandbox::prepare_browser_launch; + +const VERSION_TIMEOUT: Duration = Duration::from_secs(3); +const SUPPORTED_VERSION: &str = "0.0.3"; +const MAX_VERSION_OUTPUT_CHARS: usize = 256; + +#[derive(Clone, Debug, PartialEq, Eq)] +/// Result of validating the external Carbonyl installation used by the experiment. +pub struct BrowserDoctorReport { + /// Whether the executable passed all launch compatibility checks. + pub healthy: bool, + /// Human-readable local diagnostic suitable for the `/browser doctor` surface. + pub summary: String, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct BrowserInstallation { + pub(crate) path: PathBuf, + pub(crate) version: String, +} + +pub(crate) async fn inspect_installation( + binary: &Path, + launch_context: &BrowserLaunchContext, +) -> Result { + let path = std::fs::canonicalize(binary).context("canonicalize Carbonyl executable")?; + ensure_executable(&path)?; + let runtime = BrowserRuntime::create(/*persistent_profile*/ None)?; + let network_policy = BrowserNetworkPolicy::Disabled; + let launch = prepare_browser_launch( + &path, + vec!["--version".to_string()], + &runtime.root, + &runtime.profile, + runtime.environment(&network_policy), + &network_policy, + launch_context, + )?; + let mut command = tokio::process::Command::new(&launch.program); + #[cfg(unix)] + if let Some(arg0) = launch.arg0.as_deref() { + command.arg0(arg0); + } + command + .args(&launch.args) + .current_dir(launch.cwd.as_path()) + .env_clear() + .envs(&launch.env) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(/*kill_on_drop*/ true); + let output = tokio::time::timeout(VERSION_TIMEOUT, command.output()) + .await + .context("Carbonyl version probe timed out")? + .context("run Carbonyl version probe")?; + anyhow::ensure!( + output.status.success(), + "Carbonyl version probe exited with {}", + output.status + ); + let version = first_nonempty_line(&output.stdout) + .or_else(|| first_nonempty_line(&output.stderr)) + .context("Carbonyl version probe returned no version")?; + anyhow::ensure!( + has_supported_version(&version), + "unsupported Carbonyl version `{version}`; this experiment currently requires {SUPPORTED_VERSION}" + ); + Ok(BrowserInstallation { path, version }) +} + +pub(crate) fn unavailable_report(reason: &str) -> BrowserDoctorReport { + BrowserDoctorReport { + healthy: false, + summary: format!("Carbonyl unavailable: {reason}"), + } +} + +pub(crate) fn installation_report(result: &Result) -> BrowserDoctorReport { + match result { + Ok(installation) => BrowserDoctorReport { + healthy: true, + summary: format!( + "Carbonyl is ready: {} ({}, {})", + installation.path.display(), + installation.version, + std::env::consts::ARCH + ), + }, + Err(error) => BrowserDoctorReport { + healthy: false, + summary: format!("Carbonyl validation failed: {error}"), + }, + } +} + +fn first_nonempty_line(bytes: &[u8]) -> Option { + String::from_utf8_lossy(bytes) + .lines() + .map(str::trim) + .find(|line| !line.is_empty()) + .map(|line| line.chars().take(MAX_VERSION_OUTPUT_CHARS).collect()) +} + +fn has_supported_version(version_line: &str) -> bool { + version_line.split_ascii_whitespace().last() == Some(SUPPORTED_VERSION) +} + +#[cfg(unix)] +fn ensure_executable(path: &Path) -> Result<()> { + use std::os::unix::fs::PermissionsExt; + + let mode = std::fs::metadata(path) + .context("read Carbonyl executable metadata")? + .permissions() + .mode(); + anyhow::ensure!(mode & 0o111 != 0, "Carbonyl path is not executable"); + Ok(()) +} + +#[cfg(not(unix))] +fn ensure_executable(path: &Path) -> Result<()> { + anyhow::ensure!(path.is_file(), "Carbonyl path is not an executable file"); + Ok(()) +} + +#[cfg(test)] +#[path = "diagnostics_tests.rs"] +mod tests; diff --git a/codex-rs/terminal-browser/src/diagnostics_tests.rs b/codex-rs/terminal-browser/src/diagnostics_tests.rs new file mode 100644 index 0000000000..9446a118b5 --- /dev/null +++ b/codex-rs/terminal-browser/src/diagnostics_tests.rs @@ -0,0 +1,36 @@ +use pretty_assertions::assert_eq; + +use super::first_nonempty_line; +use super::has_supported_version; +#[cfg(any(target_os = "macos", target_os = "linux"))] +use super::inspect_installation; + +#[test] +fn version_probe_uses_the_first_nonempty_line() { + assert_eq!( + first_nonempty_line(b"\n Carbonyl 0.0.3 \nextra\n"), + Some("Carbonyl 0.0.3".to_string()) + ); +} + +#[test] +fn version_probe_requires_the_exact_pinned_version() { + assert!(has_supported_version("Carbonyl 0.0.3")); + assert!(!has_supported_version("Carbonyl 0.0.30")); + assert!(!has_supported_version("Carbonyl 1.0.0")); +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +#[tokio::test] +#[ignore = "opt-in smoke test requiring CODEX_CARBONYL_BINARY and a host sandbox"] +async fn configured_real_carbonyl_passes_the_sandboxed_version_probe() { + let binary = std::env::var_os("CODEX_CARBONYL_BINARY") + .map(std::path::PathBuf::from) + .expect("set CODEX_CARBONYL_BINARY to the real Carbonyl executable"); + + let installation = inspect_installation(&binary, &Default::default()) + .await + .expect("supported installation"); + + assert_eq!(installation.version, "Carbonyl 0.0.3"); +} diff --git a/codex-rs/terminal-browser/src/error.rs b/codex-rs/terminal-browser/src/error.rs new file mode 100644 index 0000000000..571a3e7fbf --- /dev/null +++ b/codex-rs/terminal-browser/src/error.rs @@ -0,0 +1,132 @@ +use serde::Serialize; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +enum BrowserErrorCode { + Unavailable, + UnsupportedVersion, + BrowserBusy, + NavigationTimeout, + StaleHandle, + Crashed, + PolicyChanged, + ApprovalDenied, + InvalidInput, + Internal, +} + +/// Sanitized error returned to a model-facing terminal-browser tool call. +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct BrowserToolFailure { + code: BrowserErrorCode, + message: &'static str, + retryable: bool, +} + +/// Classifies an internal browser error into a bounded, secret-free tool failure. +pub fn classify_browser_error(error: &anyhow::Error) -> BrowserToolFailure { + let message = error.to_string().to_ascii_lowercase(); + if message.contains("browser_busy") + || message.contains("another terminal browser action") + || message.contains("human_control_active") + { + return failure( + BrowserErrorCode::BrowserBusy, + "another browser action is still running", + /*retryable*/ true, + ); + } + if message.contains("unsupported carbonyl version") { + return failure( + BrowserErrorCode::UnsupportedVersion, + "the installed Carbonyl version is not supported", + /*retryable*/ false, + ); + } + if message.contains("timed out") || message.contains("navigation_timeout") { + return failure( + BrowserErrorCode::NavigationTimeout, + "the browser did not reach the requested state before the timeout", + /*retryable*/ true, + ); + } + if message.contains("node_not_found") + || message.contains("invalid nodeid") + || message.contains("could not find node") + || message.contains("no node with given id") + { + return failure( + BrowserErrorCode::StaleHandle, + "the page changed or the node handle is stale; take a new snapshot", + /*retryable*/ true, + ); + } + if message.contains("network policy changed") + || message.contains("permission profile") + || message.contains("permission policy") + || message.contains("sandbox network namespace") + || message.contains("managed terminal-browser networking is not yet supported") + { + return failure( + BrowserErrorCode::PolicyChanged, + "the active browser permission policy does not allow this action", + /*retryable*/ false, + ); + } + if message.contains("approval") && message.contains("denied") { + return failure( + BrowserErrorCode::ApprovalDenied, + "the user did not approve this browser action", + /*retryable*/ false, + ); + } + if message.contains("carbonyl exited") || message.contains("devtools connection") { + return failure( + BrowserErrorCode::Crashed, + "the Carbonyl browser process is no longer available", + /*retryable*/ true, + ); + } + if message.contains("carbonyl") + && (message.contains("unavailable") + || message.contains("not found") + || message.contains("not discovered")) + { + return failure( + BrowserErrorCode::Unavailable, + "Carbonyl is unavailable; run /browser doctor for local diagnostics", + /*retryable*/ false, + ); + } + if message.contains("unknown field") + || message.contains("missing field") + || message.contains("invalid type") + || message.contains("does not accept") + || message.contains("must not") + || message.contains("requires a url") + { + return failure( + BrowserErrorCode::InvalidInput, + "the terminal-browser tool arguments are invalid", + /*retryable*/ false, + ); + } + failure( + BrowserErrorCode::Internal, + "the terminal-browser action failed; inspect local logs for details", + /*retryable*/ false, + ) +} + +fn failure(code: BrowserErrorCode, message: &'static str, retryable: bool) -> BrowserToolFailure { + BrowserToolFailure { + code, + message, + retryable, + } +} + +#[cfg(test)] +#[path = "error_tests.rs"] +mod tests; diff --git a/codex-rs/terminal-browser/src/error_tests.rs b/codex-rs/terminal-browser/src/error_tests.rs new file mode 100644 index 0000000000..54ed8ee8f9 --- /dev/null +++ b/codex-rs/terminal-browser/src/error_tests.rs @@ -0,0 +1,38 @@ +use pretty_assertions::assert_eq; +use serde_json::json; + +use super::BrowserErrorCode; +use super::classify_browser_error; + +#[test] +fn model_facing_failures_are_structured_and_sanitized() { + let error = anyhow::anyhow!("CDP Runtime.evaluate failed with secret=do-not-leak"); + let failure = classify_browser_error(&error); + + assert_eq!( + serde_json::to_value(failure).expect("serialize failure"), + json!({ + "code": "internal", + "message": "the terminal-browser action failed; inspect local logs for details", + "retryable": false, + }) + ); +} + +#[test] +fn busy_errors_are_retryable() { + let failure = classify_browser_error(&anyhow::anyhow!("browser_busy: already running")); + assert_eq!(failure.code, BrowserErrorCode::BrowserBusy); + assert!(failure.retryable); +} + +#[test] +fn policy_and_stale_node_errors_use_actionable_codes() { + let policy = classify_browser_error(&anyhow::anyhow!( + "browser navigation blocked by the active permission policy" + )); + let stale = classify_browser_error(&anyhow::anyhow!("CDP: Could not find node with id 7")); + + assert_eq!(policy.code, BrowserErrorCode::PolicyChanged); + assert_eq!(stale.code, BrowserErrorCode::StaleHandle); +} diff --git a/codex-rs/terminal-browser/src/handles.rs b/codex-rs/terminal-browser/src/handles.rs new file mode 100644 index 0000000000..b651764c44 --- /dev/null +++ b/codex-rs/terminal-browser/src/handles.rs @@ -0,0 +1,57 @@ +use std::collections::HashMap; +use std::sync::atomic::AtomicU64; +use std::sync::atomic::Ordering; + +use anyhow::Result; + +static NEXT_DOCUMENT_GENERATION: AtomicU64 = AtomicU64::new(1); + +#[derive(Debug)] +pub(crate) struct BrowserHandles { + document_generation: u64, + next_node: u64, + backend_nodes: HashMap, +} + +impl Default for BrowserHandles { + fn default() -> Self { + Self { + document_generation: next_document_generation(), + next_node: 1, + backend_nodes: HashMap::new(), + } + } +} + +impl BrowserHandles { + pub(crate) fn begin_snapshot(&mut self) { + self.clear(); + } + + pub(crate) fn insert(&mut self, backend_node_id: u64) -> String { + let node_id = format!("d{:016x}n{}", self.document_generation, self.next_node); + self.next_node += 1; + self.backend_nodes.insert(node_id.clone(), backend_node_id); + node_id + } + + pub(crate) fn resolve(&self, node_id: &str) -> Result { + self.backend_nodes.get(node_id).copied().ok_or_else(|| { + anyhow::anyhow!("node_not_found: take a new snapshot and use its nodeId") + }) + } + + pub(crate) fn clear(&mut self) { + self.document_generation = next_document_generation(); + self.next_node = 1; + self.backend_nodes.clear(); + } +} + +fn next_document_generation() -> u64 { + NEXT_DOCUMENT_GENERATION.fetch_add(/*val*/ 1, Ordering::Relaxed) +} + +#[cfg(test)] +#[path = "handles_tests.rs"] +mod tests; diff --git a/codex-rs/terminal-browser/src/handles_tests.rs b/codex-rs/terminal-browser/src/handles_tests.rs new file mode 100644 index 0000000000..0eccba0074 --- /dev/null +++ b/codex-rs/terminal-browser/src/handles_tests.rs @@ -0,0 +1,17 @@ +use pretty_assertions::assert_eq; + +use super::BrowserHandles; + +#[test] +fn handles_are_document_scoped_and_stale_after_refresh() { + let mut handles = BrowserHandles::default(); + let first = handles.insert(/*backend_node_id*/ 42); + assert_eq!(handles.resolve(&first).expect("resolve handle"), 42); + + handles.begin_snapshot(); + let second = handles.insert(/*backend_node_id*/ 42); + + assert_ne!(first, second); + assert!(handles.resolve(&first).is_err()); + assert_eq!(handles.resolve(&second).expect("resolve new handle"), 42); +} diff --git a/codex-rs/terminal-browser/src/human_control.rs b/codex-rs/terminal-browser/src/human_control.rs new file mode 100644 index 0000000000..48def70f19 --- /dev/null +++ b/codex-rs/terminal-browser/src/human_control.rs @@ -0,0 +1,248 @@ +use std::sync::Arc; +use std::sync::Weak; +use std::sync::atomic::Ordering; + +use anyhow::Result; +use anyhow::anyhow; +use tokio::sync::mpsc; + +use crate::actions; +use crate::actions::HumanMouseDispatchState; +use crate::input::BrowserKeyInput; +use crate::input::BrowserMouseInput; +use crate::input::BrowserMouseKind; +use crate::session::Inner; +use crate::session::TerminalBrowser; + +const HUMAN_INPUT_CAPACITY: usize = 128; +const MAX_HUMAN_TEXT_BYTES: usize = 64 * 1024; + +pub(crate) struct HumanInputSender(mpsc::Sender); + +enum HumanInput { + Key(BrowserKeyInput), + Text(String), + Mouse(BrowserMouseInput), +} + +pub(crate) struct QueuedHumanInput { + generation: u64, + input: HumanInput, +} + +impl TerminalBrowser { + pub(crate) fn human_input_channel() -> (HumanInputSender, mpsc::Receiver) { + let (sender, receiver) = mpsc::channel(HUMAN_INPUT_CAPACITY); + (HumanInputSender(sender), receiver) + } + + pub fn is_human_control_active(&self) -> bool { + self.inner.human_control.load(Ordering::SeqCst) + } + + pub async fn toggle_human_control(&self) -> Result { + self.inner + .human_control_transition + .compare_exchange( + /*current*/ false, + /*new*/ true, + Ordering::SeqCst, + Ordering::SeqCst, + ) + .map_err(|_| anyhow!("browser_busy: browser control is already changing"))?; + let _transition = HumanControlTransition { + active: &self.inner.human_control_transition, + }; + if self.is_human_control_active() { + self.end_human_control().await?; + } else { + self.begin_human_control().await?; + } + Ok(self.is_human_control_active()) + } + + #[expect( + clippy::await_holding_invalid_type, + reason = "entering human control must serialize the live-session check with model actions" + )] + pub async fn begin_human_control(&self) -> Result<()> { + if let Some(receiver) = self + .inner + .human_input_rx + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take() + { + tokio::spawn(run_human_input_worker( + Arc::downgrade(&self.inner), + receiver, + )); + } + let _operation = self + .inner + .operation + .try_lock() + .map_err(|_| anyhow!("browser_busy: another terminal browser action is running"))?; + anyhow::ensure!( + self.inner.session.lock().await.is_some(), + "terminal browser is not open" + ); + let process_running = self + .inner + .process + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_ref() + .is_some_and(|process| !process.has_exited()); + let view_running = matches!( + &self + .inner + .view + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .status, + crate::screen::BrowserStatus::Running + ); + anyhow::ensure!( + process_running && view_running, + "terminal browser is not running" + ); + self.inner + .human_control_generation + .fetch_add(/*val*/ 1, Ordering::SeqCst); + self.inner + .human_control + .store(/*val*/ true, Ordering::SeqCst); + self.inner.update_view(|view| { + view.human_control = true; + view.visible = true; + }); + Ok(()) + } + + #[expect( + clippy::await_holding_invalid_type, + reason = "ending human control must serialize handle invalidation with model actions" + )] + pub async fn end_human_control(&self) -> Result<()> { + self.inner + .human_control_generation + .fetch_add(/*val*/ 1, Ordering::SeqCst); + self.inner + .human_control + .store(/*val*/ false, Ordering::SeqCst); + self.inner.update_view(|view| view.human_control = false); + let _operation = self.inner.operation.lock().await; + let mut session = self.inner.session.lock().await; + if let Some(session) = session.as_mut() { + session.handles.clear(); + } + Ok(()) + } + + pub fn send_human_key(&self, input: BrowserKeyInput) -> Result<()> { + self.queue_human_input(HumanInput::Key(input)) + } + + pub fn send_human_text(&self, text: &str) -> Result<()> { + anyhow::ensure!( + text.len() <= MAX_HUMAN_TEXT_BYTES, + "browser text input exceeds the 64 KiB limit" + ); + self.queue_human_input(HumanInput::Text(text.to_string())) + } + + pub fn send_human_mouse(&self, input: BrowserMouseInput) -> Result<()> { + self.queue_human_input(HumanInput::Mouse(input)) + } + + fn queue_human_input(&self, input: HumanInput) -> Result<()> { + anyhow::ensure!( + self.is_human_control_active(), + "human control is not active" + ); + let is_motion = matches!( + input, + HumanInput::Mouse(BrowserMouseInput { + kind: BrowserMouseKind::Move, + .. + }) + ); + let queued = QueuedHumanInput { + generation: self.inner.human_control_generation.load(Ordering::SeqCst), + input, + }; + match self.inner.human_input_tx.0.try_send(queued) { + Ok(()) => Ok(()), + Err(mpsc::error::TrySendError::Full(_)) if is_motion => Ok(()), + Err(mpsc::error::TrySendError::Full(_)) => { + anyhow::bail!("browser_busy: human input queue is full") + } + Err(mpsc::error::TrySendError::Closed(_)) => { + anyhow::bail!("terminal browser input worker stopped") + } + } + } +} + +struct HumanControlTransition<'a> { + active: &'a std::sync::atomic::AtomicBool, +} + +impl Drop for HumanControlTransition<'_> { + fn drop(&mut self) { + self.active.store(/*val*/ false, Ordering::SeqCst); + } +} + +#[expect( + clippy::await_holding_invalid_type, + reason = "human input must remain serialized with model actions until its CDP call completes" +)] +async fn run_human_input_worker( + inner: Weak, + mut receiver: mpsc::Receiver, +) { + let mut mouse_state = HumanMouseDispatchState::default(); + let mut mouse_generation = None; + while let Some(queued) = receiver.recv().await { + let Some(inner) = inner.upgrade() else { + return; + }; + if !inner.human_control.load(Ordering::SeqCst) + || inner.human_control_generation.load(Ordering::SeqCst) != queued.generation + { + continue; + } + if mouse_generation != Some(queued.generation) { + mouse_state = HumanMouseDispatchState::default(); + mouse_generation = Some(queued.generation); + } + let result = { + let _operation = inner.operation.lock().await; + if !inner.human_control.load(Ordering::SeqCst) + || inner.human_control_generation.load(Ordering::SeqCst) != queued.generation + { + continue; + } + let cdp = { + let session = inner.session.lock().await; + let Some(session) = session.as_ref() else { + continue; + }; + session.cdp.clone() + }; + match queued.input { + HumanInput::Key(input) => actions::dispatch_human_key(&cdp, &input).await, + HumanInput::Text(text) => actions::insert_human_text(&cdp, &text).await, + HumanInput::Mouse(input) => { + actions::dispatch_human_mouse(&cdp, input, &mut mouse_state).await + } + } + }; + if let Err(error) = result { + tracing::warn!(%error, "terminal-browser human input failed"); + inner.set_crashed("Terminal browser input connection failed".to_string()); + } + } +} diff --git a/codex-rs/terminal-browser/src/input.rs b/codex-rs/terminal-browser/src/input.rs new file mode 100644 index 0000000000..8ab94bfffc --- /dev/null +++ b/codex-rs/terminal-browser/src/input.rs @@ -0,0 +1,96 @@ +/// Modifier keys held while forwarding input to the browser. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct BrowserInputModifiers { + /// Alt/Option is held. + pub alt: bool, + /// Control is held. + pub control: bool, + /// Command/Super is held. + pub meta: bool, + /// Shift is held. + pub shift: bool, +} + +impl BrowserInputModifiers { + pub(crate) fn cdp_mask(self) -> u8 { + u8::from(self.alt) + | (u8::from(self.control) << 1) + | (u8::from(self.meta) << 2) + | (u8::from(self.shift) << 3) + } +} + +/// One keyboard event forwarded while the user exclusively controls the browser. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BrowserKeyInput { + /// DOM key value, such as `Enter` or `a`. + pub key: String, + /// DOM physical-key code, such as `Enter` or `KeyA`. + pub code: String, + /// Text to insert for printable keys; absent for shortcuts and control keys. + pub text: Option, + /// Modifier state accompanying the key. + pub modifiers: BrowserInputModifiers, +} + +/// Mouse button used by a forwarded terminal mouse event. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum BrowserMouseButton { + #[default] + None, + Left, + Middle, + Right, +} + +impl BrowserMouseButton { + pub(crate) fn as_cdp(self) -> &'static str { + match self { + Self::None => "none", + Self::Left => "left", + Self::Middle => "middle", + Self::Right => "right", + } + } + + pub(crate) fn cdp_buttons_mask(self) -> u8 { + match self { + Self::None => 0, + Self::Left => 1, + Self::Right => 2, + Self::Middle => 4, + } + } +} + +/// Kind of mouse event forwarded during exclusive human control. +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum BrowserMouseKind { + /// Pointer movement without a button transition. + Move, + /// Mouse-button press. + Down, + /// Mouse-button release. + Up, + /// Scroll-wheel movement in CSS pixels. + Wheel { delta_x: f64, delta_y: f64 }, +} + +/// Terminal-relative mouse event that Carbonyl maps into its current CSS viewport. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct BrowserMouseInput { + /// Mouse operation to perform. + pub kind: BrowserMouseKind, + /// Button associated with the event. + pub button: BrowserMouseButton, + /// Zero-based terminal column inside the rendered browser viewport. + pub column: u16, + /// Zero-based terminal row inside the rendered browser viewport. + pub row: u16, + /// Width of the rendered browser viewport in terminal cells. + pub viewport_cols: u16, + /// Height of the rendered browser viewport in terminal cells. + pub viewport_rows: u16, + /// Modifier state accompanying the mouse event. + pub modifiers: BrowserInputModifiers, +} diff --git a/codex-rs/terminal-browser/src/lib.rs b/codex-rs/terminal-browser/src/lib.rs new file mode 100644 index 0000000000..a86b6106aa --- /dev/null +++ b/codex-rs/terminal-browser/src/lib.rs @@ -0,0 +1,46 @@ +#![deny(clippy::print_stdout, clippy::print_stderr)] + +mod accessibility; +mod actions; +mod cdp; +mod devtools; +mod diagnostics; +mod error; +mod handles; +mod human_control; +mod input; +mod navigation; +mod network; +mod process; +mod profile; +mod profile_control; +mod runtime; +mod sandbox; +mod screen; +mod scripts; +mod session; +mod tool_dispatch; +mod url_policy; + +pub use actions::BrowserToolOutput; +pub use diagnostics::BrowserDoctorReport; +pub use error::BrowserToolFailure; +pub use error::classify_browser_error; +pub use input::BrowserInputModifiers; +pub use input::BrowserKeyInput; +pub use input::BrowserMouseButton; +pub use input::BrowserMouseInput; +pub use input::BrowserMouseKind; +pub use network::BrowserNetworkPolicy; +pub use sandbox::BrowserLaunchContext; +pub use screen::BrowserCell; +pub use screen::BrowserColor; +pub use screen::BrowserScreen; +pub use screen::BrowserStatus; +pub use screen::BrowserView; +pub use screen::TerminalSize; +pub use session::TerminalBrowser; + +#[cfg(test)] +#[path = "terminal_browser_tests.rs"] +mod tests; diff --git a/codex-rs/terminal-browser/src/navigation.rs b/codex-rs/terminal-browser/src/navigation.rs new file mode 100644 index 0000000000..b92e5242de --- /dev/null +++ b/codex-rs/terminal-browser/src/navigation.rs @@ -0,0 +1,423 @@ +use std::time::Duration; +use std::time::Instant; + +use anyhow::Context; +use anyhow::Result; +use serde::Deserialize; +use serde::Serialize; +use serde_json::Value; +use serde_json::json; +use tokio::sync::broadcast; + +use crate::accessibility::node_center; +use crate::accessibility::node_is_attached; +use crate::actions::BrowserToolOutput; +use crate::actions::page_metadata; +use crate::cdp::CdpClient; +use crate::cdp::CdpEvent; +use crate::handles::BrowserHandles; + +const DEFAULT_WAIT_TIMEOUT: Duration = Duration::from_secs(10); +const MAX_WAIT_TIMEOUT: Duration = Duration::from_secs(30); +const WAIT_POLL_INTERVAL: Duration = Duration::from_millis(100); + +#[derive(Clone, Copy, Debug, Default, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub(crate) enum LoadState { + DomContentLoaded, + #[default] + Load, +} + +impl LoadState { + fn event_method(self) -> &'static str { + match self { + Self::DomContentLoaded => "Page.domContentEventFired", + Self::Load => "Page.loadEventFired", + } + } + + fn is_reached(self, ready_state: &str) -> bool { + match self { + Self::DomContentLoaded => matches!(ready_state, "interactive" | "complete"), + Self::Load => ready_state == "complete", + } + } +} + +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub(crate) enum NavigationAction { + Goto, + Back, + Forward, + Reload, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct NavigateRequest { + pub(crate) action: NavigationAction, + pub(crate) url: Option, + #[serde(default)] + pub(crate) wait_until: LoadState, + pub(crate) timeout_ms: Option, +} + +#[derive(Clone, Copy, Debug, Default, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub(crate) enum UrlMatch { + #[default] + Exact, + Contains, +} + +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub(crate) enum TextState { + Present, + Absent, +} + +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub(crate) enum NodeState { + Visible, + Hidden, + Attached, + Detached, +} + +#[derive(Debug, Deserialize)] +#[serde( + tag = "type", + rename_all = "camelCase", + rename_all_fields = "camelCase", + deny_unknown_fields +)] +pub(crate) enum WaitRequest { + Url { + value: String, + #[serde(default, rename = "match")] + comparison: UrlMatch, + timeout_ms: Option, + }, + LoadState { + state: LoadState, + timeout_ms: Option, + }, + Text { + value: String, + state: TextState, + timeout_ms: Option, + }, + Node { + node_id: String, + state: NodeState, + timeout_ms: Option, + }, +} + +pub(crate) async fn navigate(client: &CdpClient, url: &str) -> Result<()> { + tokio::time::timeout(Duration::from_secs(15), async { + let events = client.subscribe_events(); + let result = client.call("Page.navigate", json!({ "url": url })).await?; + if let Some(error) = result.get("errorText").and_then(Value::as_str) { + anyhow::bail!("navigation failed: {error}"); + } + if let Some(loader_id) = result.get("loaderId").and_then(Value::as_str) { + wait_for_load_event( + events, + LoadState::DomContentLoaded, + Duration::from_secs(15), + Some(loader_id), + ) + .await?; + } + Ok(()) + }) + .await + .context("navigation_timeout: navigation did not complete within 15 seconds")? +} + +pub(crate) async fn navigate_request( + client: &CdpClient, + request: &NavigateRequest, +) -> Result { + let timeout = wait_timeout(request.timeout_ms)?; + tokio::time::timeout(timeout, navigate_request_inner(client, request, timeout)) + .await + .context("navigation_timeout: navigation did not complete before timeoutMs")? +} + +async fn navigate_request_inner( + client: &CdpClient, + request: &NavigateRequest, + timeout: Duration, +) -> Result { + let mut events = client.subscribe_events(); + let (expected_loader, target_url) = match request.action { + NavigationAction::Goto => { + let url = request + .url + .as_deref() + .context("navigate goto requires a URL")?; + let result = client.call("Page.navigate", json!({ "url": url })).await?; + if let Some(error) = result.get("errorText").and_then(Value::as_str) { + anyhow::bail!("navigation failed: {error}"); + } + ( + result + .get("loaderId") + .and_then(Value::as_str) + .map(str::to_string), + Some(url.to_string()), + ) + } + NavigationAction::Back => { + anyhow::ensure!(request.url.is_none(), "navigate back does not accept a URL"); + (None, Some(navigate_history(client, /*offset*/ -1).await?)) + } + NavigationAction::Forward => { + anyhow::ensure!( + request.url.is_none(), + "navigate forward does not accept a URL" + ); + (None, Some(navigate_history(client, /*offset*/ 1).await?)) + } + NavigationAction::Reload => { + anyhow::ensure!( + request.url.is_none(), + "navigate reload does not accept a URL" + ); + client.call("Page.reload", json!({})).await?; + (None, None) + } + }; + if expected_loader.is_none() + && let Some(target_url) = target_url.as_deref() + { + wait_for_url_ready_state(client, target_url, request.wait_until).await?; + } else { + wait_for_load_event_receiver( + &mut events, + request.wait_until, + timeout, + expected_loader.as_deref(), + ) + .await?; + } + let metadata = page_metadata(client).await?; + Ok(BrowserToolOutput::Text(serde_json::to_string(&json!({ + "action": request.action, + "url": metadata.url, + "title": metadata.title, + "loadState": request.wait_until, + }))?)) +} + +pub(crate) async fn wait( + client: &CdpClient, + handles: &BrowserHandles, + request: &WaitRequest, +) -> Result { + let timeout = match request { + WaitRequest::Url { timeout_ms, .. } + | WaitRequest::LoadState { timeout_ms, .. } + | WaitRequest::Text { timeout_ms, .. } + | WaitRequest::Node { timeout_ms, .. } => wait_timeout(*timeout_ms)?, + }; + let deadline = Instant::now() + timeout; + loop { + let remaining = deadline.saturating_duration_since(Instant::now()); + anyhow::ensure!( + !remaining.is_zero(), + "navigation_timeout: wait condition timed out" + ); + let matched = tokio::time::timeout(remaining, wait_condition_met(client, handles, request)) + .await + .context("navigation_timeout: wait condition timed out")??; + if matched { + return Ok(BrowserToolOutput::Text(serde_json::to_string(&json!({ + "matched": true, + "elapsedMs": timeout.saturating_sub(deadline.saturating_duration_since(Instant::now())).as_millis(), + }))?)); + } + anyhow::ensure!( + Instant::now() < deadline, + "navigation_timeout: wait condition timed out" + ); + tokio::time::sleep( + WAIT_POLL_INTERVAL.min(deadline.saturating_duration_since(Instant::now())), + ) + .await; + } +} + +async fn navigate_history(client: &CdpClient, offset: i64) -> Result { + let history = client.call("Page.getNavigationHistory", json!({})).await?; + let current_index = history + .get("currentIndex") + .and_then(Value::as_i64) + .context("navigation history omitted currentIndex")?; + let target_index = current_index + offset; + let entries = history + .get("entries") + .and_then(Value::as_array) + .context("navigation history omitted entries")?; + anyhow::ensure!( + target_index >= 0 && usize::try_from(target_index).is_ok_and(|index| index < entries.len()), + "navigation history has no entry in that direction" + ); + let entry_id = entries[usize::try_from(target_index)?] + .get("id") + .and_then(Value::as_i64) + .context("navigation history entry omitted id")?; + let target_url = entries[usize::try_from(target_index)?] + .get("url") + .and_then(Value::as_str) + .context("navigation history entry omitted URL")? + .to_string(); + client + .call( + "Page.navigateToHistoryEntry", + json!({ "entryId": entry_id }), + ) + .await?; + Ok(target_url) +} + +async fn wait_for_url_ready_state( + client: &CdpClient, + target_url: &str, + load_state: LoadState, +) -> Result<()> { + loop { + let state = client + .evaluate("({ url: location.href, readyState: document.readyState })") + .await?; + if state.get("url").and_then(Value::as_str) == Some(target_url) + && state + .get("readyState") + .and_then(Value::as_str) + .is_some_and(|state| load_state.is_reached(state)) + { + return Ok(()); + } + tokio::time::sleep(WAIT_POLL_INTERVAL).await; + } +} + +async fn wait_for_load_event( + mut events: broadcast::Receiver, + load_state: LoadState, + timeout: Duration, + expected_loader: Option<&str>, +) -> Result<()> { + wait_for_load_event_receiver(&mut events, load_state, timeout, expected_loader).await +} + +async fn wait_for_load_event_receiver( + events: &mut broadcast::Receiver, + load_state: LoadState, + wait_timeout: Duration, + expected_loader: Option<&str>, +) -> Result<()> { + tokio::time::timeout(wait_timeout, async { + loop { + match events.recv().await { + Ok(CdpEvent::Message(message)) + if navigation_event_matches(&message, load_state, expected_loader) => + { + return Ok(()); + } + Ok(CdpEvent::Message(_)) | Err(broadcast::error::RecvError::Lagged(_)) => {} + Ok(CdpEvent::Disconnected(reason)) => anyhow::bail!(reason), + Err(broadcast::error::RecvError::Closed) => { + anyhow::bail!("Carbonyl closed the DevTools connection") + } + } + } + }) + .await + .context("navigation_timeout: navigation did not reach the requested load state")? +} + +fn navigation_event_matches( + message: &Value, + load_state: LoadState, + expected_loader: Option<&str>, +) -> bool { + if message.get("method").and_then(Value::as_str) == Some("Page.lifecycleEvent") { + let name = message.pointer("/params/name").and_then(Value::as_str); + let reached = match load_state { + LoadState::DomContentLoaded => matches!(name, Some("DOMContentLoaded" | "load")), + LoadState::Load => name == Some("load"), + }; + return reached + && expected_loader.is_none_or(|loader| { + message.pointer("/params/loaderId").and_then(Value::as_str) == Some(loader) + }); + } + expected_loader.is_none() + && message.get("method").and_then(Value::as_str) == Some(load_state.event_method()) +} + +fn wait_timeout(timeout_ms: Option) -> Result { + let timeout = timeout_ms.map_or(DEFAULT_WAIT_TIMEOUT, Duration::from_millis); + anyhow::ensure!(!timeout.is_zero(), "timeoutMs must be greater than zero"); + anyhow::ensure!(timeout <= MAX_WAIT_TIMEOUT, "timeoutMs cannot exceed 30000"); + Ok(timeout) +} + +async fn wait_condition_met( + client: &CdpClient, + handles: &BrowserHandles, + request: &WaitRequest, +) -> Result { + match request { + WaitRequest::Url { + value, comparison, .. + } => { + anyhow::ensure!(value.len() <= 4 * 1024, "URL wait value exceeds 4 KiB"); + let current = client.evaluate("location.href").await?; + let current = current.as_str().unwrap_or_default(); + Ok(match comparison { + UrlMatch::Exact => current == value, + UrlMatch::Contains => current.contains(value), + }) + } + WaitRequest::LoadState { state, .. } => { + let ready_state = client.evaluate("document.readyState").await?; + Ok(ready_state + .as_str() + .is_some_and(|value| state.is_reached(value))) + } + WaitRequest::Text { value, state, .. } => { + anyhow::ensure!(value.len() <= 4 * 1024, "text wait value exceeds 4 KiB"); + let value = serde_json::to_string(value)?; + let present = client + .evaluate(&format!( + "Boolean(document.body?.innerText.includes({value}))" + )) + .await? + .as_bool() + .unwrap_or(/*default*/ false); + Ok(match state { + TextState::Present => present, + TextState::Absent => !present, + }) + } + WaitRequest::Node { node_id, state, .. } => { + let backend_node_id = handles.resolve(node_id)?; + let attached = node_is_attached(client, backend_node_id).await?; + let visible = attached && node_center(client, backend_node_id).await.is_ok(); + Ok(match state { + NodeState::Visible => visible, + NodeState::Hidden => attached && !visible, + NodeState::Attached => attached, + NodeState::Detached => !attached, + }) + } + } +} diff --git a/codex-rs/terminal-browser/src/network.rs b/codex-rs/terminal-browser/src/network.rs new file mode 100644 index 0000000000..a977747bc5 --- /dev/null +++ b/codex-rs/terminal-browser/src/network.rs @@ -0,0 +1,11 @@ +use std::net::SocketAddr; + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub enum BrowserNetworkPolicy { + #[default] + Disabled, + Direct, + ManagedProxy { + http_addr: SocketAddr, + }, +} diff --git a/codex-rs/terminal-browser/src/process.rs b/codex-rs/terminal-browser/src/process.rs new file mode 100644 index 0000000000..4cc1898d9c --- /dev/null +++ b/codex-rs/terminal-browser/src/process.rs @@ -0,0 +1,493 @@ +use std::sync::Arc; +use std::sync::PoisonError; +use std::sync::atomic::Ordering; +use std::time::Duration; + +use anyhow::Context; +use anyhow::Result; +use anyhow::anyhow; +use codex_utils_pty::ProcessHandle; +use codex_utils_pty::spawn_pty_process; +use tokio::sync::mpsc; +use tokio::sync::watch; +use tokio::task::JoinHandle; + +use crate::cdp::CdpClient; +use crate::devtools::deny_downloads; +use crate::devtools::discover_page_target; +use crate::devtools::prepare_devtools_active_port; +use crate::handles::BrowserHandles; +use crate::network::BrowserNetworkPolicy; +use crate::profile::BrowserProfileLock; +use crate::runtime::BrowserRuntime; +use crate::sandbox::prepare_browser_launch; +use crate::screen::BrowserStatus; +use crate::screen::TerminalScreen; +use crate::screen::TerminalSize; +use crate::session::Inner; +use crate::session::RenderMode; +use crate::session::SessionConfig; +use crate::url_policy::spawn_navigation_policy_task; + +pub(crate) struct BrowserSession { + pub(crate) config: SessionConfig, + pub(crate) cdp: CdpClient, + pub(crate) handles: BrowserHandles, + _runtime: BrowserRuntime, + _profile_lock: Option, + output_task: JoinHandle<()>, + exit_task: JoinHandle<()>, + navigation_policy_task: JoinHandle<()>, +} + +struct StartupGuard<'a> { + process_slot: &'a std::sync::Mutex>>, + process: Option>, + profile_lock: Option, + output_task: Option>, + exit_task: Option>, + navigation_policy_task: Option>, +} + +struct CommittedStartup { + profile_lock: Option, + output_task: JoinHandle<()>, + exit_task: JoinHandle<()>, + navigation_policy_task: JoinHandle<()>, +} + +impl<'a> StartupGuard<'a> { + fn new( + process_slot: &'a std::sync::Mutex>>, + process: Arc, + profile_lock: Option, + ) -> Self { + Self { + process_slot, + process: Some(process), + profile_lock, + output_task: None, + exit_task: None, + navigation_policy_task: None, + } + } + + fn set_output_task(&mut self, output_task: JoinHandle<()>) { + self.output_task = Some(output_task); + } + + fn set_exit_task(&mut self, exit_task: JoinHandle<()>) { + self.exit_task = Some(exit_task); + } + + fn set_navigation_policy_task(&mut self, task: JoinHandle<()>) { + self.navigation_policy_task = Some(task); + } + + fn commit(mut self) -> Result { + let output_task = self + .output_task + .take() + .context("startup output task must be present")?; + let exit_task = self + .exit_task + .take() + .context("startup exit task must be present")?; + let navigation_policy_task = self + .navigation_policy_task + .take() + .context("startup navigation policy task must be present")?; + self.process.take(); + Ok(CommittedStartup { + profile_lock: self.profile_lock.take(), + output_task, + exit_task, + navigation_policy_task, + }) + } +} + +impl Drop for StartupGuard<'_> { + fn drop(&mut self) { + let Some(process) = self.process.take() else { + return; + }; + if let Some(task) = self.output_task.take() { + task.abort(); + } + if let Some(task) = self.exit_task.take() { + task.abort(); + } + if let Some(task) = self.navigation_policy_task.take() { + task.abort(); + } + process.terminate(); + let mut current = self + .process_slot + .lock() + .unwrap_or_else(PoisonError::into_inner); + if current + .as_ref() + .is_some_and(|current| Arc::ptr_eq(current, &process)) + { + current.take(); + } + } +} + +impl Inner { + pub(crate) async fn ensure_session(self: &Arc, config: SessionConfig) -> Result<()> { + let expected_network_policy = config.network_policy.clone(); + let process_exited = self + .process + .lock() + .unwrap_or_else(PoisonError::into_inner) + .as_ref() + .is_some_and(|process| process.has_exited()); + let existing_config = self + .session + .lock() + .await + .as_ref() + .map(|session| session.config.clone()); + match existing_config { + None => self.start_session(config).await?, + Some(existing) if process_exited || existing != config => { + let (visible, url) = { + let view = self.view.read().unwrap_or_else(PoisonError::into_inner); + (view.visible, view.url.clone()) + }; + self.close_session().await; + self.update_view(|view| { + view.status = BrowserStatus::Starting; + view.visible = visible; + view.url = url; + }); + self.start_session(config).await?; + } + Some(_) => {} + } + if self.network_policy() != expected_network_policy { + self.close_session().await; + anyhow::bail!("browser network policy changed while Carbonyl was starting"); + } + Ok(()) + } + + async fn validated_binary(&self) -> Result { + let binary = self + .binary + .read() + .unwrap_or_else(PoisonError::into_inner) + .clone() + .context("Carbonyl is unavailable; install it or set CODEX_CARBONYL_BINARY")?; + let installation = + crate::diagnostics::inspect_installation(&binary, &self.launch_context).await?; + let path = installation.path.clone(); + *self.installation.lock().await = Some(installation); + Ok(path) + } + + async fn start_session(self: &Arc, config: SessionConfig) -> Result<()> { + let binary = self.validated_binary().await?; + self.closing.store(/*val*/ false, Ordering::SeqCst); + let persistent_profile = self.selected_profile_resources()?; + let runtime = + BrowserRuntime::create(persistent_profile.as_ref().map(|(profile, _lock)| profile))?; + prepare_devtools_active_port(&runtime.profile)?; + let args = carbonyl_args( + /*debugging_port*/ 0, + runtime.profile.as_path().to_string_lossy().as_ref(), + &config.network_policy, + config.render_mode, + ); + // Carbonyl is an external browser process. Pass only the process basics it needs instead + // of forwarding Codex credentials and service-specific environment variables. + let env = runtime.environment(&config.network_policy); + let launch = prepare_browser_launch( + &binary, + args, + &runtime.root, + &runtime.profile, + env, + &config.network_policy, + &self.launch_context, + )?; + let size = *self.resize_tx.borrow(); + let spawned = match spawn_pty_process( + &launch.program, + &launch.args, + launch.cwd.as_path(), + &launch.env, + &launch.arg0, + size.into(), + ) + .await + { + Ok(spawned) => spawned, + Err(error) => { + return Err(error).context("launch Carbonyl"); + } + }; + + let process = Arc::new(spawned.session); + let writer = process.writer_sender(); + let profile_lock = persistent_profile.map(|(_profile, lock)| lock); + let mut startup = StartupGuard::new(&self.process, process.clone(), profile_lock); + let resize_result = { + let mut current_process = self.process.lock().unwrap_or_else(PoisonError::into_inner); + *current_process = Some(process.clone()); + let current_size = *self.resize_tx.borrow(); + (current_size != size) + .then(|| process.resize(current_size.into())) + .transpose() + }; + if let Err(error) = resize_result { + return Err(error).context("resize Carbonyl after startup"); + } + if self.network_policy() != config.network_policy { + self.closing.store(/*val*/ true, Ordering::SeqCst); + anyhow::bail!("browser network policy changed while Carbonyl was starting"); + } + let output_task = spawn_screen_task( + self.clone(), + spawned.stdout_rx, + writer, + self.resize_tx.subscribe(), + ); + startup.set_output_task(output_task); + let mut exit_rx = spawned.exit_rx; + + let target = tokio::select! { + target = discover_page_target(&runtime.profile) => target, + exit = &mut exit_rx => Err(anyhow!("Carbonyl exited during startup: {exit:?}")), + }; + let target = target?; + let cdp = CdpClient::connect(&target.websocket_url).await?; + deny_downloads(&cdp).await?; + startup.set_navigation_policy_task(spawn_navigation_policy_task(self.clone(), cdp.clone())); + let exit_inner = self.clone(); + let exit_task = tokio::spawn(async move { + let exit = exit_rx.await; + if !exit_inner.closing.load(Ordering::SeqCst) { + exit_inner.set_crashed(match exit { + Ok(code) => format!("Carbonyl exited with status {code}"), + Err(_) => "lost Carbonyl exit status".to_string(), + }); + } + }); + startup.set_exit_task(exit_task); + let mut session_slot = self.session.lock().await; + let CommittedStartup { + profile_lock, + output_task, + exit_task, + navigation_policy_task, + } = startup.commit()?; + *session_slot = Some(BrowserSession { + config, + cdp, + handles: BrowserHandles::default(), + _runtime: runtime, + _profile_lock: profile_lock, + output_task, + exit_task, + navigation_policy_task, + }); + drop(session_slot); + self.update_view(|view| { + view.status = BrowserStatus::Running; + if !target.title.is_empty() { + view.title = Some(target.title); + } + }); + Ok(()) + } + + fn selected_profile_resources( + &self, + ) -> Result< + Option<( + codex_utils_absolute_path::AbsolutePathBuf, + BrowserProfileLock, + )>, + > { + let selected = self + .selected_profile + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone(); + let Some(selected) = selected else { + return Ok(None); + }; + let store = self + .profile_store + .as_ref() + .context("named terminal-browser profiles are unavailable")?; + store.lock_existing(&selected).map(Some) + } + + pub(crate) async fn close_session(&self) { + self.closing.store(/*val*/ true, Ordering::SeqCst); + self.human_control.store(/*val*/ false, Ordering::SeqCst); + self.human_control_generation + .fetch_add(/*val*/ 1, Ordering::SeqCst); + let session = self.session.lock().await.take(); + let process = self + .process + .lock() + .unwrap_or_else(PoisonError::into_inner) + .take(); + if let Some(process) = process { + let _ = process.writer_sender().send(vec![b'\x03']).await; + let graceful = tokio::time::timeout( + Duration::from_millis(/*millis*/ 500), + wait_for_exit(process.clone()), + ) + .await + .is_ok(); + if !graceful { + process.terminate(); + } + } + if let Some(session) = session { + session.output_task.abort(); + session.exit_task.abort(); + session.navigation_policy_task.abort(); + } + let status = if self + .binary + .read() + .unwrap_or_else(PoisonError::into_inner) + .is_some() + { + BrowserStatus::Idle + } else { + self.view + .read() + .unwrap_or_else(PoisonError::into_inner) + .status + .clone() + }; + self.update_view(|view| { + view.status = status; + view.visible = false; + view.title = None; + view.url = None; + view.human_control = false; + }); + } +} + +fn spawn_screen_task( + inner: Arc, + mut output: mpsc::Receiver>, + writer: mpsc::Sender>, + mut resize_rx: watch::Receiver, +) -> JoinHandle<()> { + tokio::spawn(async move { + let size = *resize_rx.borrow_and_update(); + let mut terminal = TerminalScreen::new(size); + let mut interval = tokio::time::interval(Duration::from_millis(/*millis*/ 33)); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let mut dirty = true; + loop { + tokio::select! { + chunk = output.recv() => match chunk { + Some(chunk) => { + for response in terminal.process(&chunk) { + let _ = writer.send(response).await; + } + let starting = matches!( + &inner + .view + .read() + .unwrap_or_else(PoisonError::into_inner) + .status, + BrowserStatus::Starting + ); + if starting { + update_screen_view(&inner, &terminal); + dirty = false; + } else { + dirty = true; + } + } + None => break, + }, + changed = resize_rx.changed() => match changed { + Ok(()) => { + let size = *resize_rx.borrow_and_update(); + terminal.resize(size); + dirty = true; + } + Err(_) => break, + }, + _ = interval.tick(), if dirty => { + update_screen_view(&inner, &terminal); + dirty = false; + } + } + } + }) +} + +fn update_screen_view(inner: &Inner, terminal: &TerminalScreen) { + inner.update_view(|view| { + view.screen = terminal.snapshot(); + if let Some(title) = terminal.title() { + view.title = Some(title); + } + }); +} + +async fn wait_for_exit(process: Arc) { + while !process.has_exited() { + tokio::time::sleep(Duration::from_millis(/*millis*/ 20)).await; + } +} + +pub(crate) fn carbonyl_args( + debugging_port: u16, + profile: &str, + network_policy: &BrowserNetworkPolicy, + render_mode: RenderMode, +) -> Vec { + let mut args = vec![ + "--remote-debugging-address=127.0.0.1".to_string(), + format!("--remote-debugging-port={debugging_port}"), + format!("--user-data-dir={profile}"), + "--disable-extensions".to_string(), + "--disable-background-networking".to_string(), + "--disable-sync".to_string(), + "--disable-default-apps".to_string(), + "--no-first-run".to_string(), + "--no-default-browser-check".to_string(), + "--disable-component-update".to_string(), + "--password-store=basic".to_string(), + // Every Carbonyl process already inherits the Codex platform sandbox. Chromium cannot + // initialize a second Seatbelt profile from inside that sandbox on macOS. + "--no-sandbox".to_string(), + // Carbonyl does not need a hardware GPU for terminal rendering, and keeping rendering in + // software avoids granting the browser access to platform GPU device interfaces. + "--disable-gpu".to_string(), + "--disable-quic".to_string(), + "--force-webrtc-ip-handling-policy=disable_non_proxied_udp".to_string(), + ]; + match network_policy { + BrowserNetworkPolicy::Disabled | BrowserNetworkPolicy::Direct => {} + BrowserNetworkPolicy::ManagedProxy { http_addr } => { + args.push(format!("--proxy-server=http://{http_addr}")); + args.push("--proxy-bypass-list=<-loopback>".to_string()); + } + } + if render_mode == RenderMode::Bitmap { + args.push("--bitmap".to_string()); + } + args.push("about:blank".to_string()); + args +} + +#[cfg(test)] +#[path = "process_tests.rs"] +mod tests; diff --git a/codex-rs/terminal-browser/src/process_tests.rs b/codex-rs/terminal-browser/src/process_tests.rs new file mode 100644 index 0000000000..784fb50c10 --- /dev/null +++ b/codex-rs/terminal-browser/src/process_tests.rs @@ -0,0 +1,98 @@ +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; + +use codex_utils_pty::ProcessDriver; +use codex_utils_pty::spawn_from_driver; +use pretty_assertions::assert_eq; +use tokio::sync::broadcast; +use tokio::sync::mpsc; +use tokio::sync::oneshot; + +use super::StartupGuard; +use crate::devtools::prepare_devtools_active_port; + +#[tokio::test] +async fn canceled_startup_terminates_the_process_and_aborts_output() { + let (writer_tx, _writer_rx) = mpsc::channel(/*buffer*/ 1); + let (_output_tx, output_rx) = broadcast::channel(/*capacity*/ 1); + let (_exit_tx, exit_rx) = oneshot::channel(); + let terminated = Arc::new(AtomicBool::new(/*v*/ false)); + let terminate_flag = terminated.clone(); + let spawned = spawn_from_driver(ProcessDriver { + writer_tx, + stdout_rx: output_rx, + stderr_rx: None, + exit_rx, + terminator: Some(Box::new(move || { + terminate_flag.store(/*val*/ true, Ordering::SeqCst); + })), + writer_handle: None, + resizer: None, + }); + let process = Arc::new(spawned.session); + let process_slot = Mutex::new(Some(process.clone())); + let output_task = tokio::spawn(std::future::pending::<()>()); + let output_abort = output_task.abort_handle(); + let mut startup = StartupGuard::new(&process_slot, process, /*profile_lock*/ None); + startup.set_output_task(output_task); + + drop(startup); + tokio::task::yield_now().await; + + assert!(terminated.load(Ordering::SeqCst)); + assert!(output_abort.is_finished()); + assert!(process_slot.lock().expect("process slot").is_none()); +} + +#[test] +fn stale_devtools_active_port_is_removed_before_spawn() { + let profile = tempfile::tempdir().expect("profile directory"); + let active_port = profile.path().join("DevToolsActivePort"); + std::fs::write(&active_port, "9222\n").expect("stale active port"); + + prepare_devtools_active_port(profile.path()).expect("prepare active port"); + + assert!(!active_port.exists()); + prepare_devtools_active_port(profile.path()).expect("missing active port is valid"); +} + +#[test] +fn non_file_devtools_active_port_is_rejected() { + let profile = tempfile::tempdir().expect("profile directory"); + let active_port = profile.path().join("DevToolsActivePort"); + std::fs::create_dir(&active_port).expect("active port directory"); + + let error = prepare_devtools_active_port(profile.path()) + .expect_err("active port directory must be rejected"); + + assert_eq!( + error.to_string(), + "refusing non-file Carbonyl DevToolsActivePort" + ); +} + +#[cfg(unix)] +#[test] +fn symbolic_link_devtools_active_port_is_rejected() { + use std::os::unix::fs::symlink; + + let profile = tempfile::tempdir().expect("profile directory"); + let target = profile.path().join("target"); + let active_port = profile.path().join("DevToolsActivePort"); + std::fs::write(&target, "9222\n").expect("target active port"); + symlink(&target, &active_port).expect("active port symlink"); + + let error = prepare_devtools_active_port(profile.path()) + .expect_err("active port symlink must be rejected"); + + assert_eq!( + error.to_string(), + "refusing symbolic-link Carbonyl DevToolsActivePort" + ); + assert_eq!( + std::fs::read_to_string(target).expect("target remains"), + "9222\n" + ); +} diff --git a/codex-rs/terminal-browser/src/profile.rs b/codex-rs/terminal-browser/src/profile.rs new file mode 100644 index 0000000000..ca627ed359 --- /dev/null +++ b/codex-rs/terminal-browser/src/profile.rs @@ -0,0 +1,228 @@ +use std::fmt::Write; +#[cfg(unix)] +use std::fs::File; +#[cfg(unix)] +use std::fs::OpenOptions; +use std::path::Path; + +use anyhow::Context; +use anyhow::Result; +use codex_utils_absolute_path::AbsolutePathBuf; +use sha2::Digest; +use sha2::Sha256; + +use crate::sandbox::BrowserLaunchContext; + +const MAX_PROFILE_NAME_BYTES: usize = 64; +const MAX_LISTED_PROFILES: usize = 100; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct BrowserProfileName(String); + +impl BrowserProfileName { + pub(crate) fn parse(name: &str) -> Result { + let mut bytes = name.bytes(); + let first = bytes.next().context("profile name must not be empty")?; + let valid_first = first.is_ascii_alphanumeric(); + let valid_rest = + bytes.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-')); + anyhow::ensure!( + name.len() <= MAX_PROFILE_NAME_BYTES && valid_first && valid_rest, + "profile names must start with a letter or number and contain at most 64 ASCII letters, numbers, dots, underscores, or dashes" + ); + Ok(Self(name.to_string())) + } + + pub(crate) fn as_str(&self) -> &str { + &self.0 + } +} + +#[derive(Clone, Debug)] +pub(crate) struct BrowserProfileStore { + root: AbsolutePathBuf, +} + +pub(crate) struct BrowserProfileListing { + pub(crate) profiles: Vec, + pub(crate) total: usize, + pub(crate) truncated: bool, +} + +pub(crate) struct BrowserProfileLock { + #[cfg(unix)] + file: File, +} + +impl BrowserProfileStore { + pub(crate) fn from_context(context: &BrowserLaunchContext) -> Result> { + let (Some(codex_home), Some(workspace_root)) = + (&context.codex_home, &context.workspace_root) + else { + return Ok(None); + }; + let mut workspace_hash = String::with_capacity(/*capacity*/ 16); + let digest = Sha256::digest(workspace_root.as_os_str().as_encoded_bytes()); + for byte in &digest[..8] { + write!(&mut workspace_hash, "{byte:02x}").context("format workspace profile hash")?; + } + let root = codex_home + .join("browser-profiles") + .join("terminal-browser") + .join(workspace_hash); + Ok(Some(Self { root })) + } + + pub(crate) fn list(&self) -> Result { + let entries = match std::fs::read_dir(self.root.as_path()) { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Ok(BrowserProfileListing { + profiles: Vec::new(), + total: 0, + truncated: false, + }); + } + Err(error) => return Err(error).context("list terminal-browser profiles"), + }; + let mut profiles = Vec::new(); + let mut total = 0; + for entry in entries { + let entry = entry.context("read terminal-browser profile entry")?; + let file_type = entry + .file_type() + .context("read terminal-browser profile type")?; + let Some(name) = entry.file_name().to_str().map(str::to_string) else { + continue; + }; + if file_type.is_dir() && BrowserProfileName::parse(&name).is_ok() { + total += 1; + if profiles.len() < MAX_LISTED_PROFILES { + profiles.push(name); + } + } + } + profiles.sort_unstable(); + Ok(BrowserProfileListing { + profiles, + total, + truncated: total > MAX_LISTED_PROFILES, + }) + } + + pub(crate) fn create(&self, name: &BrowserProfileName) -> Result { + create_private_directory(self.root.as_path())?; + let path = self.root.join(name.as_str()); + anyhow::ensure!( + !std::fs::symlink_metadata(path.as_path()) + .is_ok_and(|metadata| metadata.file_type().is_symlink()), + "profile path must not be a symbolic link" + ); + create_private_directory(path.as_path())?; + Ok(path) + } + + pub(crate) fn existing_path(&self, name: &BrowserProfileName) -> Result { + let path = self.root.join(name.as_str()); + let metadata = std::fs::symlink_metadata(path.as_path()) + .with_context(|| format!("browser profile `{}` does not exist", name.as_str()))?; + anyhow::ensure!( + metadata.is_dir() && !metadata.file_type().is_symlink(), + "browser profile path is not a safe directory" + ); + Ok(path) + } + + pub(crate) fn lock_existing( + &self, + name: &BrowserProfileName, + ) -> Result<(AbsolutePathBuf, BrowserProfileLock)> { + self.existing_path(name)?; + let lock = self.acquire_lock(name)?; + // A cooperating process cannot delete the profile after the lock is acquired, but + // rechecking keeps us fail-closed if an older Codex process raced with this one. + let path = self.existing_path(name)?; + Ok((path, lock)) + } + + pub(crate) fn forget(&self, name: &BrowserProfileName) -> Result<()> { + let _lock = self.acquire_lock(name)?; + let path = self.existing_path(name)?; + std::fs::remove_dir_all(path.as_path()) + .with_context(|| format!("delete browser profile `{}`", name.as_str())) + } + + #[cfg(unix)] + fn acquire_lock(&self, name: &BrowserProfileName) -> Result { + use std::os::fd::AsRawFd; + use std::os::unix::fs::OpenOptionsExt; + + create_private_directory(self.root.as_path())?; + let lock_path = self.root.join(format!(".{}.lock", name.as_str())); + let file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .mode(/*mode*/ 0o600) + .custom_flags(libc::O_CLOEXEC | libc::O_NOFOLLOW) + .open(lock_path.as_path()) + .with_context(|| format!("open browser profile lock for `{}`", name.as_str()))?; + anyhow::ensure!( + file.metadata() + .context("read browser profile lock metadata")? + .is_file(), + "browser profile lock path is not a regular file" + ); + // SAFETY: `file` owns a valid descriptor for the duration of this call and remains owned + // by `BrowserProfileLock` for as long as the advisory lock must be held. + let result = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) }; + if result == 0 { + return Ok(BrowserProfileLock { file }); + } + let error = std::io::Error::last_os_error(); + if error.kind() == std::io::ErrorKind::WouldBlock { + anyhow::bail!("browser profile `{}` is already in use", name.as_str()); + } + Err(error).with_context(|| format!("lock browser profile `{}`", name.as_str())) + } + + #[cfg(not(unix))] + fn acquire_lock(&self, _name: &BrowserProfileName) -> Result { + anyhow::bail!("named terminal-browser profiles are unsupported on this platform") + } +} + +#[cfg(unix)] +impl Drop for BrowserProfileLock { + fn drop(&mut self) { + use std::os::fd::AsRawFd; + + // SAFETY: `self.file` remains open for the duration of the call. Closing the file would + // also release the lock, but an explicit unlock makes the lifetime boundary unambiguous. + let _ = unsafe { libc::flock(self.file.as_raw_fd(), libc::LOCK_UN) }; + } +} + +fn create_private_directory(path: &Path) -> Result<()> { + std::fs::create_dir_all(path) + .with_context(|| format!("create browser profile directory {}", path.display()))?; + set_private_permissions(path) +} + +#[cfg(unix)] +fn set_private_permissions(path: &Path) -> Result<()> { + use std::os::unix::fs::PermissionsExt; + + std::fs::set_permissions(path, std::fs::Permissions::from_mode(/*mode*/ 0o700)) + .with_context(|| format!("restrict browser profile directory {}", path.display())) +} + +#[cfg(not(unix))] +fn set_private_permissions(_path: &Path) -> Result<()> { + Ok(()) +} + +#[cfg(test)] +#[path = "profile_tests.rs"] +mod tests; diff --git a/codex-rs/terminal-browser/src/profile_control.rs b/codex-rs/terminal-browser/src/profile_control.rs new file mode 100644 index 0000000000..83be967a35 --- /dev/null +++ b/codex-rs/terminal-browser/src/profile_control.rs @@ -0,0 +1,199 @@ +use std::sync::PoisonError; + +use anyhow::Context; +use anyhow::Result; +use anyhow::anyhow; +use serde::Deserialize; +use serde_json::Value; + +use crate::actions::BrowserToolOutput; +use crate::profile::BrowserProfileName; +use crate::session::TerminalBrowser; + +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +enum ProfileAction { + List, + RequestCreate, + RequestSelect, + RequestEphemeral, + RequestForget, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct ProfileArgs { + action: ProfileAction, + name: Option, +} + +impl TerminalBrowser { + pub fn profiles(&self) -> Result> { + Ok(self + .inner + .profile_store + .as_ref() + .context("named terminal-browser profiles are unavailable")? + .list()? + .profiles) + } + + pub fn selected_profile(&self) -> Option { + self.inner + .selected_profile + .lock() + .unwrap_or_else(PoisonError::into_inner) + .as_ref() + .map(|name| name.as_str().to_string()) + } + + #[expect( + clippy::await_holding_invalid_type, + reason = "profile changes must serialize with browser actions and lifecycle transitions" + )] + pub async fn create_profile(&self, name: &str) -> Result<()> { + let _operation = self.inner.operation.lock().await; + let name = BrowserProfileName::parse(name)?; + let store = self + .inner + .profile_store + .as_ref() + .context("named terminal-browser profiles are unavailable")?; + store.create(&name)?; + self.inner.close_session().await; + *self + .inner + .selected_profile + .lock() + .unwrap_or_else(PoisonError::into_inner) = Some(name); + Ok(()) + } + + #[expect( + clippy::await_holding_invalid_type, + reason = "profile changes must serialize with browser actions and lifecycle transitions" + )] + pub async fn select_profile(&self, name: &str) -> Result<()> { + let _operation = self.inner.operation.lock().await; + let name = BrowserProfileName::parse(name)?; + let store = self + .inner + .profile_store + .as_ref() + .context("named terminal-browser profiles are unavailable")?; + store.existing_path(&name)?; + self.inner.close_session().await; + *self + .inner + .selected_profile + .lock() + .unwrap_or_else(PoisonError::into_inner) = Some(name); + Ok(()) + } + + #[expect( + clippy::await_holding_invalid_type, + reason = "profile changes must serialize with browser actions and lifecycle transitions" + )] + pub async fn select_ephemeral_profile(&self) -> Result<()> { + let _operation = self.inner.operation.lock().await; + self.inner.close_session().await; + self.inner + .selected_profile + .lock() + .unwrap_or_else(PoisonError::into_inner) + .take(); + Ok(()) + } + + #[expect( + clippy::await_holding_invalid_type, + reason = "profile changes must serialize with browser actions and lifecycle transitions" + )] + pub async fn forget_profile(&self, name: &str) -> Result<()> { + let _operation = self.inner.operation.lock().await; + let name = BrowserProfileName::parse(name)?; + let selected = self + .inner + .selected_profile + .lock() + .unwrap_or_else(PoisonError::into_inner) + .as_ref() + == Some(&name); + if selected { + self.inner.close_session().await; + self.inner + .selected_profile + .lock() + .unwrap_or_else(PoisonError::into_inner) + .take(); + } + self.inner + .profile_store + .as_ref() + .context("named terminal-browser profiles are unavailable")? + .forget(&name) + } + + pub(crate) fn execute_profile_tool(&self, arguments: Value) -> Result { + let args: ProfileArgs = serde_json::from_value(arguments)?; + match args.action { + ProfileAction::List => { + anyhow::ensure!(args.name.is_none(), "profile list does not accept a name"); + let listing = self + .inner + .profile_store + .as_ref() + .context("named terminal-browser profiles are unavailable")? + .list()?; + Ok(BrowserToolOutput::Text(serde_json::to_string( + &serde_json::json!({ + "profiles": listing.profiles, + "total": listing.total, + "truncated": listing.truncated, + "selected": self.selected_profile(), + }), + )?)) + } + ProfileAction::RequestEphemeral => { + anyhow::ensure!( + args.name.is_none(), + "profile requestEphemeral does not accept a name" + ); + approval_required("ephemeral", "/browser profile ephemeral") + } + action => { + let name = args + .name + .as_deref() + .context("profile mutation requests require a name")?; + let name = BrowserProfileName::parse(name)?; + let command = match action { + ProfileAction::RequestCreate => { + format!("/browser profile create {}", name.as_str()) + } + ProfileAction::RequestSelect => { + format!("/browser profile use {}", name.as_str()) + } + ProfileAction::RequestForget => { + format!("/browser profile forget {} --confirm", name.as_str()) + } + ProfileAction::List | ProfileAction::RequestEphemeral => { + return Err(anyhow!("invalid profile action")); + } + }; + approval_required(name.as_str(), &command) + } + } + } +} + +fn approval_required(profile: &str, command: &str) -> Result { + Ok(BrowserToolOutput::Text(serde_json::to_string( + &serde_json::json!({ + "status": "approvalRequired", + "profile": profile, + "command": command, + }), + )?)) +} diff --git a/codex-rs/terminal-browser/src/profile_tests.rs b/codex-rs/terminal-browser/src/profile_tests.rs new file mode 100644 index 0000000000..dbac79995c --- /dev/null +++ b/codex-rs/terminal-browser/src/profile_tests.rs @@ -0,0 +1,74 @@ +use codex_utils_absolute_path::AbsolutePathBuf; +use pretty_assertions::assert_eq; + +use super::BrowserProfileName; +use super::BrowserProfileStore; +use crate::sandbox::BrowserLaunchContext; + +#[test] +fn profile_names_reject_traversal_and_non_ascii_input() { + for name in ["", ".hidden", "../other", "name/child", "café"] { + assert!( + BrowserProfileName::parse(name).is_err(), + "accepted {name:?}" + ); + } + assert_eq!( + BrowserProfileName::parse("work-login_2") + .expect("valid profile") + .as_str(), + "work-login_2" + ); +} + +#[test] +fn profiles_are_scoped_by_workspace_and_can_be_forgotten() { + let codex_home = tempfile::tempdir().expect("codex home"); + let codex_home = AbsolutePathBuf::from_absolute_path(codex_home.path()).expect("absolute home"); + let workspace_root = codex_home.join("workspace"); + let context = BrowserLaunchContext { + codex_home: Some(codex_home), + workspace_root: Some(workspace_root), + ..Default::default() + }; + let store = BrowserProfileStore::from_context(&context) + .expect("profile store") + .expect("configured profile store"); + let profile = BrowserProfileName::parse("work").expect("profile name"); + + let path = store.create(&profile).expect("create profile"); + assert!(path.as_path().is_dir()); + assert_eq!(store.list().expect("list profiles").profiles, vec!["work"]); + store.forget(&profile).expect("forget profile"); + assert!(store.list().expect("list profiles").profiles.is_empty()); +} + +#[cfg(unix)] +#[test] +fn forgetting_a_locked_profile_is_refused() { + let codex_home = tempfile::tempdir().expect("codex home"); + let codex_home = AbsolutePathBuf::from_absolute_path(codex_home.path()).expect("absolute home"); + let workspace_root = codex_home.join("workspace"); + let context = BrowserLaunchContext { + codex_home: Some(codex_home), + workspace_root: Some(workspace_root), + ..Default::default() + }; + let store = BrowserProfileStore::from_context(&context) + .expect("profile store") + .expect("configured profile store"); + let profile = BrowserProfileName::parse("work").expect("profile name"); + store.create(&profile).expect("create profile"); + + let (_path, profile_lock) = store.lock_existing(&profile).expect("lock profile"); + let error = store + .forget(&profile) + .expect_err("locked profile must not be deleted"); + assert_eq!( + error.to_string(), + "browser profile `work` is already in use" + ); + + drop(profile_lock); + store.forget(&profile).expect("forget unlocked profile"); +} diff --git a/codex-rs/terminal-browser/src/runtime.rs b/codex-rs/terminal-browser/src/runtime.rs new file mode 100644 index 0000000000..d09003a6c6 --- /dev/null +++ b/codex-rs/terminal-browser/src/runtime.rs @@ -0,0 +1,115 @@ +use std::collections::HashMap; +use std::path::Path; + +use anyhow::Context; +use anyhow::Result; +use codex_utils_absolute_path::AbsolutePathBuf; +use tempfile::TempDir; + +use crate::network::BrowserNetworkPolicy; + +pub(crate) struct BrowserRuntime { + _root: TempDir, + pub(crate) root: AbsolutePathBuf, + pub(crate) profile: AbsolutePathBuf, + home: AbsolutePathBuf, + temporary: AbsolutePathBuf, + xdg_runtime: AbsolutePathBuf, +} + +impl BrowserRuntime { + pub(crate) fn create(persistent_profile: Option<&AbsolutePathBuf>) -> Result { + let root = tempfile::Builder::new() + .prefix("codex-carbonyl-") + .tempdir() + .context("create Carbonyl runtime directory")?; + let absolute_root = AbsolutePathBuf::from_absolute_path(root.path()) + .context("resolve Carbonyl runtime directory")?; + let profile = persistent_profile + .cloned() + .unwrap_or_else(|| absolute_root.join("profile")); + let home = absolute_root.join("home"); + let temporary = absolute_root.join("tmp"); + let xdg_runtime = absolute_root.join("runtime"); + for directory in [&profile, &home, &temporary, &xdg_runtime] { + std::fs::create_dir_all(directory.as_path()).with_context(|| { + format!( + "create Carbonyl runtime path {}", + directory.as_path().display() + ) + })?; + set_private_permissions(directory.as_path())?; + } + Ok(Self { + _root: root, + root: absolute_root, + profile, + home, + temporary, + xdg_runtime, + }) + } + + pub(crate) fn environment( + &self, + network_policy: &BrowserNetworkPolicy, + ) -> HashMap { + let mut env = HashMap::new(); + for key in [ + "PATH", + "LANG", + "LC_ALL", + "LC_CTYPE", + "TZ", + "SSL_CERT_FILE", + "SSL_CERT_DIR", + ] { + if let Ok(value) = std::env::var(key) + && (!matches!(key, "SSL_CERT_FILE" | "SSL_CERT_DIR") + || Path::new(&value).is_absolute()) + { + env.insert(key.to_string(), value); + } + } + env.insert( + "HOME".to_string(), + self.home.as_path().display().to_string(), + ); + for key in ["TMPDIR", "TMP", "TEMP"] { + env.insert( + key.to_string(), + self.temporary.as_path().display().to_string(), + ); + } + env.insert( + "XDG_RUNTIME_DIR".to_string(), + self.xdg_runtime.as_path().display().to_string(), + ); + env.insert("TERM".to_string(), "xterm-256color".to_string()); + env.insert("COLORTERM".to_string(), "truecolor".to_string()); + if let BrowserNetworkPolicy::ManagedProxy { http_addr } = network_policy { + let proxy = format!("http://{http_addr}"); + env.insert("HTTP_PROXY".to_string(), proxy.clone()); + env.insert("HTTPS_PROXY".to_string(), proxy); + env.insert("NO_PROXY".to_string(), "127.0.0.1,localhost".to_string()); + } + env + } +} + +#[cfg(unix)] +fn set_private_permissions(path: &Path) -> Result<()> { + use std::os::unix::fs::PermissionsExt; + + std::fs::set_permissions(path, std::fs::Permissions::from_mode(/*mode*/ 0o700)) + .with_context(|| format!("restrict Carbonyl runtime path {}", path.display())) +} + +#[cfg(not(unix))] +fn set_private_permissions(_path: &Path) -> Result<()> { + Ok(()) +} + +#[cfg(test)] +#[path = "runtime_tests.rs"] +mod tests; diff --git a/codex-rs/terminal-browser/src/runtime_tests.rs b/codex-rs/terminal-browser/src/runtime_tests.rs new file mode 100644 index 0000000000..b7d9bc723f --- /dev/null +++ b/codex-rs/terminal-browser/src/runtime_tests.rs @@ -0,0 +1,20 @@ +use pretty_assertions::assert_eq; + +use super::BrowserRuntime; +use crate::network::BrowserNetworkPolicy; + +#[test] +fn runtime_replaces_the_user_home_and_working_storage() { + let runtime = BrowserRuntime::create(/*persistent_profile*/ None).expect("create runtime"); + let env = runtime.environment(&BrowserNetworkPolicy::Direct); + + assert_eq!( + env.get("HOME"), + Some(&runtime.home.as_path().display().to_string()) + ); + assert_eq!( + env.get("TMPDIR"), + Some(&runtime.temporary.as_path().display().to_string()) + ); + assert!(runtime.profile.starts_with(runtime.root.as_path())); +} diff --git a/codex-rs/terminal-browser/src/sandbox.rs b/codex-rs/terminal-browser/src/sandbox.rs new file mode 100644 index 0000000000..10a19ee281 --- /dev/null +++ b/codex-rs/terminal-browser/src/sandbox.rs @@ -0,0 +1,259 @@ +use std::collections::HashMap; +use std::ffi::OsString; +use std::path::Path; + +use anyhow::Context; +use anyhow::Result; +use codex_protocol::config_types::WindowsSandboxLevel; +use codex_protocol::models::PermissionProfile; +use codex_protocol::permissions::FileSystemAccessMode; +use codex_protocol::permissions::FileSystemPath; +use codex_protocol::permissions::FileSystemSandboxEntry; +use codex_protocol::permissions::FileSystemSandboxPolicy; +use codex_protocol::permissions::FileSystemSpecialPath; +use codex_protocol::permissions::NetworkSandboxPolicy; +use codex_sandboxing::SandboxCommand; +use codex_sandboxing::SandboxDirectSpawnTransformRequest; +use codex_sandboxing::SandboxManager; +use codex_sandboxing::SandboxTransformRequest; +use codex_sandboxing::SandboxType; +use codex_sandboxing::SandboxablePreference; +use codex_sandboxing::WindowsSandboxProxySettingsMode; +use codex_utils_absolute_path::AbsolutePathBuf; +use codex_utils_path_uri::PathUri; + +use crate::network::BrowserNetworkPolicy; + +/// Host-local inputs needed to wrap Carbonyl in the Codex platform sandbox. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct BrowserLaunchContext { + /// Codex data directory used to store optional named browser profiles. + pub codex_home: Option, + /// Workspace whose identity scopes optional named browser profiles. + pub workspace_root: Option, + /// Absolute path to the Codex Linux sandbox helper, when running on Linux. + pub codex_linux_sandbox_exe: Option, + /// Whether to request the deprecated Landlock implementation instead of bubblewrap. + pub use_legacy_landlock: bool, +} + +pub(crate) struct PreparedBrowserLaunch { + pub(crate) program: String, + pub(crate) args: Vec, + pub(crate) cwd: AbsolutePathBuf, + pub(crate) env: HashMap, + pub(crate) arg0: Option, +} + +pub(crate) fn prepare_browser_launch( + binary: &Path, + args: Vec, + browser_root: &AbsolutePathBuf, + profile_root: &AbsolutePathBuf, + env: HashMap, + network_policy: &BrowserNetworkPolicy, + context: &BrowserLaunchContext, +) -> Result { + if matches!(network_policy, BrowserNetworkPolicy::ManagedProxy { .. }) { + anyhow::bail!( + "managed terminal-browser networking is not yet supported because Carbonyl needs a loopback CDP listener that the current sandbox cannot permit without bypassing the managed proxy" + ); + } + + let binary = AbsolutePathBuf::from_absolute_path(binary) + .context("resolve Carbonyl executable path")? + .canonicalize() + .context("canonicalize Carbonyl executable")?; + let binary_root = binary + .parent() + .context("Carbonyl executable has no parent directory")?; + ensure_isolated_binary_root(&binary_root, browser_root, profile_root, context)?; + let file_system_policy = + browser_file_system_policy(browser_root, profile_root, &binary_root, &env)?; + let network_sandbox_policy = match network_policy { + BrowserNetworkPolicy::Disabled | BrowserNetworkPolicy::ManagedProxy { .. } => { + NetworkSandboxPolicy::Restricted + } + BrowserNetworkPolicy::Direct => NetworkSandboxPolicy::Enabled, + }; + let permissions = + PermissionProfile::from_runtime_permissions(&file_system_policy, network_sandbox_policy); + let enforce_managed_network = false; + let manager = SandboxManager::new(); + let sandbox = manager.select_initial( + &file_system_policy, + network_sandbox_policy, + SandboxablePreference::Require, + WindowsSandboxLevel::Disabled, + enforce_managed_network, + ); + anyhow::ensure!( + sandbox != SandboxType::None, + "terminal browser sandbox is unavailable on this platform" + ); + let browser_root_uri = PathUri::from_abs_path(browser_root); + let mut request = manager + .transform_for_direct_spawn(SandboxDirectSpawnTransformRequest { + transform: SandboxTransformRequest { + command: SandboxCommand { + program: OsString::from(binary.as_os_str()), + args, + cwd: browser_root_uri.clone(), + env, + managed_network: None, + additional_permissions: None, + }, + permissions: &permissions, + sandbox, + enforce_managed_network, + environment_id: None, + network: None, + sandbox_policy_cwd: &browser_root_uri, + codex_linux_sandbox_exe: context + .codex_linux_sandbox_exe + .as_ref() + .map(AbsolutePathBuf::as_path), + use_legacy_landlock: context.use_legacy_landlock, + windows_sandbox_level: WindowsSandboxLevel::Disabled, + windows_sandbox_private_desktop: false, + }, + workspace_roots: &[], + windows_sandbox_proxy_settings_mode: WindowsSandboxProxySettingsMode::Reconcile, + }) + .context("prepare Carbonyl sandbox")?; + #[cfg(target_os = "macos")] + if request.sandbox == SandboxType::MacosSeatbelt { + anyhow::ensure!( + request.command.get(1).map(String::as_str) == Some("-p"), + "Carbonyl sandbox returned an unexpected Seatbelt command" + ); + let profile = request + .command + .get_mut(2) + .context("Carbonyl sandbox omitted the Seatbelt profile")?; + // Chromium's browser process registers a PID-scoped rendezvous service that its children + // look up to exchange Mach ports. Keep both grants specific to Carbonyl's fixed service + // name; the shared Codex Seatbelt policy intentionally does not permit arbitrary Mach IPC. + profile.push_str( + r#" +(allow mach-register mach-lookup + (global-name-prefix "org.chromium.Chromium.MachPortRendezvousServer."))"#, + ); + } + let (program, args) = request + .command + .split_first() + .context("Carbonyl sandbox returned an empty command")?; + let cwd = request + .cwd + .to_abs_path() + .context("resolve sandboxed Carbonyl working directory")?; + Ok(PreparedBrowserLaunch { + program: program.clone(), + args: args.to_vec(), + cwd, + env: request.env, + arg0: request.arg0, + }) +} + +fn ensure_isolated_binary_root( + binary_root: &AbsolutePathBuf, + browser_root: &AbsolutePathBuf, + profile_root: &AbsolutePathBuf, + context: &BrowserLaunchContext, +) -> Result<()> { + for protected_root in [ + Some(browser_root), + Some(profile_root), + context.codex_home.as_ref(), + context.workspace_root.as_ref(), + ] + .into_iter() + .flatten() + { + let canonical_root = protected_root + .canonicalize() + .unwrap_or_else(|_| protected_root.clone()); + if binary_root.as_path().starts_with(canonical_root.as_path()) + || canonical_root.as_path().starts_with(binary_root.as_path()) + { + anyhow::bail!( + "Carbonyl must be installed in a dedicated bundle outside the workspace, Codex home, and browser data directories" + ); + } + } + Ok(()) +} + +fn browser_file_system_policy( + browser_root: &AbsolutePathBuf, + profile_root: &AbsolutePathBuf, + binary_root: &AbsolutePathBuf, + env: &HashMap, +) -> Result { + let mut entries = vec![ + FileSystemSandboxEntry { + path: FileSystemPath::Special { + value: FileSystemSpecialPath::Minimal, + }, + access: FileSystemAccessMode::Read, + }, + FileSystemSandboxEntry { + path: FileSystemPath::Path { + path: binary_root.clone(), + }, + access: FileSystemAccessMode::Read, + }, + FileSystemSandboxEntry { + path: FileSystemPath::Path { + path: browser_root.clone(), + }, + access: FileSystemAccessMode::Write, + }, + ]; + #[cfg(target_os = "macos")] + for (path, label) in [ + ( + "/System/Library/CoreServices/SystemAppearance.bundle", + "macOS system appearance bundle", + ), + ( + "/System/Library/CoreServices/SystemVersion.bundle", + "macOS system version bundle", + ), + ("/System/Library/Fonts", "macOS system fonts"), + ] { + entries.push(FileSystemSandboxEntry { + path: FileSystemPath::Path { + path: AbsolutePathBuf::from_absolute_path(path) + .with_context(|| format!("resolve the {label}"))?, + }, + access: FileSystemAccessMode::Read, + }); + } + if profile_root != browser_root && !profile_root.as_path().starts_with(browser_root.as_path()) { + entries.push(FileSystemSandboxEntry { + path: FileSystemPath::Path { + path: profile_root.clone(), + }, + access: FileSystemAccessMode::Write, + }); + } + for key in ["SSL_CERT_FILE", "SSL_CERT_DIR"] { + let Some(path) = env.get(key) else { + continue; + }; + let path = AbsolutePathBuf::from_absolute_path_checked(path) + .with_context(|| format!("resolve {key} for the Carbonyl sandbox"))?; + entries.push(FileSystemSandboxEntry { + path: FileSystemPath::Path { path }, + access: FileSystemAccessMode::Read, + }); + } + Ok(FileSystemSandboxPolicy::restricted(entries)) +} + +#[cfg(test)] +#[path = "sandbox_tests.rs"] +mod tests; diff --git a/codex-rs/terminal-browser/src/sandbox_tests.rs b/codex-rs/terminal-browser/src/sandbox_tests.rs new file mode 100644 index 0000000000..8caa4fa4a9 --- /dev/null +++ b/codex-rs/terminal-browser/src/sandbox_tests.rs @@ -0,0 +1,119 @@ +use std::collections::HashMap; + +use codex_protocol::permissions::FileSystemAccessMode; +use pretty_assertions::assert_eq; + +use super::browser_file_system_policy; +use super::ensure_isolated_binary_root; +use super::prepare_browser_launch; +use crate::network::BrowserNetworkPolicy; +use crate::sandbox::BrowserLaunchContext; +use codex_utils_absolute_path::AbsolutePathBuf; + +#[test] +fn browser_policy_can_write_only_its_runtime_root() { + let root = tempfile::tempdir().expect("test root"); + let root = AbsolutePathBuf::from_absolute_path(root.path()).expect("absolute test root"); + let browser_root = root.join("browser-runtime"); + let binary_root = root.join("carbonyl-bundle"); + let workspace_root = root.join("workspace"); + let policy = browser_file_system_policy( + &browser_root, + &browser_root.join("profile"), + &binary_root, + &HashMap::new(), + ) + .expect("browser policy"); + + assert_eq!( + policy.resolve_access_with_cwd(browser_root.as_path(), browser_root.as_path()), + FileSystemAccessMode::Write + ); + assert_eq!( + policy.resolve_access_with_cwd(binary_root.as_path(), browser_root.as_path()), + FileSystemAccessMode::Read + ); + assert_eq!( + policy.resolve_access_with_cwd( + browser_root.join("workspace").as_path(), + browser_root.as_path() + ), + FileSystemAccessMode::Write + ); + assert_eq!( + policy.resolve_access_with_cwd(workspace_root.as_path(), browser_root.as_path()), + FileSystemAccessMode::Deny + ); + #[cfg(target_os = "macos")] + { + let readable_system_resources = [ + "/System/Library/CoreServices/SystemAppearance.bundle/Contents/Resources/SystemAppearance.car", + "/System/Library/CoreServices/SystemVersion.bundle/Contents/Resources/English.lproj/SystemVersion.strings", + "/System/Library/Fonts/LastResort.otf", + ]; + let unrelated_core_service = AbsolutePathBuf::from_absolute_path( + "/System/Library/CoreServices/Finder.app/Contents/MacOS/Finder", + ) + .expect("absolute unrelated CoreServices path"); + + for path in readable_system_resources { + let path = AbsolutePathBuf::from_absolute_path(path).expect("absolute system path"); + assert_eq!( + policy.resolve_access_with_cwd(path.as_path(), browser_root.as_path()), + FileSystemAccessMode::Read + ); + } + assert_eq!( + policy + .resolve_access_with_cwd(unrelated_core_service.as_path(), browser_root.as_path()), + FileSystemAccessMode::Deny + ); + } +} + +#[test] +fn managed_proxy_launches_fail_closed() { + let root = tempfile::tempdir().expect("test root"); + let root = AbsolutePathBuf::from_absolute_path(root.path()).expect("absolute test root"); + let Err(error) = prepare_browser_launch( + root.join("carbonyl").as_path(), + Vec::new(), + &root.join("runtime"), + &root.join("runtime/profile"), + HashMap::new(), + &BrowserNetworkPolicy::ManagedProxy { + http_addr: "127.0.0.1:8080".parse().expect("proxy address"), + }, + &BrowserLaunchContext::default(), + ) else { + panic!("managed proxy must be rejected"); + }; + + assert!( + error + .to_string() + .contains("without bypassing the managed proxy") + ); +} + +#[test] +fn binary_bundle_must_not_contain_the_workspace() { + let root = tempfile::tempdir().expect("test root"); + let root = AbsolutePathBuf::from_absolute_path(root.path()).expect("absolute test root"); + let bundle = root.join("bundle"); + let workspace = bundle.join("workspace"); + let context = BrowserLaunchContext { + workspace_root: Some(workspace), + ..Default::default() + }; + + let error = ensure_isolated_binary_root( + &bundle, + &root.join("runtime"), + &root.join("runtime/profile"), + &context, + ) + .expect_err("overlapping bundle must be rejected"); + + assert!(error.to_string().contains("dedicated bundle")); +} diff --git a/codex-rs/terminal-browser/src/screen.rs b/codex-rs/terminal-browser/src/screen.rs new file mode 100644 index 0000000000..c8476849e6 --- /dev/null +++ b/codex-rs/terminal-browser/src/screen.rs @@ -0,0 +1,236 @@ +use vt100::Callbacks; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct TerminalSize { + pub rows: u16, + pub cols: u16, +} + +impl From for codex_utils_pty::TerminalSize { + fn from(value: TerminalSize) -> Self { + Self { + rows: value.rows, + cols: value.cols, + } + } +} + +impl Default for TerminalSize { + fn default() -> Self { + Self { + rows: 30, + cols: 100, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum BrowserStatus { + Unavailable { reason: String }, + Idle, + Starting, + Running, + Crashed { message: String }, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum BrowserColor { + #[default] + Default, + Indexed(u8), + Rgb(u8, u8, u8), +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct BrowserCell { + pub text: String, + pub foreground: BrowserColor, + pub background: BrowserColor, + pub bold: bool, + pub dim: bool, + pub italic: bool, + pub underlined: bool, + pub reversed: bool, + pub wide_continuation: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BrowserScreen { + pub rows: u16, + pub cols: u16, + pub cells: Vec, + pub cursor: Option<(u16, u16)>, +} + +impl BrowserScreen { + pub fn blank(size: TerminalSize) -> Self { + let len = usize::from(size.rows) * usize::from(size.cols); + Self { + rows: size.rows, + cols: size.cols, + cells: vec![BrowserCell::default(); len], + cursor: None, + } + } + + pub fn cell(&self, row: u16, col: u16) -> Option<&BrowserCell> { + if row >= self.rows || col >= self.cols { + return None; + } + let index = usize::from(row) * usize::from(self.cols) + usize::from(col); + self.cells.get(index) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BrowserView { + pub status: BrowserStatus, + pub title: Option, + pub url: Option, + pub visible: bool, + pub human_control: bool, + pub screen: BrowserScreen, +} + +impl BrowserView { + pub(crate) fn new(status: BrowserStatus, size: TerminalSize) -> Self { + Self { + status, + title: None, + url: None, + visible: false, + human_control: false, + screen: BrowserScreen::blank(size), + } + } +} + +#[derive(Default)] +struct ScreenCallbacks { + title: Option, +} + +impl Callbacks for ScreenCallbacks { + fn set_window_title(&mut self, _screen: &mut vt100::Screen, title: &[u8]) { + let title = String::from_utf8_lossy(title); + self.title = Some(title.chars().take(/*n*/ 512).collect()); + } +} + +pub(crate) struct TerminalScreen { + parser: vt100::Parser, + query_responder: TerminalQueryResponder, +} + +impl TerminalScreen { + pub(crate) fn new(size: TerminalSize) -> Self { + Self { + parser: vt100::Parser::new_with_callbacks( + size.rows, + size.cols, + /*scrollback_len*/ 0, + ScreenCallbacks::default(), + ), + query_responder: TerminalQueryResponder::default(), + } + } + + pub(crate) fn process(&mut self, bytes: &[u8]) -> Vec> { + self.parser.process(bytes); + self.query_responder.process(bytes) + } + + pub(crate) fn resize(&mut self, size: TerminalSize) { + self.parser.screen_mut().set_size(size.rows, size.cols); + } + + pub(crate) fn title(&self) -> Option { + self.parser.callbacks().title.clone() + } + + pub(crate) fn snapshot(&self) -> BrowserScreen { + let screen = self.parser.screen(); + let (rows, cols) = screen.size(); + let mut cells = Vec::with_capacity(usize::from(rows) * usize::from(cols)); + for row in 0..rows { + for col in 0..cols { + cells.push(screen.cell(row, col).map(cell_from_vt).unwrap_or_default()); + } + } + let cursor = (!screen.hide_cursor()).then(|| screen.cursor_position()); + BrowserScreen { + rows, + cols, + cells, + cursor, + } + } +} + +fn cell_from_vt(cell: &vt100::Cell) -> BrowserCell { + BrowserCell { + text: cell.contents().to_string(), + foreground: color_from_vt(cell.fgcolor()), + background: color_from_vt(cell.bgcolor()), + bold: cell.bold(), + dim: cell.dim(), + italic: cell.italic(), + underlined: cell.underline(), + reversed: cell.inverse(), + wide_continuation: cell.is_wide_continuation(), + } +} + +fn color_from_vt(color: vt100::Color) -> BrowserColor { + match color { + vt100::Color::Default => BrowserColor::Default, + vt100::Color::Idx(index) => BrowserColor::Indexed(index), + vt100::Color::Rgb(red, green, blue) => BrowserColor::Rgb(red, green, blue), + } +} + +const TRUE_COLOR_QUERY: &[u8] = b"\x1bP$qm\x1b\\"; +const TRUE_COLOR_RESPONSE: &[u8] = b"\x1bP1$r48:2:0:0:0m\x1b\\"; +const TERMINAL_NAME_QUERY: &[u8] = b"\x1bP+q544e\x1b\\"; +const TERMINAL_NAME_RESPONSE: &[u8] = b"\x1bP1+r544e=787465726d2d323536636f6c6f72\x1b\\"; +const QUERY_TAIL_LEN: usize = 32; + +#[derive(Default)] +pub(crate) struct TerminalQueryResponder { + pending: Vec, +} + +impl TerminalQueryResponder { + pub(crate) fn process(&mut self, bytes: &[u8]) -> Vec> { + self.pending.extend_from_slice(bytes); + let mut responses = Vec::new(); + drain_query( + &mut self.pending, + TRUE_COLOR_QUERY, + TRUE_COLOR_RESPONSE, + &mut responses, + ); + drain_query( + &mut self.pending, + TERMINAL_NAME_QUERY, + TERMINAL_NAME_RESPONSE, + &mut responses, + ); + if self.pending.len() > QUERY_TAIL_LEN { + let start = self.pending.len() - QUERY_TAIL_LEN; + self.pending.drain(..start); + } + responses + } +} + +fn drain_query(pending: &mut Vec, query: &[u8], response: &[u8], responses: &mut Vec>) { + while let Some(start) = pending + .windows(query.len()) + .position(|window| window == query) + { + let end = start + query.len(); + pending.drain(start..end); + responses.push(response.to_vec()); + } +} diff --git a/codex-rs/terminal-browser/src/scripts.rs b/codex-rs/terminal-browser/src/scripts.rs new file mode 100644 index 0000000000..5c260695a7 --- /dev/null +++ b/codex-rs/terminal-browser/src/scripts.rs @@ -0,0 +1,22 @@ +pub(crate) fn scroll_expression(delta_x: i64, delta_y: i64) -> String { + format!("(() => {{ window.scrollBy({delta_x}, {delta_y}); return {{ scrollX, scrollY }}; }})()") +} + +pub(crate) fn key_code(key: &str) -> &str { + match key { + "Enter" => "Enter", + "Tab" => "Tab", + "Escape" => "Escape", + "Backspace" => "Backspace", + "Delete" => "Delete", + "ArrowUp" => "ArrowUp", + "ArrowDown" => "ArrowDown", + "ArrowLeft" => "ArrowLeft", + "ArrowRight" => "ArrowRight", + "Home" => "Home", + "End" => "End", + "PageUp" => "PageUp", + "PageDown" => "PageDown", + _ => key, + } +} diff --git a/codex-rs/terminal-browser/src/session.rs b/codex-rs/terminal-browser/src/session.rs new file mode 100644 index 0000000000..a98ed75776 --- /dev/null +++ b/codex-rs/terminal-browser/src/session.rs @@ -0,0 +1,278 @@ +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::PoisonError; +use std::sync::RwLock; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::AtomicU64; +use std::sync::atomic::Ordering; + +use crate::diagnostics::BrowserDoctorReport; +use crate::diagnostics::BrowserInstallation; +use crate::diagnostics::inspect_installation; +use crate::diagnostics::installation_report; +use crate::diagnostics::unavailable_report; +use crate::human_control::HumanInputSender; +use crate::human_control::QueuedHumanInput; +use crate::network::BrowserNetworkPolicy; +use crate::process::BrowserSession; +use crate::profile::BrowserProfileName; +use crate::profile::BrowserProfileStore; +use crate::sandbox::BrowserLaunchContext; +use crate::screen::BrowserStatus; +use crate::screen::BrowserView; +use crate::screen::TerminalSize; +use anyhow::Result; +use codex_utils_pty::ProcessHandle; +use serde::Deserialize; +use tokio::sync::Mutex as AsyncMutex; +use tokio::sync::watch; + +const CARBONYL_BINARY_ENV: &str = "CODEX_CARBONYL_BINARY"; + +#[derive(Clone)] +pub struct TerminalBrowser { + pub(crate) inner: Arc, +} + +pub(crate) struct Inner { + pub(crate) binary: RwLock>, + pub(crate) installation: AsyncMutex>, + pub(crate) launch_context: BrowserLaunchContext, + pub(crate) profile_store: Option, + pub(crate) selected_profile: Mutex>, + pub(crate) session_key: Mutex>, + pub(crate) network_policy: RwLock, + pub(crate) view: RwLock, + pub(crate) updates: watch::Sender, + pub(crate) update_sequence: AtomicU64, + pub(crate) operation: AsyncMutex<()>, + pub(crate) session: AsyncMutex>, + pub(crate) process: Mutex>>, + pub(crate) resize_tx: watch::Sender, + pub(crate) closing: AtomicBool, + pub(crate) human_control: AtomicBool, + pub(crate) human_control_transition: AtomicBool, + pub(crate) human_control_generation: AtomicU64, + pub(crate) human_input_tx: HumanInputSender, + pub(crate) human_input_rx: Mutex>>, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct SessionConfig { + pub(crate) render_mode: RenderMode, + pub(crate) network_policy: BrowserNetworkPolicy, +} + +#[derive(Clone, Copy, Debug, Default, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub(crate) enum RenderMode { + #[default] + NativeText, + Bitmap, +} + +impl TerminalBrowser { + pub fn discover() -> Self { + Self::discover_with_launch_context(BrowserLaunchContext::default()) + } + + pub fn discover_with_launch_context(launch_context: BrowserLaunchContext) -> Self { + let size = TerminalSize::default(); + let profile_store = + BrowserProfileStore::from_context(&launch_context).unwrap_or_else(|error| { + tracing::warn!(%error, "failed to configure terminal-browser profile storage"); + None + }); + let binary = discover_binary(); + let status = match &binary { + Ok(_) => BrowserStatus::Idle, + Err(reason) => BrowserStatus::Unavailable { + reason: reason.clone(), + }, + }; + let (updates, _) = watch::channel(/*init*/ 0); + let (resize_tx, _) = watch::channel(size); + let (human_input_tx, human_input_rx) = Self::human_input_channel(); + + Self { + inner: Arc::new(Inner { + binary: RwLock::new(binary.ok()), + installation: AsyncMutex::new(/*value*/ None), + launch_context, + profile_store, + selected_profile: Mutex::new(/*value*/ None), + session_key: Mutex::new(/*t*/ None), + network_policy: RwLock::new(/*t*/ BrowserNetworkPolicy::Disabled), + view: RwLock::new(BrowserView::new(status, size)), + updates, + update_sequence: AtomicU64::new(/*v*/ 0), + operation: AsyncMutex::new(()), + session: AsyncMutex::new(/*t*/ None), + process: Mutex::new(/*t*/ None), + resize_tx, + closing: AtomicBool::new(/*v*/ false), + human_control: AtomicBool::new(/*v*/ false), + human_control_transition: AtomicBool::new(/*v*/ false), + human_control_generation: AtomicU64::new(/*v*/ 0), + human_input_tx, + human_input_rx: Mutex::new(Some(human_input_rx)), + }), + } + } + + pub fn is_available(&self) -> bool { + self.inner + .binary + .read() + .unwrap_or_else(PoisonError::into_inner) + .is_some() + } + + pub async fn doctor(&self) -> BrowserDoctorReport { + let binary = match discover_binary() { + Ok(binary) => binary, + Err(reason) => { + self.inner.update_view(|view| { + view.status = BrowserStatus::Unavailable { + reason: reason.clone(), + }; + }); + return unavailable_report(&reason); + } + }; + let result = inspect_installation(&binary, &self.inner.launch_context).await; + let report = installation_report(&result); + if let Ok(installation) = result { + *self + .inner + .binary + .write() + .unwrap_or_else(PoisonError::into_inner) = Some(installation.path.clone()); + *self.inner.installation.lock().await = Some(installation); + self.inner.update_view(|view| { + if matches!(view.status, BrowserStatus::Unavailable { .. }) { + view.status = BrowserStatus::Idle; + } + }); + } + report + } + + pub fn subscribe(&self) -> watch::Receiver { + self.inner.updates.subscribe() + } + + #[expect( + clippy::await_holding_invalid_type, + reason = "policy changes must serialize with browser spawn and stateful CDP calls" + )] + pub async fn set_network_policy(&self, network_policy: BrowserNetworkPolicy) { + let _operation = self.inner.operation.lock().await; + let changed = { + let mut current = self + .inner + .network_policy + .write() + .unwrap_or_else(PoisonError::into_inner); + if *current == network_policy { + false + } else { + *current = network_policy; + true + } + }; + if !changed { + return; + } + self.inner.close_session().await; + } + + pub fn view(&self) -> BrowserView { + self.inner + .view + .read() + .unwrap_or_else(PoisonError::into_inner) + .clone() + } + + pub fn set_visibility(&self, visible: bool) { + self.inner.update_view(|view| view.visible = visible); + } + + pub fn resize(&self, size: TerminalSize) -> Result<()> { + anyhow::ensure!( + size.rows > 0 && size.cols > 0, + "browser size must be non-zero" + ); + let process = self + .inner + .process + .lock() + .unwrap_or_else(PoisonError::into_inner); + let changed = self.inner.resize_tx.send_if_modified(|current| { + if *current == size { + false + } else { + *current = size; + true + } + }); + if !changed { + return Ok(()); + } + if let Some(process) = process.as_ref() { + process.resize(size.into())?; + } + Ok(()) + } + + #[expect( + clippy::await_holding_invalid_type, + reason = "the operation guard prevents close from racing a stateful CDP call" + )] + pub async fn close(&self) { + let _operation = self.inner.operation.lock().await; + self.inner.close_session().await; + } +} + +impl Inner { + pub(crate) fn network_policy(&self) -> BrowserNetworkPolicy { + self.network_policy + .read() + .unwrap_or_else(PoisonError::into_inner) + .clone() + } + + pub(crate) fn set_crashed(&self, message: String) { + self.human_control.store(/*val*/ false, Ordering::SeqCst); + self.human_control_generation + .fetch_add(/*val*/ 1, Ordering::SeqCst); + self.update_view(|view| { + view.human_control = false; + view.status = BrowserStatus::Crashed { message }; + }); + } + + pub(crate) fn update_view(&self, update: impl FnOnce(&mut BrowserView)) { + update(&mut self.view.write().unwrap_or_else(PoisonError::into_inner)); + let sequence = self.update_sequence.fetch_add(/*val*/ 1, Ordering::Relaxed) + 1; + self.updates.send_replace(sequence); + } +} + +fn discover_binary() -> std::result::Result { + if !cfg!(any(target_os = "macos", target_os = "linux")) { + return Err("Carbonyl terminal browsing is only supported on macOS and Linux".to_string()); + } + if let Some(path) = std::env::var_os(CARBONYL_BINARY_ENV) { + let path = PathBuf::from(path); + return path.is_file().then_some(path).ok_or_else(|| { + format!("{CARBONYL_BINARY_ENV} does not point to a Carbonyl executable") + }); + } + which::which("carbonyl").map_err(|_| { + "Carbonyl was not found on PATH; install it or set CODEX_CARBONYL_BINARY".to_string() + }) +} diff --git a/codex-rs/terminal-browser/src/terminal_browser_tests.rs b/codex-rs/terminal-browser/src/terminal_browser_tests.rs new file mode 100644 index 0000000000..eee0d44e17 --- /dev/null +++ b/codex-rs/terminal-browser/src/terminal_browser_tests.rs @@ -0,0 +1,349 @@ +use pretty_assertions::assert_eq; + +use crate::BrowserColor; +use crate::BrowserLaunchContext; +use crate::BrowserNetworkPolicy; +use crate::TerminalBrowser; +use crate::TerminalSize; +use crate::actions::bounded_snapshot_json; +use crate::devtools::validated_websocket_url; +use crate::handles::BrowserHandles; +use crate::process::carbonyl_args; +use crate::screen::TerminalQueryResponder; +use crate::screen::TerminalScreen; +use crate::session::RenderMode; +use codex_utils_absolute_path::AbsolutePathBuf; +use std::sync::atomic::Ordering; + +#[tokio::test] +async fn open_requires_network_enabled_by_the_active_permission_profile() { + let browser = TerminalBrowser::discover(); + let error = browser + .execute( + "test-session", + "open", + serde_json::json!({ "url": "https://example.com" }), + ) + .await + .expect_err("disabled network policy should reject browser navigation"); + + assert_eq!( + error.to_string(), + "terminal browser network access is disabled by the active permission profile" + ); +} + +#[tokio::test] +async fn model_actions_are_rejected_during_human_control() { + let browser = TerminalBrowser::discover(); + browser + .inner + .human_control + .store(/*val*/ true, Ordering::SeqCst); + + let error = browser + .execute("test-session", "snapshot", serde_json::json!({})) + .await + .expect_err("model action should be rejected"); + + assert!(error.to_string().contains("human_control_active")); +} + +#[test] +fn terminal_browser_ignores_duplicate_resize_events() { + let browser = TerminalBrowser::discover(); + let mut resize_rx = browser.inner.resize_tx.subscribe(); + let size = TerminalSize { + rows: 40, + cols: 120, + }; + + browser.resize(size).expect("first resize should succeed"); + assert!(resize_rx.has_changed().expect("resize sender is open")); + assert_eq!(*resize_rx.borrow_and_update(), size); + + browser + .resize(size) + .expect("duplicate resize should succeed"); + assert!(!resize_rx.has_changed().expect("resize sender is open")); +} + +#[tokio::test] +async fn model_profile_requests_do_not_mutate_without_user_approval() { + let root = tempfile::tempdir().expect("test root"); + let root = AbsolutePathBuf::from_absolute_path(root.path()).expect("absolute test root"); + let browser = TerminalBrowser::discover_with_launch_context(BrowserLaunchContext { + codex_home: Some(root.join("codex-home")), + workspace_root: Some(root.join("workspace")), + ..Default::default() + }); + + let output = browser + .execute( + "test-session", + "profile", + serde_json::json!({ "action": "requestCreate", "name": "work" }), + ) + .await + .expect("profile request"); + let crate::BrowserToolOutput::Text(output) = output else { + panic!("expected text output"); + }; + let output: serde_json::Value = serde_json::from_str(&output).expect("approval JSON"); + + assert_eq!(output["status"], "approvalRequired"); + assert_eq!(output["command"], "/browser profile create work"); + assert!(browser.profiles().expect("profile listing").is_empty()); +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +#[tokio::test] +#[ignore = "opt-in smoke test requiring CODEX_CARBONYL_BINARY and a host sandbox"] +async fn configured_real_carbonyl_opens_local_page_and_snapshots() { + use std::io::Read; + use std::io::Write; + + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind smoke server"); + listener + .set_nonblocking(/*nonblocking*/ true) + .expect("configure smoke server"); + let address = listener.local_addr().expect("smoke server address"); + let server = std::thread::spawn(move || { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(/*secs*/ 15); + loop { + match listener.accept() { + Ok((mut stream, _)) => { + stream + .set_read_timeout(Some(std::time::Duration::from_secs(/*secs*/ 2))) + .expect("set smoke read timeout"); + let mut request = [0_u8; 2_048]; + let _ = stream.read(&mut request); + let body = + "Carbonyl smoke"; + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + stream + .write_all(response.as_bytes()) + .expect("write smoke response"); + return; + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + assert!( + std::time::Instant::now() < deadline, + "Carbonyl did not connect to the smoke server" + ); + std::thread::sleep(std::time::Duration::from_millis(/*millis*/ 10)); + } + Err(error) => panic!("accept smoke request: {error}"), + } + } + }); + let browser = TerminalBrowser::discover(); + browser + .resize(TerminalSize { + rows: 40, + cols: 120, + }) + .expect("enlarge startup diagnostics"); + browser + .set_network_policy(BrowserNetworkPolicy::Direct) + .await; + + let open_result = browser + .execute( + "real-smoke", + "open", + serde_json::json!({ + "url": format!("http://{address}"), + "visible": false, + }), + ) + .await; + if let Err(error) = open_result { + let view = browser.view(); + let screen = (0..view.screen.rows) + .map(|row| { + (0..view.screen.cols) + .filter_map(|col| view.screen.cell(row, col)) + .map(|cell| cell.text.as_str()) + .collect::() + .trim_end() + .to_string() + }) + .collect::>() + .join("\n"); + panic!("open local smoke page: {error}\nCarbonyl output:\n{screen}"); + } + let snapshot = browser + .execute("real-smoke", "snapshot", serde_json::json!({})) + .await + .expect("snapshot local smoke page"); + let crate::BrowserToolOutput::Text(snapshot) = snapshot else { + panic!("expected text snapshot"); + }; + + assert!(snapshot.contains("Smoke button")); + browser.close().await; + server.join().expect("smoke server thread"); +} + +#[tokio::test] +async fn terminal_browser_resize_updates_coalesce_to_the_latest_size() { + let browser = TerminalBrowser::discover(); + let mut resize_rx = browser.inner.resize_tx.subscribe(); + let sizes = [ + TerminalSize { + rows: 31, + cols: 101, + }, + TerminalSize { + rows: 32, + cols: 102, + }, + TerminalSize { + rows: 33, + cols: 103, + }, + ]; + + for size in sizes { + browser.resize(size).expect("resize should succeed"); + } + + resize_rx.changed().await.expect("resize sender is open"); + assert_eq!(*resize_rx.borrow_and_update(), sizes[2]); + assert!(!resize_rx.has_changed().expect("resize sender is open")); +} + +#[test] +fn terminal_screen_preserves_styles_wide_cells_cursor_and_title() { + let mut terminal = TerminalScreen::new(TerminalSize { rows: 2, cols: 8 }); + terminal.process(b"\x1b]2;Example"); + terminal.process(b"\x07\x1b[31;44;1;3;4m"); + terminal.process("\u{754c}".as_bytes()); + + let screen = terminal.snapshot(); + let cell = screen.cell(/*row*/ 0, /*col*/ 0).expect("first cell"); + assert_eq!(cell.text, "\u{754c}"); + assert_eq!(cell.foreground, BrowserColor::Indexed(1)); + assert_eq!(cell.background, BrowserColor::Indexed(4)); + assert!(cell.bold); + assert!(cell.italic); + assert!(cell.underlined); + assert!( + screen + .cell(/*row*/ 0, /*col*/ 1) + .expect("wide continuation") + .wide_continuation + ); + assert_eq!(screen.cursor, Some((0, 2))); + assert_eq!(terminal.title().as_deref(), Some("Example")); +} + +#[test] +fn terminal_query_responder_handles_queries_split_across_chunks() { + let mut responder = TerminalQueryResponder::default(); + assert_eq!(responder.process(b"prefix\x1bP$q"), Vec::>::new()); + assert_eq!( + responder.process(b"m\x1b\\suffix"), + vec![b"\x1bP1$r48:2:0:0:0m\x1b\\".to_vec()] + ); + assert_eq!(responder.process(b"\x1bP+q54"), Vec::>::new()); + assert_eq!( + responder.process(b"4e\x1b\\"), + vec![b"\x1bP1+r544e=787465726d2d323536636f6c6f72\x1b\\".to_vec()] + ); +} + +#[test] +fn carbonyl_uses_the_managed_proxy_only_when_policy_provides_one() { + let direct = carbonyl_args( + /*debugging_port*/ 9_222, + "/tmp/profile", + &BrowserNetworkPolicy::Direct, + RenderMode::NativeText, + ); + assert!(!direct.iter().any(|arg| arg.starts_with("--proxy-server="))); + + let http_addr = "127.0.0.1:43128".parse().expect("valid proxy address"); + let proxied = carbonyl_args( + /*debugging_port*/ 9_222, + "/tmp/profile", + &BrowserNetworkPolicy::ManagedProxy { http_addr }, + RenderMode::NativeText, + ); + assert!(proxied.contains(&"--proxy-server=http://127.0.0.1:43128".to_string())); + assert!(proxied.contains(&"--proxy-bypass-list=<-loopback>".to_string())); +} + +#[test] +fn carbonyl_devtools_websocket_must_match_the_private_loopback_listener() { + assert_eq!( + validated_websocket_url( + "ws://127.0.0.1:9222/devtools/page/1", + /*expected_port*/ 9_222, + ) + .unwrap(), + "ws://127.0.0.1:9222/devtools/page/1" + ); + assert!( + validated_websocket_url( + "ws://192.0.2.10:9222/devtools/page/1", + /*expected_port*/ 9_222, + ) + .is_err() + ); + assert!( + validated_websocket_url( + "ws://127.0.0.1:9223/devtools/page/1", + /*expected_port*/ 9_222, + ) + .is_err() + ); + assert!( + validated_websocket_url( + "wss://127.0.0.1:9222/devtools/page/1", + /*expected_port*/ 9_222, + ) + .is_err() + ); +} + +#[test] +fn browser_node_handles_reject_unknown_and_stale_ids() { + let mut handles = BrowserHandles::default(); + let node_id = handles.insert(/*backend_node_id*/ 42); + assert_eq!(handles.resolve(&node_id).expect("current handle"), 42); + assert!(handles.resolve("body > button").is_err()); + handles.clear(); + assert!(handles.resolve(&node_id).is_err()); +} + +#[test] +fn serialized_snapshot_has_a_hard_output_cap_and_remains_valid_json() { + let nodes = (0..100) + .map(|index| { + serde_json::json!({ + "nodeId": format!("d0123456789abcdefn{index}"), + "text": "x".repeat(/*n*/ 1_000), + "value": "y".repeat(/*n*/ 1_000), + }) + }) + .collect::>(); + let output = bounded_snapshot_json(serde_json::json!({ + "url": format!("https://example.com/{}", "u".repeat(/*n*/ 100_000)), + "title": "t".repeat(/*n*/ 100_000), + "nodes": nodes, + "text": "body ".repeat(/*n*/ 10_000), + })) + .unwrap(); + + assert!(output.len() <= 8 * 1024); + let parsed: serde_json::Value = serde_json::from_str(&output).unwrap(); + assert_eq!( + parsed.get("truncated"), + Some(&serde_json::Value::Bool(true)) + ); +} diff --git a/codex-rs/terminal-browser/src/tool_dispatch.rs b/codex-rs/terminal-browser/src/tool_dispatch.rs new file mode 100644 index 0000000000..8e0b68206a --- /dev/null +++ b/codex-rs/terminal-browser/src/tool_dispatch.rs @@ -0,0 +1,318 @@ +use std::sync::PoisonError; + +use anyhow::Context; +use anyhow::Result; +use anyhow::anyhow; +use serde::Deserialize; +use serde_json::Value; +use url::Url; + +use crate::accessibility; +use crate::actions; +use crate::actions::BrowserToolOutput; +use crate::navigation; +use crate::navigation::NavigateRequest; +use crate::navigation::NavigationAction; +use crate::navigation::WaitRequest; +use crate::network::BrowserNetworkPolicy; +use crate::process::BrowserSession; +use crate::screen::BrowserStatus; +use crate::session::RenderMode; +use crate::session::SessionConfig; +use crate::session::TerminalBrowser; +use crate::url_policy::is_allowed_browser_url; + +const MAX_URL_INPUT_BYTES: usize = 4 * 1024; +const MAX_FILL_TEXT_BYTES: usize = 64 * 1024; + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct OpenArgs { + url: String, + visible: Option, + render_mode: Option, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct NodeArgs { + node_id: String, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct FillArgs { + node_id: String, + text: String, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct PressArgs { + key: String, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct ScrollArgs { + #[serde(default)] + delta_x: i64, + #[serde(default)] + delta_y: i64, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct VisibilityArgs { + visible: bool, +} + +impl TerminalBrowser { + #[expect( + clippy::await_holding_invalid_type, + reason = "the operation and session guards serialize stateful CDP calls with lifecycle changes" + )] + pub async fn execute( + &self, + session_key: &str, + tool: &str, + arguments: Value, + ) -> Result { + if self.is_human_control_active() && !matches!(tool, "set_visibility" | "close") { + anyhow::bail!("human_control_active: return control to the agent before using tools"); + } + let _operation = self + .inner + .operation + .try_lock() + .map_err(|_| anyhow!("browser_busy: another terminal browser action is running"))?; + let session_changed = { + let mut current = self + .inner + .session_key + .lock() + .unwrap_or_else(PoisonError::into_inner); + if current.as_deref() == Some(session_key) { + false + } else { + *current = Some(session_key.to_string()); + true + } + }; + if session_changed { + self.inner.close_session().await; + } + if !matches!(tool, "open" | "set_visibility" | "close") { + let network_policy = self.inner.network_policy(); + let policy_changed = self + .inner + .session + .lock() + .await + .as_ref() + .is_some_and(|session| session.config.network_policy != network_policy); + if policy_changed { + self.inner.close_session().await; + anyhow::bail!("browser network policy changed; reopen the page before continuing"); + } + } + match tool { + "open" => self.open(serde_json::from_value(arguments)?).await, + "navigate" => { + let request: NavigateRequest = serde_json::from_value(arguments)?; + if request.action == NavigationAction::Goto { + let url = request + .url + .as_deref() + .context("navigate goto requires a URL")?; + validate_browser_url(url)?; + } + let mut session = self.inner.session.lock().await; + let session = session.as_mut().context("terminal browser is not open")?; + session.handles.clear(); + let output = navigation::navigate_request(&session.cdp, &request).await?; + self.refresh_page_metadata(session).await?; + Ok(output) + } + "wait" => { + let request: WaitRequest = serde_json::from_value(arguments)?; + let mut session = self.inner.session.lock().await; + let session = session.as_mut().context("terminal browser is not open")?; + let output = navigation::wait(&session.cdp, &session.handles, &request).await?; + self.refresh_page_metadata(session).await?; + Ok(output) + } + "profile" => self.execute_profile_tool(arguments), + "snapshot" => { + let mut session = self.inner.session.lock().await; + let session = session.as_mut().context("terminal browser is not open")?; + let output = accessibility::snapshot(&session.cdp, &mut session.handles).await?; + self.refresh_page_metadata(session).await?; + Ok(output) + } + "click" => { + let args: NodeArgs = serde_json::from_value(arguments)?; + let mut session = self.inner.session.lock().await; + let session = session.as_mut().context("terminal browser is not open")?; + let output = + accessibility::click(&session.cdp, &session.handles, &args.node_id).await?; + self.refresh_page_metadata(session).await?; + Ok(output) + } + "fill" => { + let args: FillArgs = serde_json::from_value(arguments)?; + anyhow::ensure!( + args.text.len() <= MAX_FILL_TEXT_BYTES, + "fill text exceeds the 64 KiB input limit" + ); + let mut session = self.inner.session.lock().await; + let session = session.as_mut().context("terminal browser is not open")?; + let output = + accessibility::fill(&session.cdp, &session.handles, &args.node_id, &args.text) + .await?; + self.refresh_page_metadata(session).await?; + Ok(output) + } + "press" => { + let args: PressArgs = serde_json::from_value(arguments)?; + let mut session = self.inner.session.lock().await; + let session = session.as_mut().context("terminal browser is not open")?; + let output = actions::press(&session.cdp, &args.key).await?; + self.refresh_page_metadata(session).await?; + Ok(output) + } + "scroll" => { + let args: ScrollArgs = serde_json::from_value(arguments)?; + let mut session = self.inner.session.lock().await; + let session = session.as_mut().context("terminal browser is not open")?; + let output = actions::scroll(&session.cdp, args.delta_x, args.delta_y).await?; + self.refresh_page_metadata(session).await?; + Ok(output) + } + "screenshot" => { + let mut session = self.inner.session.lock().await; + let session = session.as_mut().context("terminal browser is not open")?; + let output = actions::screenshot(&session.cdp).await?; + self.refresh_page_metadata(session).await?; + Ok(output) + } + "set_visibility" => { + let args: VisibilityArgs = serde_json::from_value(arguments)?; + self.set_visibility(args.visible); + Ok(BrowserToolOutput::Text(format!( + "terminal browser visibility set to {}", + args.visible + ))) + } + "close" => { + self.inner.close_session().await; + Ok(BrowserToolOutput::Text( + "terminal browser closed".to_string(), + )) + } + _ => anyhow::bail!("unknown terminal browser tool: {tool}"), + } + } + + #[expect( + clippy::await_holding_invalid_type, + reason = "the session guard keeps navigation and metadata refresh on the same CDP client" + )] + async fn open(&self, args: OpenArgs) -> Result { + let url = validate_browser_url(&args.url)?; + let network_policy = self.inner.network_policy(); + anyhow::ensure!( + !matches!(&network_policy, BrowserNetworkPolicy::Disabled), + "terminal browser network access is disabled by the active permission profile" + ); + let config = SessionConfig { + render_mode: args.render_mode.unwrap_or_default(), + network_policy, + }; + let visible = args.visible.unwrap_or(/*default*/ true); + self.inner.update_view(|view| { + view.status = BrowserStatus::Starting; + view.visible = visible; + view.url = Some(url.as_str().to_string()); + }); + if let Err(error) = self.inner.ensure_session(config).await { + self.inner.set_crashed(error.to_string()); + return Err(error); + } + + let mut session = self.inner.session.lock().await; + let session = session.as_mut().context("terminal browser did not start")?; + session.handles.clear(); + let metadata = match async { + navigation::navigate(&session.cdp, url.as_str()).await?; + let metadata = actions::page_metadata(&session.cdp).await?; + if let Some(final_url) = metadata.url.as_deref() + && !is_allowed_browser_url(final_url) + { + blank_disallowed_page(session).await; + anyhow::bail!("browser navigation blocked by the active permission policy"); + } + Ok::<_, anyhow::Error>(metadata) + } + .await + { + Ok(metadata) => metadata, + Err(error) => { + self.inner + .update_view(|view| view.status = BrowserStatus::Running); + return Err(error); + } + }; + let final_url = metadata.url.unwrap_or_else(|| url.as_str().to_string()); + self.inner.update_view(|view| { + view.status = BrowserStatus::Running; + view.title = metadata.title; + view.url = Some(final_url); + view.visible = visible; + }); + Ok(BrowserToolOutput::Text(format!( + "opened {} in the terminal browser", + url.as_str() + ))) + } + + async fn refresh_page_metadata(&self, session: &mut BrowserSession) -> Result<()> { + let metadata = actions::page_metadata(&session.cdp).await?; + if let Some(url) = metadata.url.as_deref() + && !is_allowed_browser_url(url) + { + blank_disallowed_page(session).await; + anyhow::bail!("browser navigation blocked by the active permission policy"); + } + self.inner.update_view(|view| { + view.url = metadata.url; + view.title = metadata.title; + }); + Ok(()) + } +} + +async fn blank_disallowed_page(session: &mut BrowserSession) { + let _ = session + .cdp + .call("Page.stopLoading", serde_json::json!({})) + .await; + let _ = session + .cdp + .call("Page.navigate", serde_json::json!({ "url": "about:blank" })) + .await; + session.handles.clear(); +} + +fn validate_browser_url(raw_url: &str) -> Result { + anyhow::ensure!( + raw_url.len() <= MAX_URL_INPUT_BYTES, + "URL exceeds the 4 KiB input limit" + ); + let url = Url::parse(raw_url).context("parse terminal browser URL")?; + anyhow::ensure!( + is_allowed_browser_url(url.as_str()), + "terminal_browser only supports http and https URLs" + ); + Ok(url) +} diff --git a/codex-rs/terminal-browser/src/url_policy.rs b/codex-rs/terminal-browser/src/url_policy.rs new file mode 100644 index 0000000000..fd90b7335d --- /dev/null +++ b/codex-rs/terminal-browser/src/url_policy.rs @@ -0,0 +1,56 @@ +use std::sync::Arc; + +use tokio::task::JoinHandle; +use url::Url; + +use crate::cdp::CdpClient; +use crate::cdp::CdpEvent; +use crate::session::Inner; + +pub(crate) fn is_allowed_browser_url(raw_url: &str) -> bool { + matches!( + Url::parse(raw_url).ok().as_ref().map(Url::scheme), + Some("http" | "https") + ) +} + +pub(crate) fn is_allowed_observed_url(raw_url: &str) -> bool { + raw_url == "about:blank" || is_allowed_browser_url(raw_url) +} + +pub(crate) fn spawn_navigation_policy_task(inner: Arc, cdp: CdpClient) -> JoinHandle<()> { + let mut events = cdp.subscribe_events(); + tokio::spawn(async move { + loop { + match events.recv().await { + Ok(CdpEvent::Message(message)) => { + let is_main_frame = message.get("method").and_then(serde_json::Value::as_str) + == Some("Page.frameNavigated") + && message.pointer("/params/frame/parentId").is_none(); + let Some(url) = is_main_frame + .then(|| message.pointer("/params/frame/url")) + .flatten() + .and_then(serde_json::Value::as_str) + else { + continue; + }; + if is_allowed_observed_url(url) { + continue; + } + tracing::warn!("blocked disallowed terminal-browser main-frame navigation"); + let _ = cdp.call("Page.stopLoading", serde_json::json!({})).await; + let _ = cdp + .call("Page.navigate", serde_json::json!({ "url": "about:blank" })) + .await; + inner.update_view(|view| { + view.url = Some("about:blank".to_string()); + view.title = Some("Blocked navigation".to_string()); + }); + } + Ok(CdpEvent::Disconnected(_)) + | Err(tokio::sync::broadcast::error::RecvError::Closed) => return, + Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => {} + } + } + }) +}