diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index be5d06e3dc..d3d082765e 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -1972,7 +1972,7 @@ impl CodexMessageProcessor { experimental_raw_events: bool, request_trace: Option, ) { - let config = match derive_config_from_params( + let mut config = match derive_config_from_params( &cli_overrides, config_overrides, typesafe_overrides, @@ -1992,6 +1992,11 @@ impl CodexMessageProcessor { } }; + if listener_task_context.remote_browser_api.is_configured() { + config.user_instructions = + append_remote_browser_user_instructions(config.user_instructions.take()); + } + let dynamic_tools = if listener_task_context.remote_browser_api.is_configured() { merge_atlas_command_dynamic_tool(dynamic_tools.unwrap_or_default()) } else { @@ -7823,6 +7828,31 @@ fn config_load_error(err: &std::io::Error) -> JSONRPCErrorError { } } +fn render_remote_browser_user_instructions() -> &'static str { + "## Remote Browsing (AtlasLib)\n\ +- This session exposes remote browsing inside `js_repl` via `AtlasLib` (also available as `codex.browser`).\n\ +- When the user asks to use the remote browser, use `js_repl` and this API instead of probing MCP resources, searching for `remote_browser` tools, or fetching pages directly.\n\ +- Common flow: `const tab = await AtlasLib.open(\"https://www.google.com\"); console.log(tab?.title ?? \"\");`\n\ +- Available methods: `open`, `createTab`, `listTabs`, `getSelectedTab`, `getTab`, `navigate`, `waitForLoadState`, `count`, `click`, `dblclick`, `waitFor`, `textContent`, `innerText`, `getAttribute`, `isVisible`, `isEnabled`, `screenshot`, `visibleScreenshot`, and `tabsContent`.\n\ +- `AtlasLib.screenshot(...)` and `AtlasLib.visibleScreenshot(...)` return `{ base64, dataUrl }`; use `await codex.emitImage((await AtlasLib.visibleScreenshot({ tabId })).dataUrl)` when you need to show the browser image in the response." +} + +fn append_remote_browser_user_instructions(existing: Option) -> Option { + let remote_browser_instructions = render_remote_browser_user_instructions(); + match existing { + Some(instructions) => { + if instructions.contains("## Remote Browsing (AtlasLib)") { + Some(instructions) + } else if instructions.trim().is_empty() { + Some(remote_browser_instructions.to_string()) + } else { + Some(format!("{instructions}\n\n{remote_browser_instructions}")) + } + } + None => Some(remote_browser_instructions.to_string()), + } +} + fn validate_dynamic_tools(tools: &[ApiDynamicToolSpec]) -> Result<(), String> { let mut seen = HashSet::new(); for tool in tools { @@ -8477,6 +8507,27 @@ mod tests { validate_dynamic_tools(&tools).expect("valid schema"); } + #[test] + fn append_remote_browser_user_instructions_appends_section() { + let instructions = + append_remote_browser_user_instructions(Some("base instructions".to_string())) + .expect("instructions"); + + assert!(instructions.contains("base instructions")); + assert!(instructions.contains("## Remote Browsing (AtlasLib)")); + assert!(instructions.contains("AtlasLib.open(\"https://www.google.com\")")); + } + + #[test] + fn append_remote_browser_user_instructions_is_idempotent() { + let once = append_remote_browser_user_instructions(None).expect("instructions"); + let twice = + append_remote_browser_user_instructions(Some(once.clone())).expect("instructions"); + + assert_eq!(twice.matches("## Remote Browsing (AtlasLib)").count(), 1); + assert_eq!(once, twice); + } + #[test] fn config_load_error_marks_cloud_requirements_failures_for_relogin() { let err = std::io::Error::other(CloudRequirementsLoadError::new( diff --git a/codex-rs/core/src/tools/js_repl/kernel.js b/codex-rs/core/src/tools/js_repl/kernel.js index 5d31818521..c4b3cee099 100644 --- a/codex-rs/core/src/tools/js_repl/kernel.js +++ b/codex-rs/core/src/tools/js_repl/kernel.js @@ -1437,6 +1437,338 @@ function normalizeEmitImageValue(value) { throw new Error("codex.emitImage received an unsupported value"); } +function contentItemsToText(items) { + if (!Array.isArray(items)) { + return null; + } + + const parts = []; + for (const item of items) { + if ( + isPlainObject(item) && + (item.type === "input_text" || item.type === "output_text") && + typeof item.text === "string" && + item.text + ) { + parts.push(item.text); + } + } + + return parts.length > 0 ? parts.join("\n") : null; +} + +function extractToolResponseText(value) { + if (typeof value === "string") { + return value; + } + + if (!isPlainObject(value)) { + return null; + } + + if (typeof value.output === "string") { + return value.output; + } + + const outputItems = contentItemsToText(value.output); + if (outputItems) { + return outputItems; + } + + if (isPlainObject(value.output)) { + if (typeof value.output.text === "string") { + return value.output.text; + } + const nestedItems = + contentItemsToText(value.output.content_items) ?? + contentItemsToText(value.output.contentItems); + if (nestedItems) { + return nestedItems; + } + } + + return contentItemsToText(value.content); +} + +function parseAtlasCommandResult(value) { + const text = extractToolResponseText(value); + if (typeof text !== "string" || !text) { + throw new Error("AtlasLib expected atlas_command to return JSON text"); + } + + let parsed; + try { + parsed = JSON.parse(text); + } catch (error) { + throw new Error( + `AtlasLib failed to parse atlas_command response: ${error.message}`, + ); + } + + if (!isPlainObject(parsed)) { + throw new Error("AtlasLib expected atlas_command to return a JSON object"); + } + + if (typeof parsed.error === "string" && parsed.error) { + throw new Error(parsed.error); + } + + if (parsed.type === "codex_repl") { + if ("command_output" in parsed) { + return parsed.command_output; + } + throw new Error("AtlasLib expected atlas_command response.command_output"); + } + + return parsed; +} + +function addDefinedField(target, key, value) { + if (value != null) { + target[key] = value; + } +} + +function normalizeAtlasScreenshotResult(result) { + if (!isPlainObject(result) || typeof result.data !== "string" || !result.data) { + return result; + } + + return { + base64: result.data, + dataUrl: `data:image/png;base64,${result.data}`, + }; +} + +function createAtlasLibApi(runTool) { + async function atlasCommand(payload) { + if (!isPlainObject(payload)) { + throw new Error("AtlasLib expects an object payload"); + } + if (typeof payload.type !== "string" || !payload.type) { + throw new Error("AtlasLib payload.type is required"); + } + return parseAtlasCommandResult( + await runTool("atlas_command", { + payload, + }), + ); + } + + const api = { + async runtimeConfig() { + return atlasCommand({ type: "runtime_config" }); + }, + + async createTab() { + return atlasCommand({ type: "create_tab" }); + }, + + async listTabs() { + const result = await atlasCommand({ type: "list_tabs" }); + return Array.isArray(result?.tabs) ? result.tabs : []; + }, + + async getSelectedTab() { + const selected = await atlasCommand({ type: "selected_tab" }); + if (!selected?.id) { + return null; + } + const tabs = await api.listTabs(); + return tabs.find((tab) => tab?.id === selected.id) ?? { id: selected.id }; + }, + + async getTab(tabId) { + const tabs = await api.listTabs(); + return tabs.find((tab) => tab?.id === tabId) ?? null; + }, + + async waitForLoadState(options = {}) { + const payload = { + type: "playwright_wait_for_load_state", + state: options.state ?? "load", + }; + addDefinedField(payload, "tab_id", options.tabId); + addDefinedField(payload, "timeout_ms", options.timeoutMs); + await atlasCommand(payload); + }, + + async navigate(url, options = {}) { + if (typeof url !== "string" || !url) { + throw new Error("AtlasLib.navigate expects a non-empty URL string"); + } + + const payload = { + type: "navigate_tab_url", + url, + }; + addDefinedField(payload, "tab_id", options.tabId); + addDefinedField(payload, "timeout_ms", options.timeoutMs); + await atlasCommand(payload); + + const waitForLoadState = options.waitForLoadState ?? options.waitFor ?? "load"; + if (waitForLoadState) { + await api.waitForLoadState({ + tabId: options.tabId, + state: waitForLoadState, + timeoutMs: options.timeoutMs, + }); + } + + return options.tabId ? api.getTab(options.tabId) : api.getSelectedTab(); + }, + + async open(url, options = {}) { + const tabId = options.tabId ?? (await api.createTab())?.id; + return api.navigate(url, { + ...options, + tabId, + }); + }, + + async count(selector, options = {}) { + const result = await atlasCommand({ + type: "playwright_locator_count", + selector, + tab_id: options.tabId, + timeout_ms: options.timeoutMs, + }); + return result?.count ?? result ?? 0; + }, + + async textContent(selector, options = {}) { + const result = await atlasCommand({ + type: "playwright_locator_text_content", + selector, + tab_id: options.tabId, + timeout_ms: options.timeoutMs, + }); + return result?.value ?? result ?? null; + }, + + async innerText(selector, options = {}) { + const result = await atlasCommand({ + type: "playwright_locator_inner_text", + selector, + tab_id: options.tabId, + timeout_ms: options.timeoutMs, + }); + return result?.value ?? result ?? null; + }, + + async getAttribute(selector, name, options = {}) { + const result = await atlasCommand({ + type: "playwright_locator_get_attribute", + selector, + name, + tab_id: options.tabId, + timeout_ms: options.timeoutMs, + }); + return result?.value ?? result ?? null; + }, + + async isVisible(selector, options = {}) { + const result = await atlasCommand({ + type: "playwright_locator_is_visible", + selector, + tab_id: options.tabId, + timeout_ms: options.timeoutMs, + }); + return result?.value ?? result ?? false; + }, + + async isEnabled(selector, options = {}) { + const result = await atlasCommand({ + type: "playwright_locator_is_enabled", + selector, + tab_id: options.tabId, + timeout_ms: options.timeoutMs, + }); + return result?.value ?? result ?? false; + }, + + async waitFor(selector, options = {}) { + const payload = { + type: "playwright_locator_wait_for", + selector, + state: options.state ?? "visible", + }; + addDefinedField(payload, "tab_id", options.tabId); + addDefinedField(payload, "timeout_ms", options.timeoutMs); + await atlasCommand(payload); + }, + + async click(selector, options = {}) { + const payload = { + type: "playwright_locator_click", + selector, + }; + addDefinedField(payload, "tab_id", options.tabId); + addDefinedField(payload, "timeout_ms", options.timeoutMs); + addDefinedField(payload, "modifiers", options.modifiers); + addDefinedField(payload, "button", options.button); + addDefinedField(payload, "force", options.force); + await atlasCommand(payload); + }, + + async dblclick(selector, options = {}) { + const payload = { + type: "playwright_locator_dblclick", + selector, + }; + addDefinedField(payload, "tab_id", options.tabId); + addDefinedField(payload, "timeout_ms", options.timeoutMs); + addDefinedField(payload, "modifiers", options.modifiers); + addDefinedField(payload, "button", options.button); + addDefinedField(payload, "force", options.force); + await atlasCommand(payload); + }, + + async screenshot(options = {}) { + const payload = { + type: "playwright_screenshot", + }; + addDefinedField(payload, "tab_id", options.tabId); + addDefinedField(payload, "timeout_ms", options.timeoutMs); + addDefinedField(payload, "full_page", options.fullPage); + return normalizeAtlasScreenshotResult(await atlasCommand(payload)); + }, + + async visibleScreenshot(options = {}) { + const tabId = options.tabId ?? (await api.getSelectedTab())?.id; + if (!tabId) { + throw new Error( + "AtlasLib.visibleScreenshot requires a tabId or an active selected tab", + ); + } + return normalizeAtlasScreenshotResult( + await atlasCommand({ + type: "cua_get_visible_screenshot", + tab_id: tabId, + }), + ); + }, + + async tabsContent(urls, options = {}) { + if (!Array.isArray(urls) || urls.length === 0) { + throw new Error("AtlasLib.tabsContent expects a non-empty array of URLs"); + } + + const payload = { + type: "tabs_content", + urls, + content_type: "text", + }; + addDefinedField(payload, "timeout_ms", options.timeoutMs); + + const result = await atlasCommand(payload); + return Array.isArray(result?.results) ? result.results : []; + }, + }; + + return api; +} + const codex = { cwd, homeDir, @@ -1538,6 +1870,9 @@ const codex = { }; }, }; +const atlasLib = createAtlasLibApi(codex.tool); +codex.browser = atlasLib; +codex.atlas = atlasLib; async function handleExec(message) { clearLocalFileModuleCaches(); @@ -1576,6 +1911,7 @@ async function handleExec(message) { let output = ""; context.codex = codex; + context.AtlasLib = atlasLib; context.tmpDir = tmpDir; await execContextStorage.run(execState, async () => { diff --git a/codex-rs/core/src/tools/js_repl/mod_tests.rs b/codex-rs/core/src/tools/js_repl/mod_tests.rs index 413e171d41..9d18bcac6d 100644 --- a/codex-rs/core/src/tools/js_repl/mod_tests.rs +++ b/codex-rs/core/src/tools/js_repl/mod_tests.rs @@ -2084,6 +2084,140 @@ console.log(JSON.stringify(out)); Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn js_repl_atlas_lib_open_uses_hidden_atlas_command() -> anyhow::Result<()> { + if !can_run_js_repl_runtime_tests().await { + return Ok(()); + } + + let (session, turn, rx_event) = + make_session_and_context_with_dynamic_tools_and_rx(vec![DynamicToolSpec { + name: "atlas_command".to_string(), + description: "Hidden bridge for AgentLib inside js_repl.".to_string(), + input_schema: serde_json::json!({ + "type": "object", + "properties": { + "payload": { + "type": "object", + "additionalProperties": true + } + }, + "required": ["payload"], + "additionalProperties": false + }), + defer_loading: true, + }]) + .await; + + *session.active_turn.lock().await = Some(crate::state::ActiveTurn::default()); + + let tracker = Arc::new(tokio::sync::Mutex::new(TurnDiffTracker::default())); + let manager = turn.js_repl.manager().await?; + let code = r#" +const tab = await AtlasLib.open("https://www.google.com"); +console.log(tab?.title ?? ""); +"#; + + let session_for_response = Arc::clone(&session); + let response_watcher = async move { + let expected = [ + ( + "create_tab", + serde_json::json!({ + "type": "codex_repl", + "command_output": { "id": "tab-1" } + }), + ), + ( + "navigate_tab_url", + serde_json::json!({ + "type": "codex_repl", + "command_output": {} + }), + ), + ( + "playwright_wait_for_load_state", + serde_json::json!({ + "type": "codex_repl", + "command_output": {} + }), + ), + ( + "selected_tab", + serde_json::json!({ + "type": "codex_repl", + "command_output": { "id": "tab-1" } + }), + ), + ( + "list_tabs", + serde_json::json!({ + "type": "codex_repl", + "command_output": { + "tabs": [ + { + "id": "tab-1", + "title": "Google", + "url": "https://www.google.com" + } + ] + } + }), + ), + ]; + + let mut index = 0usize; + loop { + let event = tokio::time::timeout(Duration::from_secs(2), rx_event.recv()).await??; + if let EventMsg::DynamicToolCallRequest(request) = event.msg { + assert_eq!(request.tool, "atlas_command"); + let payload_type = request + .arguments + .get("payload") + .and_then(|value| value.get("type")) + .and_then(serde_json::Value::as_str); + let (expected_type, response_payload) = &expected[index]; + assert_eq!(payload_type, Some(*expected_type)); + session_for_response + .notify_dynamic_tool_response( + &request.call_id, + DynamicToolResponse { + content_items: vec![DynamicToolCallOutputContentItem::InputText { + text: response_payload.to_string(), + }], + success: true, + }, + ) + .await; + index += 1; + if index == expected.len() { + return Ok::<(), anyhow::Error>(()); + } + } + } + }; + + let (result, response_watcher_result) = tokio::join!( + manager.execute( + Arc::clone(&session), + Arc::clone(&turn), + tracker, + JsReplArgs { + code: code.to_string(), + timeout_ms: Some(15_000), + }, + ), + response_watcher, + ); + + let result = result?; + response_watcher_result?; + assert_eq!(result.output, "Google"); + assert!(session.get_pending_input().await.is_empty()); + + Ok(()) +} + #[tokio::test] async fn js_repl_prefers_env_node_module_dirs_over_config() -> anyhow::Result<()> { if !can_run_js_repl_runtime_tests().await {