Expose AtlasLib in js_repl for remote browsing

Co-authored-by: Codex <noreply@openai.com>
2026-11-09 20:50:33 +00:00 · 2026-03-25 04:16:26 +00:00
parent 3bbd8155b3
commit 1991fb9d6d
3 changed files with 522 additions and 1 deletions
@@ -1972,7 +1972,7 @@ impl CodexMessageProcessor {
        experimental_raw_events: bool,
        request_trace: Option<W3cTraceContext>,
    ) {
-        let config = match derive_config_from_params(
+        let mut config = match derive_config_from_params(
            &cli_overrides,
            config_overrides,
            typesafe_overrides,
@@ -1992,6 +1992,11 @@ impl CodexMessageProcessor {
            }
        };

+        if listener_task_context.remote_browser_api.is_configured() {
+            config.user_instructions =
+                append_remote_browser_user_instructions(config.user_instructions.take());
+        }
+
        let dynamic_tools = if listener_task_context.remote_browser_api.is_configured() {
            merge_atlas_command_dynamic_tool(dynamic_tools.unwrap_or_default())
        } else {
@@ -7823,6 +7828,31 @@ fn config_load_error(err: &std::io::Error) -> JSONRPCErrorError {
    }
 }

+fn render_remote_browser_user_instructions() -> &'static str {
+    "## Remote Browsing (AtlasLib)\n\
+- This session exposes remote browsing inside `js_repl` via `AtlasLib` (also available as `codex.browser`).\n\
+- When the user asks to use the remote browser, use `js_repl` and this API instead of probing MCP resources, searching for `remote_browser` tools, or fetching pages directly.\n\
+- Common flow: `const tab = await AtlasLib.open(\"https://www.google.com\"); console.log(tab?.title ?? \"\");`\n\
+- Available methods: `open`, `createTab`, `listTabs`, `getSelectedTab`, `getTab`, `navigate`, `waitForLoadState`, `count`, `click`, `dblclick`, `waitFor`, `textContent`, `innerText`, `getAttribute`, `isVisible`, `isEnabled`, `screenshot`, `visibleScreenshot`, and `tabsContent`.\n\
+- `AtlasLib.screenshot(...)` and `AtlasLib.visibleScreenshot(...)` return `{ base64, dataUrl }`; use `await codex.emitImage((await AtlasLib.visibleScreenshot({ tabId })).dataUrl)` when you need to show the browser image in the response."
+}
+
+fn append_remote_browser_user_instructions(existing: Option<String>) -> Option<String> {
+    let remote_browser_instructions = render_remote_browser_user_instructions();
+    match existing {
+        Some(instructions) => {
+            if instructions.contains("## Remote Browsing (AtlasLib)") {
+                Some(instructions)
+            } else if instructions.trim().is_empty() {
+                Some(remote_browser_instructions.to_string())
+            } else {
+                Some(format!("{instructions}\n\n{remote_browser_instructions}"))
+            }
+        }
+        None => Some(remote_browser_instructions.to_string()),
+    }
+}
+
 fn validate_dynamic_tools(tools: &[ApiDynamicToolSpec]) -> Result<(), String> {
    let mut seen = HashSet::new();
    for tool in tools {
@@ -8477,6 +8507,27 @@ mod tests {
        validate_dynamic_tools(&tools).expect("valid schema");
    }

+    #[test]
+    fn append_remote_browser_user_instructions_appends_section() {
+        let instructions =
+            append_remote_browser_user_instructions(Some("base instructions".to_string()))
+                .expect("instructions");
+
+        assert!(instructions.contains("base instructions"));
+        assert!(instructions.contains("## Remote Browsing (AtlasLib)"));
+        assert!(instructions.contains("AtlasLib.open(\"https://www.google.com\")"));
+    }
+
+    #[test]
+    fn append_remote_browser_user_instructions_is_idempotent() {
+        let once = append_remote_browser_user_instructions(None).expect("instructions");
+        let twice =
+            append_remote_browser_user_instructions(Some(once.clone())).expect("instructions");
+
+        assert_eq!(twice.matches("## Remote Browsing (AtlasLib)").count(), 1);
+        assert_eq!(once, twice);
+    }
+
    #[test]
    fn config_load_error_marks_cloud_requirements_failures_for_relogin() {
        let err = std::io::Error::other(CloudRequirementsLoadError::new(
@@ -1437,6 +1437,338 @@ function normalizeEmitImageValue(value) {
  throw new Error("codex.emitImage received an unsupported value");
 }

+function contentItemsToText(items) {
+  if (!Array.isArray(items)) {
+    return null;
+  }
+
+  const parts = [];
+  for (const item of items) {
+    if (
+      isPlainObject(item) &&
+      (item.type === "input_text" || item.type === "output_text") &&
+      typeof item.text === "string" &&
+      item.text
+    ) {
+      parts.push(item.text);
+    }
+  }
+
+  return parts.length > 0 ? parts.join("\n") : null;
+}
+
+function extractToolResponseText(value) {
+  if (typeof value === "string") {
+    return value;
+  }
+
+  if (!isPlainObject(value)) {
+    return null;
+  }
+
+  if (typeof value.output === "string") {
+    return value.output;
+  }
+
+  const outputItems = contentItemsToText(value.output);
+  if (outputItems) {
+    return outputItems;
+  }
+
+  if (isPlainObject(value.output)) {
+    if (typeof value.output.text === "string") {
+      return value.output.text;
+    }
+    const nestedItems =
+      contentItemsToText(value.output.content_items) ??
+      contentItemsToText(value.output.contentItems);
+    if (nestedItems) {
+      return nestedItems;
+    }
+  }
+
+  return contentItemsToText(value.content);
+}
+
+function parseAtlasCommandResult(value) {
+  const text = extractToolResponseText(value);
+  if (typeof text !== "string" || !text) {
+    throw new Error("AtlasLib expected atlas_command to return JSON text");
+  }
+
+  let parsed;
+  try {
+    parsed = JSON.parse(text);
+  } catch (error) {
+    throw new Error(
+      `AtlasLib failed to parse atlas_command response: ${error.message}`,
+    );
+  }
+
+  if (!isPlainObject(parsed)) {
+    throw new Error("AtlasLib expected atlas_command to return a JSON object");
+  }
+
+  if (typeof parsed.error === "string" && parsed.error) {
+    throw new Error(parsed.error);
+  }
+
+  if (parsed.type === "codex_repl") {
+    if ("command_output" in parsed) {
+      return parsed.command_output;
+    }
+    throw new Error("AtlasLib expected atlas_command response.command_output");
+  }
+
+  return parsed;
+}
+
+function addDefinedField(target, key, value) {
+  if (value != null) {
+    target[key] = value;
+  }
+}
+
+function normalizeAtlasScreenshotResult(result) {
+  if (!isPlainObject(result) || typeof result.data !== "string" || !result.data) {
+    return result;
+  }
+
+  return {
+    base64: result.data,
+    dataUrl: `data:image/png;base64,${result.data}`,
+  };
+}
+
+function createAtlasLibApi(runTool) {
+  async function atlasCommand(payload) {
+    if (!isPlainObject(payload)) {
+      throw new Error("AtlasLib expects an object payload");
+    }
+    if (typeof payload.type !== "string" || !payload.type) {
+      throw new Error("AtlasLib payload.type is required");
+    }
+    return parseAtlasCommandResult(
+      await runTool("atlas_command", {
+        payload,
+      }),
+    );
+  }
+
+  const api = {
+    async runtimeConfig() {
+      return atlasCommand({ type: "runtime_config" });
+    },
+
+    async createTab() {
+      return atlasCommand({ type: "create_tab" });
+    },
+
+    async listTabs() {
+      const result = await atlasCommand({ type: "list_tabs" });
+      return Array.isArray(result?.tabs) ? result.tabs : [];
+    },
+
+    async getSelectedTab() {
+      const selected = await atlasCommand({ type: "selected_tab" });
+      if (!selected?.id) {
+        return null;
+      }
+      const tabs = await api.listTabs();
+      return tabs.find((tab) => tab?.id === selected.id) ?? { id: selected.id };
+    },
+
+    async getTab(tabId) {
+      const tabs = await api.listTabs();
+      return tabs.find((tab) => tab?.id === tabId) ?? null;
+    },
+
+    async waitForLoadState(options = {}) {
+      const payload = {
+        type: "playwright_wait_for_load_state",
+        state: options.state ?? "load",
+      };
+      addDefinedField(payload, "tab_id", options.tabId);
+      addDefinedField(payload, "timeout_ms", options.timeoutMs);
+      await atlasCommand(payload);
+    },
+
+    async navigate(url, options = {}) {
+      if (typeof url !== "string" || !url) {
+        throw new Error("AtlasLib.navigate expects a non-empty URL string");
+      }
+
+      const payload = {
+        type: "navigate_tab_url",
+        url,
+      };
+      addDefinedField(payload, "tab_id", options.tabId);
+      addDefinedField(payload, "timeout_ms", options.timeoutMs);
+      await atlasCommand(payload);
+
+      const waitForLoadState = options.waitForLoadState ?? options.waitFor ?? "load";
+      if (waitForLoadState) {
+        await api.waitForLoadState({
+          tabId: options.tabId,
+          state: waitForLoadState,
+          timeoutMs: options.timeoutMs,
+        });
+      }
+
+      return options.tabId ? api.getTab(options.tabId) : api.getSelectedTab();
+    },
+
+    async open(url, options = {}) {
+      const tabId = options.tabId ?? (await api.createTab())?.id;
+      return api.navigate(url, {
+        ...options,
+        tabId,
+      });
+    },
+
+    async count(selector, options = {}) {
+      const result = await atlasCommand({
+        type: "playwright_locator_count",
+        selector,
+        tab_id: options.tabId,
+        timeout_ms: options.timeoutMs,
+      });
+      return result?.count ?? result ?? 0;
+    },
+
+    async textContent(selector, options = {}) {
+      const result = await atlasCommand({
+        type: "playwright_locator_text_content",
+        selector,
+        tab_id: options.tabId,
+        timeout_ms: options.timeoutMs,
+      });
+      return result?.value ?? result ?? null;
+    },
+
+    async innerText(selector, options = {}) {
+      const result = await atlasCommand({
+        type: "playwright_locator_inner_text",
+        selector,
+        tab_id: options.tabId,
+        timeout_ms: options.timeoutMs,
+      });
+      return result?.value ?? result ?? null;
+    },
+
+    async getAttribute(selector, name, options = {}) {
+      const result = await atlasCommand({
+        type: "playwright_locator_get_attribute",
+        selector,
+        name,
+        tab_id: options.tabId,
+        timeout_ms: options.timeoutMs,
+      });
+      return result?.value ?? result ?? null;
+    },
+
+    async isVisible(selector, options = {}) {
+      const result = await atlasCommand({
+        type: "playwright_locator_is_visible",
+        selector,
+        tab_id: options.tabId,
+        timeout_ms: options.timeoutMs,
+      });
+      return result?.value ?? result ?? false;
+    },
+
+    async isEnabled(selector, options = {}) {
+      const result = await atlasCommand({
+        type: "playwright_locator_is_enabled",
+        selector,
+        tab_id: options.tabId,
+        timeout_ms: options.timeoutMs,
+      });
+      return result?.value ?? result ?? false;
+    },
+
+    async waitFor(selector, options = {}) {
+      const payload = {
+        type: "playwright_locator_wait_for",
+        selector,
+        state: options.state ?? "visible",
+      };
+      addDefinedField(payload, "tab_id", options.tabId);
+      addDefinedField(payload, "timeout_ms", options.timeoutMs);
+      await atlasCommand(payload);
+    },
+
+    async click(selector, options = {}) {
+      const payload = {
+        type: "playwright_locator_click",
+        selector,
+      };
+      addDefinedField(payload, "tab_id", options.tabId);
+      addDefinedField(payload, "timeout_ms", options.timeoutMs);
+      addDefinedField(payload, "modifiers", options.modifiers);
+      addDefinedField(payload, "button", options.button);
+      addDefinedField(payload, "force", options.force);
+      await atlasCommand(payload);
+    },
+
+    async dblclick(selector, options = {}) {
+      const payload = {
+        type: "playwright_locator_dblclick",
+        selector,
+      };
+      addDefinedField(payload, "tab_id", options.tabId);
+      addDefinedField(payload, "timeout_ms", options.timeoutMs);
+      addDefinedField(payload, "modifiers", options.modifiers);
+      addDefinedField(payload, "button", options.button);
+      addDefinedField(payload, "force", options.force);
+      await atlasCommand(payload);
+    },
+
+    async screenshot(options = {}) {
+      const payload = {
+        type: "playwright_screenshot",
+      };
+      addDefinedField(payload, "tab_id", options.tabId);
+      addDefinedField(payload, "timeout_ms", options.timeoutMs);
+      addDefinedField(payload, "full_page", options.fullPage);
+      return normalizeAtlasScreenshotResult(await atlasCommand(payload));
+    },
+
+    async visibleScreenshot(options = {}) {
+      const tabId = options.tabId ?? (await api.getSelectedTab())?.id;
+      if (!tabId) {
+        throw new Error(
+          "AtlasLib.visibleScreenshot requires a tabId or an active selected tab",
+        );
+      }
+      return normalizeAtlasScreenshotResult(
+        await atlasCommand({
+          type: "cua_get_visible_screenshot",
+          tab_id: tabId,
+        }),
+      );
+    },
+
+    async tabsContent(urls, options = {}) {
+      if (!Array.isArray(urls) || urls.length === 0) {
+        throw new Error("AtlasLib.tabsContent expects a non-empty array of URLs");
+      }
+
+      const payload = {
+        type: "tabs_content",
+        urls,
+        content_type: "text",
+      };
+      addDefinedField(payload, "timeout_ms", options.timeoutMs);
+
+      const result = await atlasCommand(payload);
+      return Array.isArray(result?.results) ? result.results : [];
+    },
+  };
+
+  return api;
+}
+
 const codex = {
  cwd,
  homeDir,
@@ -1538,6 +1870,9 @@ const codex = {
    };
  },
 };
+const atlasLib = createAtlasLibApi(codex.tool);
+codex.browser = atlasLib;
+codex.atlas = atlasLib;

 async function handleExec(message) {
  clearLocalFileModuleCaches();
@@ -1576,6 +1911,7 @@ async function handleExec(message) {
    let output = "";

    context.codex = codex;
+    context.AtlasLib = atlasLib;
    context.tmpDir = tmpDir;

    await execContextStorage.run(execState, async () => {
@@ -2084,6 +2084,140 @@ console.log(JSON.stringify(out));
    Ok(())
 }

+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn js_repl_atlas_lib_open_uses_hidden_atlas_command() -> anyhow::Result<()> {
+    if !can_run_js_repl_runtime_tests().await {
+        return Ok(());
+    }
+
+    let (session, turn, rx_event) =
+        make_session_and_context_with_dynamic_tools_and_rx(vec![DynamicToolSpec {
+            name: "atlas_command".to_string(),
+            description: "Hidden bridge for AgentLib inside js_repl.".to_string(),
+            input_schema: serde_json::json!({
+                "type": "object",
+                "properties": {
+                    "payload": {
+                        "type": "object",
+                        "additionalProperties": true
+                    }
+                },
+                "required": ["payload"],
+                "additionalProperties": false
+            }),
+            defer_loading: true,
+        }])
+        .await;
+
+    *session.active_turn.lock().await = Some(crate::state::ActiveTurn::default());
+
+    let tracker = Arc::new(tokio::sync::Mutex::new(TurnDiffTracker::default()));
+    let manager = turn.js_repl.manager().await?;
+    let code = r#"
+const tab = await AtlasLib.open("https://www.google.com");
+console.log(tab?.title ?? "");
+"#;
+
+    let session_for_response = Arc::clone(&session);
+    let response_watcher = async move {
+        let expected = [
+            (
+                "create_tab",
+                serde_json::json!({
+                    "type": "codex_repl",
+                    "command_output": { "id": "tab-1" }
+                }),
+            ),
+            (
+                "navigate_tab_url",
+                serde_json::json!({
+                    "type": "codex_repl",
+                    "command_output": {}
+                }),
+            ),
+            (
+                "playwright_wait_for_load_state",
+                serde_json::json!({
+                    "type": "codex_repl",
+                    "command_output": {}
+                }),
+            ),
+            (
+                "selected_tab",
+                serde_json::json!({
+                    "type": "codex_repl",
+                    "command_output": { "id": "tab-1" }
+                }),
+            ),
+            (
+                "list_tabs",
+                serde_json::json!({
+                    "type": "codex_repl",
+                    "command_output": {
+                        "tabs": [
+                            {
+                                "id": "tab-1",
+                                "title": "Google",
+                                "url": "https://www.google.com"
+                            }
+                        ]
+                    }
+                }),
+            ),
+        ];
+
+        let mut index = 0usize;
+        loop {
+            let event = tokio::time::timeout(Duration::from_secs(2), rx_event.recv()).await??;
+            if let EventMsg::DynamicToolCallRequest(request) = event.msg {
+                assert_eq!(request.tool, "atlas_command");
+                let payload_type = request
+                    .arguments
+                    .get("payload")
+                    .and_then(|value| value.get("type"))
+                    .and_then(serde_json::Value::as_str);
+                let (expected_type, response_payload) = &expected[index];
+                assert_eq!(payload_type, Some(*expected_type));
+                session_for_response
+                    .notify_dynamic_tool_response(
+                        &request.call_id,
+                        DynamicToolResponse {
+                            content_items: vec![DynamicToolCallOutputContentItem::InputText {
+                                text: response_payload.to_string(),
+                            }],
+                            success: true,
+                        },
+                    )
+                    .await;
+                index += 1;
+                if index == expected.len() {
+                    return Ok::<(), anyhow::Error>(());
+                }
+            }
+        }
+    };
+
+    let (result, response_watcher_result) = tokio::join!(
+        manager.execute(
+            Arc::clone(&session),
+            Arc::clone(&turn),
+            tracker,
+            JsReplArgs {
+                code: code.to_string(),
+                timeout_ms: Some(15_000),
+            },
+        ),
+        response_watcher,
+    );
+
+    let result = result?;
+    response_watcher_result?;
+    assert_eq!(result.output, "Google");
+    assert!(session.get_pending_input().await.is_empty());
+
+    Ok(())
+}
+
 #[tokio::test]
 async fn js_repl_prefers_env_node_module_dirs_over_config() -> anyhow::Result<()> {
    if !can_run_js_repl_runtime_tests().await {