From 4b69a17b9fc115973ae96c9d9c718243d6d81585 Mon Sep 17 00:00:00 2001 From: Liang-Ting Jiang Date: Tue, 24 Mar 2026 03:58:02 +0000 Subject: [PATCH] Add app-server remote browser bridge Add an experimental browserSession command/notification pair, post commands to a configurable remote browser HTTP endpoint, and emit browser state updates back to clients. Co-authored-by: Codex --- .../schema/json/ServerNotification.json | 76 ++++++++++++ .../codex_app_server_protocol.schemas.json | 78 ++++++++++++ .../codex_app_server_protocol.v2.schemas.json | 78 ++++++++++++ .../v2/BrowserSessionUpdatedNotification.json | 60 ++++++++++ .../schema/typescript/ServerNotification.ts | 3 +- .../typescript/v2/BrowserSessionState.ts | 6 + .../v2/BrowserSessionUpdatedNotification.ts | 6 + .../schema/typescript/v2/BrowserTabState.ts | 5 + .../schema/typescript/v2/index.ts | 3 + .../src/protocol/common.rs | 105 ++++++++++++++++ .../app-server-protocol/src/protocol/v2.rs | 46 +++++++ codex-rs/app-server/Cargo.toml | 1 + .../app-server/src/codex_message_processor.rs | 3 + codex-rs/app-server/src/in_process.rs | 1 + codex-rs/app-server/src/lib.rs | 5 + codex-rs/app-server/src/main.rs | 6 + codex-rs/app-server/src/message_processor.rs | 41 +++++++ .../src/message_processor/tracing_tests.rs | 113 +++++++++++++++++- codex-rs/app-server/src/remote_browser_api.rs | 65 ++++++++++ 19 files changed, 696 insertions(+), 5 deletions(-) create mode 100644 codex-rs/app-server-protocol/schema/json/v2/BrowserSessionUpdatedNotification.json create mode 100644 codex-rs/app-server-protocol/schema/typescript/v2/BrowserSessionState.ts create mode 100644 codex-rs/app-server-protocol/schema/typescript/v2/BrowserSessionUpdatedNotification.ts create mode 100644 codex-rs/app-server-protocol/schema/typescript/v2/BrowserTabState.ts create mode 100644 codex-rs/app-server/src/remote_browser_api.rs diff --git a/codex-rs/app-server-protocol/schema/json/ServerNotification.json b/codex-rs/app-server-protocol/schema/json/ServerNotification.json index b576581cf9..c239546e22 100644 --- a/codex-rs/app-server-protocol/schema/json/ServerNotification.json +++ b/codex-rs/app-server-protocol/schema/json/ServerNotification.json @@ -384,6 +384,62 @@ } ] }, + "BrowserSessionState": { + "properties": { + "selectedTabId": { + "type": "string" + }, + "tabs": { + "items": { + "$ref": "#/definitions/BrowserTabState" + }, + "type": "array" + } + }, + "required": [ + "selectedTabId", + "tabs" + ], + "type": "object" + }, + "BrowserSessionUpdatedNotification": { + "properties": { + "browserSessionId": { + "type": "string" + }, + "browserState": { + "$ref": "#/definitions/BrowserSessionState" + } + }, + "required": [ + "browserSessionId", + "browserState" + ], + "type": "object" + }, + "BrowserTabState": { + "properties": { + "id": { + "type": "string" + }, + "selected": { + "type": "boolean" + }, + "title": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "id", + "selected", + "title", + "url" + ], + "type": "object" + }, "ByteRange": { "properties": { "end": { @@ -4394,6 +4450,26 @@ "title": "App/list/updatedNotification", "type": "object" }, + { + "properties": { + "method": { + "enum": [ + "browserSession/updated" + ], + "title": "BrowserSession/updatedNotificationMethod", + "type": "string" + }, + "params": { + "$ref": "#/definitions/BrowserSessionUpdatedNotification" + } + }, + "required": [ + "method", + "params" + ], + "title": "BrowserSession/updatedNotification", + "type": "object" + }, { "properties": { "method": { diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index a644549325..90a0366d41 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -4053,6 +4053,26 @@ "title": "App/list/updatedNotification", "type": "object" }, + { + "properties": { + "method": { + "enum": [ + "browserSession/updated" + ], + "title": "BrowserSession/updatedNotificationMethod", + "type": "string" + }, + "params": { + "$ref": "#/definitions/v2/BrowserSessionUpdatedNotification" + } + }, + "required": [ + "method", + "params" + ], + "title": "BrowserSession/updatedNotification", + "type": "object" + }, { "properties": { "method": { @@ -5495,6 +5515,64 @@ } ] }, + "BrowserSessionState": { + "properties": { + "selectedTabId": { + "type": "string" + }, + "tabs": { + "items": { + "$ref": "#/definitions/v2/BrowserTabState" + }, + "type": "array" + } + }, + "required": [ + "selectedTabId", + "tabs" + ], + "type": "object" + }, + "BrowserSessionUpdatedNotification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "browserSessionId": { + "type": "string" + }, + "browserState": { + "$ref": "#/definitions/v2/BrowserSessionState" + } + }, + "required": [ + "browserSessionId", + "browserState" + ], + "title": "BrowserSessionUpdatedNotification", + "type": "object" + }, + "BrowserTabState": { + "properties": { + "id": { + "type": "string" + }, + "selected": { + "type": "boolean" + }, + "title": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "id", + "selected", + "title", + "url" + ], + "type": "object" + }, "ByteRange": { "properties": { "end": { diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index 71e6034529..e4889c6ee7 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -715,6 +715,64 @@ } ] }, + "BrowserSessionState": { + "properties": { + "selectedTabId": { + "type": "string" + }, + "tabs": { + "items": { + "$ref": "#/definitions/BrowserTabState" + }, + "type": "array" + } + }, + "required": [ + "selectedTabId", + "tabs" + ], + "type": "object" + }, + "BrowserSessionUpdatedNotification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "browserSessionId": { + "type": "string" + }, + "browserState": { + "$ref": "#/definitions/BrowserSessionState" + } + }, + "required": [ + "browserSessionId", + "browserState" + ], + "title": "BrowserSessionUpdatedNotification", + "type": "object" + }, + "BrowserTabState": { + "properties": { + "id": { + "type": "string" + }, + "selected": { + "type": "boolean" + }, + "title": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "id", + "selected", + "title", + "url" + ], + "type": "object" + }, "ByteRange": { "properties": { "end": { @@ -8518,6 +8576,26 @@ "title": "App/list/updatedNotification", "type": "object" }, + { + "properties": { + "method": { + "enum": [ + "browserSession/updated" + ], + "title": "BrowserSession/updatedNotificationMethod", + "type": "string" + }, + "params": { + "$ref": "#/definitions/BrowserSessionUpdatedNotification" + } + }, + "required": [ + "method", + "params" + ], + "title": "BrowserSession/updatedNotification", + "type": "object" + }, { "properties": { "method": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/BrowserSessionUpdatedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/BrowserSessionUpdatedNotification.json new file mode 100644 index 0000000000..7d166342eb --- /dev/null +++ b/codex-rs/app-server-protocol/schema/json/v2/BrowserSessionUpdatedNotification.json @@ -0,0 +1,60 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "BrowserSessionState": { + "properties": { + "selectedTabId": { + "type": "string" + }, + "tabs": { + "items": { + "$ref": "#/definitions/BrowserTabState" + }, + "type": "array" + } + }, + "required": [ + "selectedTabId", + "tabs" + ], + "type": "object" + }, + "BrowserTabState": { + "properties": { + "id": { + "type": "string" + }, + "selected": { + "type": "boolean" + }, + "title": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "id", + "selected", + "title", + "url" + ], + "type": "object" + } + }, + "properties": { + "browserSessionId": { + "type": "string" + }, + "browserState": { + "$ref": "#/definitions/BrowserSessionState" + } + }, + "required": [ + "browserSessionId", + "browserState" + ], + "title": "BrowserSessionUpdatedNotification", + "type": "object" +} \ No newline at end of file diff --git a/codex-rs/app-server-protocol/schema/typescript/ServerNotification.ts b/codex-rs/app-server-protocol/schema/typescript/ServerNotification.ts index d9e2df7797..4dbfb7b23d 100644 --- a/codex-rs/app-server-protocol/schema/typescript/ServerNotification.ts +++ b/codex-rs/app-server-protocol/schema/typescript/ServerNotification.ts @@ -8,6 +8,7 @@ import type { AccountRateLimitsUpdatedNotification } from "./v2/AccountRateLimit import type { AccountUpdatedNotification } from "./v2/AccountUpdatedNotification"; import type { AgentMessageDeltaNotification } from "./v2/AgentMessageDeltaNotification"; import type { AppListUpdatedNotification } from "./v2/AppListUpdatedNotification"; +import type { BrowserSessionUpdatedNotification } from "./v2/BrowserSessionUpdatedNotification"; import type { CommandExecOutputDeltaNotification } from "./v2/CommandExecOutputDeltaNotification"; import type { CommandExecutionOutputDeltaNotification } from "./v2/CommandExecutionOutputDeltaNotification"; import type { ConfigWarningNotification } from "./v2/ConfigWarningNotification"; @@ -56,4 +57,4 @@ import type { WindowsWorldWritableWarningNotification } from "./v2/WindowsWorldW /** * Notification sent from the server to the client. */ -export type ServerNotification = { "method": "error", "params": ErrorNotification } | { "method": "thread/started", "params": ThreadStartedNotification } | { "method": "thread/status/changed", "params": ThreadStatusChangedNotification } | { "method": "thread/archived", "params": ThreadArchivedNotification } | { "method": "thread/unarchived", "params": ThreadUnarchivedNotification } | { "method": "thread/closed", "params": ThreadClosedNotification } | { "method": "skills/changed", "params": SkillsChangedNotification } | { "method": "thread/name/updated", "params": ThreadNameUpdatedNotification } | { "method": "thread/tokenUsage/updated", "params": ThreadTokenUsageUpdatedNotification } | { "method": "turn/started", "params": TurnStartedNotification } | { "method": "hook/started", "params": HookStartedNotification } | { "method": "turn/completed", "params": TurnCompletedNotification } | { "method": "hook/completed", "params": HookCompletedNotification } | { "method": "turn/diff/updated", "params": TurnDiffUpdatedNotification } | { "method": "turn/plan/updated", "params": TurnPlanUpdatedNotification } | { "method": "item/started", "params": ItemStartedNotification } | { "method": "item/autoApprovalReview/started", "params": ItemGuardianApprovalReviewStartedNotification } | { "method": "item/autoApprovalReview/completed", "params": ItemGuardianApprovalReviewCompletedNotification } | { "method": "item/completed", "params": ItemCompletedNotification } | { "method": "rawResponseItem/completed", "params": RawResponseItemCompletedNotification } | { "method": "item/agentMessage/delta", "params": AgentMessageDeltaNotification } | { "method": "item/plan/delta", "params": PlanDeltaNotification } | { "method": "command/exec/outputDelta", "params": CommandExecOutputDeltaNotification } | { "method": "item/commandExecution/outputDelta", "params": CommandExecutionOutputDeltaNotification } | { "method": "item/commandExecution/terminalInteraction", "params": TerminalInteractionNotification } | { "method": "item/fileChange/outputDelta", "params": FileChangeOutputDeltaNotification } | { "method": "serverRequest/resolved", "params": ServerRequestResolvedNotification } | { "method": "item/mcpToolCall/progress", "params": McpToolCallProgressNotification } | { "method": "mcpServer/oauthLogin/completed", "params": McpServerOauthLoginCompletedNotification } | { "method": "mcpServer/startupStatus/updated", "params": McpServerStatusUpdatedNotification } | { "method": "account/updated", "params": AccountUpdatedNotification } | { "method": "account/rateLimits/updated", "params": AccountRateLimitsUpdatedNotification } | { "method": "app/list/updated", "params": AppListUpdatedNotification } | { "method": "item/reasoning/summaryTextDelta", "params": ReasoningSummaryTextDeltaNotification } | { "method": "item/reasoning/summaryPartAdded", "params": ReasoningSummaryPartAddedNotification } | { "method": "item/reasoning/textDelta", "params": ReasoningTextDeltaNotification } | { "method": "thread/compacted", "params": ContextCompactedNotification } | { "method": "model/rerouted", "params": ModelReroutedNotification } | { "method": "deprecationNotice", "params": DeprecationNoticeNotification } | { "method": "configWarning", "params": ConfigWarningNotification } | { "method": "fuzzyFileSearch/sessionUpdated", "params": FuzzyFileSearchSessionUpdatedNotification } | { "method": "fuzzyFileSearch/sessionCompleted", "params": FuzzyFileSearchSessionCompletedNotification } | { "method": "thread/realtime/started", "params": ThreadRealtimeStartedNotification } | { "method": "thread/realtime/itemAdded", "params": ThreadRealtimeItemAddedNotification } | { "method": "thread/realtime/transcriptUpdated", "params": ThreadRealtimeTranscriptUpdatedNotification } | { "method": "thread/realtime/outputAudio/delta", "params": ThreadRealtimeOutputAudioDeltaNotification } | { "method": "thread/realtime/error", "params": ThreadRealtimeErrorNotification } | { "method": "thread/realtime/closed", "params": ThreadRealtimeClosedNotification } | { "method": "windows/worldWritableWarning", "params": WindowsWorldWritableWarningNotification } | { "method": "windowsSandbox/setupCompleted", "params": WindowsSandboxSetupCompletedNotification } | { "method": "account/login/completed", "params": AccountLoginCompletedNotification }; +export type ServerNotification = { "method": "error", "params": ErrorNotification } | { "method": "thread/started", "params": ThreadStartedNotification } | { "method": "thread/status/changed", "params": ThreadStatusChangedNotification } | { "method": "thread/archived", "params": ThreadArchivedNotification } | { "method": "thread/unarchived", "params": ThreadUnarchivedNotification } | { "method": "thread/closed", "params": ThreadClosedNotification } | { "method": "skills/changed", "params": SkillsChangedNotification } | { "method": "thread/name/updated", "params": ThreadNameUpdatedNotification } | { "method": "thread/tokenUsage/updated", "params": ThreadTokenUsageUpdatedNotification } | { "method": "turn/started", "params": TurnStartedNotification } | { "method": "hook/started", "params": HookStartedNotification } | { "method": "turn/completed", "params": TurnCompletedNotification } | { "method": "hook/completed", "params": HookCompletedNotification } | { "method": "turn/diff/updated", "params": TurnDiffUpdatedNotification } | { "method": "turn/plan/updated", "params": TurnPlanUpdatedNotification } | { "method": "item/started", "params": ItemStartedNotification } | { "method": "item/autoApprovalReview/started", "params": ItemGuardianApprovalReviewStartedNotification } | { "method": "item/autoApprovalReview/completed", "params": ItemGuardianApprovalReviewCompletedNotification } | { "method": "item/completed", "params": ItemCompletedNotification } | { "method": "rawResponseItem/completed", "params": RawResponseItemCompletedNotification } | { "method": "item/agentMessage/delta", "params": AgentMessageDeltaNotification } | { "method": "item/plan/delta", "params": PlanDeltaNotification } | { "method": "command/exec/outputDelta", "params": CommandExecOutputDeltaNotification } | { "method": "item/commandExecution/outputDelta", "params": CommandExecutionOutputDeltaNotification } | { "method": "item/commandExecution/terminalInteraction", "params": TerminalInteractionNotification } | { "method": "item/fileChange/outputDelta", "params": FileChangeOutputDeltaNotification } | { "method": "serverRequest/resolved", "params": ServerRequestResolvedNotification } | { "method": "item/mcpToolCall/progress", "params": McpToolCallProgressNotification } | { "method": "mcpServer/oauthLogin/completed", "params": McpServerOauthLoginCompletedNotification } | { "method": "mcpServer/startupStatus/updated", "params": McpServerStatusUpdatedNotification } | { "method": "account/updated", "params": AccountUpdatedNotification } | { "method": "account/rateLimits/updated", "params": AccountRateLimitsUpdatedNotification } | { "method": "app/list/updated", "params": AppListUpdatedNotification } | { "method": "browserSession/updated", "params": BrowserSessionUpdatedNotification } | { "method": "item/reasoning/summaryTextDelta", "params": ReasoningSummaryTextDeltaNotification } | { "method": "item/reasoning/summaryPartAdded", "params": ReasoningSummaryPartAddedNotification } | { "method": "item/reasoning/textDelta", "params": ReasoningTextDeltaNotification } | { "method": "thread/compacted", "params": ContextCompactedNotification } | { "method": "model/rerouted", "params": ModelReroutedNotification } | { "method": "deprecationNotice", "params": DeprecationNoticeNotification } | { "method": "configWarning", "params": ConfigWarningNotification } | { "method": "fuzzyFileSearch/sessionUpdated", "params": FuzzyFileSearchSessionUpdatedNotification } | { "method": "fuzzyFileSearch/sessionCompleted", "params": FuzzyFileSearchSessionCompletedNotification } | { "method": "thread/realtime/started", "params": ThreadRealtimeStartedNotification } | { "method": "thread/realtime/itemAdded", "params": ThreadRealtimeItemAddedNotification } | { "method": "thread/realtime/transcriptUpdated", "params": ThreadRealtimeTranscriptUpdatedNotification } | { "method": "thread/realtime/outputAudio/delta", "params": ThreadRealtimeOutputAudioDeltaNotification } | { "method": "thread/realtime/error", "params": ThreadRealtimeErrorNotification } | { "method": "thread/realtime/closed", "params": ThreadRealtimeClosedNotification } | { "method": "windows/worldWritableWarning", "params": WindowsWorldWritableWarningNotification } | { "method": "windowsSandbox/setupCompleted", "params": WindowsSandboxSetupCompletedNotification } | { "method": "account/login/completed", "params": AccountLoginCompletedNotification }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/BrowserSessionState.ts b/codex-rs/app-server-protocol/schema/typescript/v2/BrowserSessionState.ts new file mode 100644 index 0000000000..4ac9455943 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/BrowserSessionState.ts @@ -0,0 +1,6 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { BrowserTabState } from "./BrowserTabState"; + +export type BrowserSessionState = { selectedTabId: string, tabs: Array, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/BrowserSessionUpdatedNotification.ts b/codex-rs/app-server-protocol/schema/typescript/v2/BrowserSessionUpdatedNotification.ts new file mode 100644 index 0000000000..89259f2c25 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/BrowserSessionUpdatedNotification.ts @@ -0,0 +1,6 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { BrowserSessionState } from "./BrowserSessionState"; + +export type BrowserSessionUpdatedNotification = { browserSessionId: string, browserState: BrowserSessionState, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/BrowserTabState.ts b/codex-rs/app-server-protocol/schema/typescript/v2/BrowserTabState.ts new file mode 100644 index 0000000000..3f06eac601 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/BrowserTabState.ts @@ -0,0 +1,5 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type BrowserTabState = { id: string, title: string, url: string, selected: boolean, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/index.ts b/codex-rs/app-server-protocol/schema/typescript/v2/index.ts index c649aec06a..a506c435ff 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/index.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/index.ts @@ -25,6 +25,9 @@ export type { AppsDefaultConfig } from "./AppsDefaultConfig"; export type { AppsListParams } from "./AppsListParams"; export type { AppsListResponse } from "./AppsListResponse"; export type { AskForApproval } from "./AskForApproval"; +export type { BrowserSessionState } from "./BrowserSessionState"; +export type { BrowserSessionUpdatedNotification } from "./BrowserSessionUpdatedNotification"; +export type { BrowserTabState } from "./BrowserTabState"; export type { ByteRange } from "./ByteRange"; export type { CancelLoginAccountParams } from "./CancelLoginAccountParams"; export type { CancelLoginAccountResponse } from "./CancelLoginAccountResponse"; diff --git a/codex-rs/app-server-protocol/src/protocol/common.rs b/codex-rs/app-server-protocol/src/protocol/common.rs index 56897566d9..4f0c966a70 100644 --- a/codex-rs/app-server-protocol/src/protocol/common.rs +++ b/codex-rs/app-server-protocol/src/protocol/common.rs @@ -308,6 +308,11 @@ client_request_definitions! { params: v2::AppsListParams, response: v2::AppsListResponse, }, + #[experimental("browserSession/command")] + BrowserSessionCommand => "browserSession/command" { + params: v2::BrowserSessionCommandParams, + response: v2::BrowserSessionCommandResponse, + }, FsReadFile => "fs/readFile" { params: v2::FsReadFileParams, response: v2::FsReadFileResponse, @@ -909,6 +914,8 @@ server_notification_definitions! { AccountUpdated => "account/updated" (v2::AccountUpdatedNotification), AccountRateLimitsUpdated => "account/rateLimits/updated" (v2::AccountRateLimitsUpdatedNotification), AppListUpdated => "app/list/updated" (v2::AppListUpdatedNotification), + #[experimental("browserSession/updated")] + BrowserSessionUpdated => "browserSession/updated" (v2::BrowserSessionUpdatedNotification), ReasoningSummaryTextDelta => "item/reasoning/summaryTextDelta" (v2::ReasoningSummaryTextDeltaNotification), ReasoningSummaryPartAdded => "item/reasoning/summaryPartAdded" (v2::ReasoningSummaryPartAddedNotification), ReasoningTextDelta => "item/reasoning/textDelta" (v2::ReasoningTextDeltaNotification), @@ -1609,6 +1616,75 @@ mod tests { Ok(()) } + #[test] + fn serialize_browser_session_command_request() -> Result<()> { + let request = ClientRequest::BrowserSessionCommand { + request_id: RequestId::Integer(7), + params: v2::BrowserSessionCommandParams { + browser_session_id: Some("sess_123".to_string()), + command: "navigateTabUrl".to_string(), + arguments: Some(json!({ + "tabId": "tab_123", + "url": "https://example.com", + })), + }, + }; + assert_eq!( + json!({ + "method": "browserSession/command", + "id": 7, + "params": { + "browserSessionId": "sess_123", + "command": "navigateTabUrl", + "arguments": { + "tabId": "tab_123", + "url": "https://example.com" + } + } + }), + serde_json::to_value(&request)?, + ); + Ok(()) + } + + #[test] + fn serialize_browser_session_updated_notification() -> Result<()> { + let notification = + ServerNotification::BrowserSessionUpdated(v2::BrowserSessionUpdatedNotification { + browser_session_id: "sess_123".to_string(), + browser_state: v2::BrowserSessionState { + selected_tab_id: "tab_123".to_string(), + tabs: vec![v2::BrowserTabState { + id: "tab_123".to_string(), + title: "Example".to_string(), + url: "https://example.com".to_string(), + selected: true, + }], + }, + }); + assert_eq!( + json!({ + "method": "browserSession/updated", + "params": { + "browserSessionId": "sess_123", + "browserState": { + "selectedTabId": "tab_123", + "tabs": [ + { + "id": "tab_123", + "title": "Example", + "url": "https://example.com", + "selected": true + } + ] + } + } + }), + serde_json::to_value(¬ification)?, + ); + Ok(()) + } + #[test] fn mock_experimental_method_is_marked_experimental() { let request = ClientRequest::MockExperimentalMethod { @@ -1618,6 +1694,35 @@ mod tests { let reason = crate::experimental_api::ExperimentalApi::experimental_reason(&request); assert_eq!(reason, Some("mock/experimentalMethod")); } + + #[test] + fn browser_session_command_is_marked_experimental() { + let request = ClientRequest::BrowserSessionCommand { + request_id: RequestId::Integer(1), + params: v2::BrowserSessionCommandParams { + browser_session_id: None, + command: "ping".to_string(), + arguments: None, + }, + }; + let reason = crate::experimental_api::ExperimentalApi::experimental_reason(&request); + assert_eq!(reason, Some("browserSession/command")); + } + + #[test] + fn browser_session_updated_is_marked_experimental() { + let notification = + ServerNotification::BrowserSessionUpdated(v2::BrowserSessionUpdatedNotification { + browser_session_id: "sess_123".to_string(), + browser_state: v2::BrowserSessionState { + selected_tab_id: String::new(), + tabs: Vec::new(), + }, + }); + let reason = crate::experimental_api::ExperimentalApi::experimental_reason(¬ification); + assert_eq!(reason, Some("browserSession/updated")); + } + #[test] fn thread_realtime_start_is_marked_experimental() { let request = ClientRequest::ThreadRealtimeStart { diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index 3e30f81323..22ee3a0dde 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -2099,6 +2099,52 @@ pub struct AppListUpdatedNotification { pub data: Vec, } +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct BrowserTabState { + pub id: String, + pub title: String, + pub url: String, + pub selected: bool, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct BrowserSessionState { + pub selected_tab_id: String, + pub tabs: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct BrowserSessionCommandParams { + #[ts(optional = nullable)] + pub browser_session_id: Option, + pub command: String, + #[ts(optional = nullable)] + pub arguments: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct BrowserSessionCommandResponse { + pub browser_session_id: String, + pub result: JsonValue, + pub browser_state: BrowserSessionState, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct BrowserSessionUpdatedNotification { + pub browser_session_id: String, + pub browser_state: BrowserSessionState, +} + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] diff --git a/codex-rs/app-server/Cargo.toml b/codex-rs/app-server/Cargo.toml index c6be85984d..c6b0085667 100644 --- a/codex-rs/app-server/Cargo.toml +++ b/codex-rs/app-server/Cargo.toml @@ -53,6 +53,7 @@ chrono = { workspace = true } clap = { workspace = true, features = ["derive"] } futures = { workspace = true } owo-colors = { workspace = true, features = ["supports-colors"] } +reqwest = { workspace = true, features = ["json", "rustls-tls"] } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } tempfile = { workspace = true } diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index 947a49ff2f..354b74434d 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -722,6 +722,9 @@ impl CodexMessageProcessor { self.apps_list(to_connection_request_id(request_id), params) .await; } + ClientRequest::BrowserSessionCommand { .. } => { + unreachable!("browserSession/command is handled by MessageProcessor"); + } ClientRequest::SkillsConfigWrite { request_id, params } => { self.skills_config_write(to_connection_request_id(request_id), params) .await; diff --git a/codex-rs/app-server/src/in_process.rs b/codex-rs/app-server/src/in_process.rs index 4288d15393..b653b684c9 100644 --- a/codex-rs/app-server/src/in_process.rs +++ b/codex-rs/app-server/src/in_process.rs @@ -417,6 +417,7 @@ fn start_uninitialized(args: InProcessStartArgs) -> InProcessClientHandle { config_warnings: args.config_warnings, session_source: args.session_source, enable_codex_api_key_env: args.enable_codex_api_key_env, + remote_browser_endpoint: None, }); let mut thread_created_rx = processor.thread_created_receiver(); let mut session = ConnectionSessionState::default(); diff --git a/codex-rs/app-server/src/lib.rs b/codex-rs/app-server/src/lib.rs index a4994d34b8..b558e37373 100644 --- a/codex-rs/app-server/src/lib.rs +++ b/codex-rs/app-server/src/lib.rs @@ -71,6 +71,7 @@ pub mod in_process; mod message_processor; mod models; mod outgoing_message; +mod remote_browser_api; mod server_request_error; mod thread_state; mod thread_status; @@ -329,6 +330,7 @@ pub async fn run_main( cli_config_overrides: CliConfigOverrides, loader_overrides: LoaderOverrides, default_analytics_enabled: bool, + remote_browser_endpoint: Option, ) -> IoResult<()> { run_main_with_transport( arg0_paths, @@ -337,6 +339,7 @@ pub async fn run_main( default_analytics_enabled, AppServerTransport::Stdio, SessionSource::VSCode, + remote_browser_endpoint, ) .await } @@ -348,6 +351,7 @@ pub async fn run_main_with_transport( default_analytics_enabled: bool, transport: AppServerTransport, session_source: SessionSource, + remote_browser_endpoint: Option, ) -> IoResult<()> { let (transport_event_tx, mut transport_event_rx) = mpsc::channel::(CHANNEL_CAPACITY); @@ -625,6 +629,7 @@ pub async fn run_main_with_transport( config_warnings, session_source, enable_codex_api_key_env: false, + remote_browser_endpoint, }); let mut thread_created_rx = processor.thread_created_receiver(); let mut running_turn_count_rx = processor.subscribe_running_assistant_turn_count(); diff --git a/codex-rs/app-server/src/main.rs b/codex-rs/app-server/src/main.rs index 60fa0a777b..0999e5d90d 100644 --- a/codex-rs/app-server/src/main.rs +++ b/codex-rs/app-server/src/main.rs @@ -31,6 +31,10 @@ struct AppServerArgs { value_parser = SessionSource::from_startup_arg )] session_source: SessionSource, + + /// Optional HTTP endpoint that receives remote browser commands. + #[arg(long = "remote-browser-endpoint", value_name = "URL")] + remote_browser_endpoint: Option, } fn main() -> anyhow::Result<()> { @@ -43,6 +47,7 @@ fn main() -> anyhow::Result<()> { }; let transport = args.listen; let session_source = args.session_source; + let remote_browser_endpoint = args.remote_browser_endpoint; run_main_with_transport( arg0_paths, @@ -51,6 +56,7 @@ fn main() -> anyhow::Result<()> { /*default_analytics_enabled*/ false, transport, session_source, + remote_browser_endpoint, ) .await?; Ok(()) diff --git a/codex-rs/app-server/src/message_processor.rs b/codex-rs/app-server/src/message_processor.rs index 287fe79755..61c407c398 100644 --- a/codex-rs/app-server/src/message_processor.rs +++ b/codex-rs/app-server/src/message_processor.rs @@ -15,8 +15,11 @@ use crate::outgoing_message::ConnectionId; use crate::outgoing_message::ConnectionRequestId; use crate::outgoing_message::OutgoingMessageSender; use crate::outgoing_message::RequestContext; +use crate::remote_browser_api::RemoteBrowserApi; use crate::transport::AppServerTransport; use async_trait::async_trait; +use codex_app_server_protocol::BrowserSessionCommandParams; +use codex_app_server_protocol::BrowserSessionUpdatedNotification; use codex_app_server_protocol::ChatgptAuthTokensRefreshParams; use codex_app_server_protocol::ChatgptAuthTokensRefreshReason; use codex_app_server_protocol::ChatgptAuthTokensRefreshResponse; @@ -151,6 +154,7 @@ pub(crate) struct MessageProcessor { config_api: ConfigApi, external_agent_config_api: ExternalAgentConfigApi, fs_api: FsApi, + remote_browser_api: RemoteBrowserApi, auth_manager: Arc, config: Arc, config_warnings: Arc>, @@ -179,6 +183,7 @@ pub(crate) struct MessageProcessorArgs { pub(crate) config_warnings: Vec, pub(crate) session_source: SessionSource, pub(crate) enable_codex_api_key_env: bool, + pub(crate) remote_browser_endpoint: Option, } impl MessageProcessor { @@ -199,6 +204,7 @@ impl MessageProcessor { config_warnings, session_source, enable_codex_api_key_env, + remote_browser_endpoint, } = args; let (auth_manager, thread_manager) = match (auth_manager, thread_manager) { (Some(auth_manager), Some(thread_manager)) => (auth_manager, thread_manager), @@ -259,6 +265,7 @@ impl MessageProcessor { ); let external_agent_config_api = ExternalAgentConfigApi::new(config.codex_home.clone()); let fs_api = FsApi::default(); + let remote_browser_api = RemoteBrowserApi::new(remote_browser_endpoint); Self { outgoing, @@ -266,6 +273,7 @@ impl MessageProcessor { config_api, external_agent_config_api, fs_api, + remote_browser_api, auth_manager, config, config_warnings: Arc::new(config_warnings), @@ -766,6 +774,16 @@ impl MessageProcessor { ) .await; } + ClientRequest::BrowserSessionCommand { request_id, params } => { + self.handle_browser_session_command( + ConnectionRequestId { + connection_id, + request_id, + }, + params, + ) + .await; + } other => { // Box the delegated future so this wrapper's async state machine does not // inline the full `CodexMessageProcessor::process_request` future, which @@ -917,6 +935,29 @@ impl MessageProcessor { Err(error) => self.outgoing.send_error(request_id, error).await, } } + + async fn handle_browser_session_command( + &self, + request_id: ConnectionRequestId, + params: BrowserSessionCommandParams, + ) { + match self.remote_browser_api.command(params).await { + Ok(response) => { + self.outgoing + .send_response(request_id, response.clone()) + .await; + self.outgoing + .send_server_notification(ServerNotification::BrowserSessionUpdated( + BrowserSessionUpdatedNotification { + browser_session_id: response.browser_session_id.clone(), + browser_state: response.browser_state.clone(), + }, + )) + .await; + } + Err(error) => self.outgoing.send_error(request_id, error).await, + } + } } #[cfg(test)] diff --git a/codex-rs/app-server/src/message_processor/tracing_tests.rs b/codex-rs/app-server/src/message_processor/tracing_tests.rs index 58499745bd..90a4abcf44 100644 --- a/codex-rs/app-server/src/message_processor/tracing_tests.rs +++ b/codex-rs/app-server/src/message_processor/tracing_tests.rs @@ -7,6 +7,11 @@ use crate::transport::AppServerTransport; use anyhow::Result; use app_test_support::create_mock_responses_server_repeating_assistant; use app_test_support::write_mock_responses_config_toml; +use codex_app_server_protocol::BrowserSessionCommandParams; +use codex_app_server_protocol::BrowserSessionCommandResponse; +use codex_app_server_protocol::BrowserSessionState; +use codex_app_server_protocol::BrowserSessionUpdatedNotification; +use codex_app_server_protocol::BrowserTabState; use codex_app_server_protocol::ClientInfo; use codex_app_server_protocol::ClientRequest; use codex_app_server_protocol::InitializeCapabilities; @@ -113,11 +118,11 @@ struct TracingHarness { } impl TracingHarness { - async fn new() -> Result { + async fn new(remote_browser_endpoint: Option) -> Result { let server = create_mock_responses_server_repeating_assistant("Done").await; let codex_home = TempDir::new()?; let config = Arc::new(build_test_config(codex_home.path(), &server.uri()).await?); - let (processor, outgoing_rx) = build_test_processor(config); + let (processor, outgoing_rx) = build_test_processor(config, remote_browser_endpoint); let tracing = init_test_tracing(); tracing.exporter.reset(); tracing::callsite::rebuild_interest_cache(); @@ -226,6 +231,7 @@ async fn build_test_config(codex_home: &Path, server_uri: &str) -> Result, + remote_browser_endpoint: Option, ) -> ( MessageProcessor, mpsc::Receiver, @@ -246,6 +252,7 @@ fn build_test_processor( config_warnings: Vec::new(), session_source: SessionSource::VSCode, enable_codex_api_key_env: false, + remote_browser_endpoint, }); (processor, outgoing_rx) } @@ -455,6 +462,30 @@ async fn read_thread_started_notification( } } +async fn read_browser_session_updated_notification( + outgoing_rx: &mut mpsc::Receiver, +) -> BrowserSessionUpdatedNotification { + loop { + let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), outgoing_rx.recv()) + .await + .expect("timed out waiting for browserSession/updated notification") + .expect("outgoing channel closed"); + let message = match envelope { + crate::outgoing_message::OutgoingEnvelope::ToConnection { message, .. } + | crate::outgoing_message::OutgoingEnvelope::Broadcast { message } => message, + }; + let crate::outgoing_message::OutgoingMessage::AppServerNotification(notification) = message + else { + continue; + }; + if let codex_app_server_protocol::ServerNotification::BrowserSessionUpdated(notification) = + notification + { + return notification; + } + } +} + async fn wait_for_exported_spans(tracing: &TestTracing, predicate: F) -> Vec where F: Fn(&[SpanData]) -> bool, @@ -498,7 +529,7 @@ where #[tokio::test(flavor = "current_thread")] async fn thread_start_jsonrpc_span_exports_server_span_and_parents_children() -> Result<()> { let _guard = tracing_test_guard().lock().await; - let mut harness = TracingHarness::new().await?; + let mut harness = TracingHarness::new(None).await?; let RemoteTrace { trace_id: remote_trace_id, @@ -569,7 +600,7 @@ async fn thread_start_jsonrpc_span_exports_server_span_and_parents_children() -> #[tokio::test(flavor = "current_thread")] async fn turn_start_jsonrpc_span_parents_core_turn_spans() -> Result<()> { let _guard = tracing_test_guard().lock().await; - let mut harness = TracingHarness::new().await?; + let mut harness = TracingHarness::new(None).await?; let thread_start_response = harness.start_thread(2, None).await; let thread_id = thread_start_response.thread.id.clone(); @@ -637,3 +668,77 @@ async fn turn_start_jsonrpc_span_parents_core_turn_spans() -> Result<()> { Ok(()) } + +#[tokio::test(flavor = "current_thread")] +async fn browser_session_command_posts_and_emits_state_notification() -> Result<()> { + let _guard = tracing_test_guard().lock().await; + let remote_browser = wiremock::MockServer::start().await; + let browser_state = BrowserSessionState { + selected_tab_id: "tab_123".to_string(), + tabs: vec![BrowserTabState { + id: "tab_123".to_string(), + title: "Example".to_string(), + url: "https://example.com".to_string(), + selected: true, + }], + }; + let response_body = serde_json::json!({ + "browserSessionId": "sess_123", + "result": { + "ok": true, + }, + "browserState": { + "selectedTabId": "tab_123", + "tabs": [{ + "id": "tab_123", + "title": "Example", + "url": "https://example.com", + "selected": true, + }] + } + }); + + wiremock::Mock::given(wiremock::matchers::method("POST")) + .and(wiremock::matchers::path("/")) + .and(wiremock::matchers::body_json(serde_json::json!({ + "browserSessionId": "sess_123", + "command": "navigateTabUrl", + "arguments": { + "tabId": "tab_123", + "url": "https://example.com" + } + }))) + .respond_with(wiremock::ResponseTemplate::new(200).set_body_json(response_body)) + .mount(&remote_browser) + .await; + + let mut harness = TracingHarness::new(Some(remote_browser.uri())).await?; + let response: BrowserSessionCommandResponse = harness + .request( + ClientRequest::BrowserSessionCommand { + request_id: RequestId::Integer(4), + params: BrowserSessionCommandParams { + browser_session_id: Some("sess_123".to_string()), + command: "navigateTabUrl".to_string(), + arguments: Some(serde_json::json!({ + "tabId": "tab_123", + "url": "https://example.com", + })), + }, + }, + None, + ) + .await; + assert_eq!(response.browser_session_id, "sess_123"); + assert_eq!(response.result, serde_json::json!({"ok": true})); + assert_eq!(response.browser_state, browser_state); + + let notification = read_browser_session_updated_notification(&mut harness.outgoing_rx).await; + assert_eq!(notification.browser_session_id, "sess_123"); + assert_eq!(notification.browser_state, browser_state); + harness.shutdown().await; + + remote_browser.verify().await; + + Ok(()) +} diff --git a/codex-rs/app-server/src/remote_browser_api.rs b/codex-rs/app-server/src/remote_browser_api.rs new file mode 100644 index 0000000000..4ad7cd3757 --- /dev/null +++ b/codex-rs/app-server/src/remote_browser_api.rs @@ -0,0 +1,65 @@ +use crate::error_code::INTERNAL_ERROR_CODE; +use crate::error_code::INVALID_REQUEST_ERROR_CODE; +use codex_app_server_protocol::BrowserSessionCommandParams; +use codex_app_server_protocol::BrowserSessionCommandResponse; +use codex_app_server_protocol::JSONRPCErrorError; + +#[derive(Clone)] +pub(crate) struct RemoteBrowserApi { + endpoint: Option, + client: reqwest::Client, +} + +impl RemoteBrowserApi { + pub(crate) fn new(endpoint: Option) -> Self { + Self { + endpoint, + client: reqwest::Client::new(), + } + } + + pub(crate) async fn command( + &self, + params: BrowserSessionCommandParams, + ) -> Result { + let Some(endpoint) = &self.endpoint else { + return Err(JSONRPCErrorError { + code: INVALID_REQUEST_ERROR_CODE, + message: "Remote browser endpoint is not configured".to_string(), + data: None, + }); + }; + + let response = self + .client + .post(endpoint) + .json(¶ms) + .send() + .await + .map_err(map_reqwest_error)?; + + let status = response.status(); + let body = response.text().await.map_err(map_reqwest_error)?; + if !status.is_success() { + return Err(JSONRPCErrorError { + code: INTERNAL_ERROR_CODE, + message: format!("remote browser endpoint returned HTTP {status}: {body}"), + data: None, + }); + } + + serde_json::from_str(&body).map_err(|err| JSONRPCErrorError { + code: INTERNAL_ERROR_CODE, + message: format!("failed to decode remote browser response: {err}; body={body}"), + data: None, + }) + } +} + +fn map_reqwest_error(err: reqwest::Error) -> JSONRPCErrorError { + JSONRPCErrorError { + code: INTERNAL_ERROR_CODE, + message: format!("remote browser request failed: {err}"), + data: None, + } +}