From 917c2df201ff1b22041217856588074c08c96ca6 Mon Sep 17 00:00:00 2001 From: sayan-oai Date: Wed, 11 Mar 2026 20:33:17 -0700 Subject: [PATCH 1/6] chore: use AVAILABLE and ON_INSTALL as default plugin install and auth policies (#14407) make `AVAILABLE` the default plugin installPolicy when unset in `marketplace.json`. similarly, make `ON_INSTALL` the default authPolicy. this means, when unset, plugins are available to be installed (but not auto-installed), and the contained connectors will be authed at install-time. updated tests. --- .../codex_app_server_protocol.schemas.json | 32 +++------- .../codex_app_server_protocol.v2.schemas.json | 32 +++------- .../schema/json/v2/PluginInstallResponse.json | 12 +--- .../schema/json/v2/PluginListResponse.json | 20 ++---- .../typescript/v2/PluginInstallResponse.ts | 2 +- .../schema/typescript/v2/PluginSummary.ts | 2 +- .../app-server-protocol/src/protocol/v2.rs | 6 +- codex-rs/app-server/README.md | 4 +- .../app-server/src/codex_message_processor.rs | 6 +- .../tests/suite/v2/plugin_install.rs | 6 +- .../app-server/tests/suite/v2/plugin_list.rs | 28 ++++++++- codex-rs/core/src/plugins/manager.rs | 32 +++++----- codex-rs/core/src/plugins/marketplace.rs | 62 +++++++++++-------- 13 files changed, 111 insertions(+), 133 deletions(-) diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index 3e88ff54d1..c159cf7be0 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -12805,18 +12805,12 @@ "type": "array" }, "authPolicy": { - "anyOf": [ - { - "$ref": "#/definitions/v2/PluginAuthPolicy" - }, - { - "type": "null" - } - ] + "$ref": "#/definitions/v2/PluginAuthPolicy" } }, "required": [ - "appsNeedingAuth" + "appsNeedingAuth", + "authPolicy" ], "title": "PluginInstallResponse", "type": "object" @@ -13014,14 +13008,7 @@ "PluginSummary": { "properties": { "authPolicy": { - "anyOf": [ - { - "$ref": "#/definitions/v2/PluginAuthPolicy" - }, - { - "type": "null" - } - ] + "$ref": "#/definitions/v2/PluginAuthPolicy" }, "enabled": { "type": "boolean" @@ -13030,14 +13017,7 @@ "type": "string" }, "installPolicy": { - "anyOf": [ - { - "$ref": "#/definitions/v2/PluginInstallPolicy" - }, - { - "type": "null" - } - ] + "$ref": "#/definitions/v2/PluginInstallPolicy" }, "installed": { "type": "boolean" @@ -13060,8 +13040,10 @@ } }, "required": [ + "authPolicy", "enabled", "id", + "installPolicy", "installed", "name", "source" diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index 704e6d12f7..9afc7201cc 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -9153,18 +9153,12 @@ "type": "array" }, "authPolicy": { - "anyOf": [ - { - "$ref": "#/definitions/PluginAuthPolicy" - }, - { - "type": "null" - } - ] + "$ref": "#/definitions/PluginAuthPolicy" } }, "required": [ - "appsNeedingAuth" + "appsNeedingAuth", + "authPolicy" ], "title": "PluginInstallResponse", "type": "object" @@ -9362,14 +9356,7 @@ "PluginSummary": { "properties": { "authPolicy": { - "anyOf": [ - { - "$ref": "#/definitions/PluginAuthPolicy" - }, - { - "type": "null" - } - ] + "$ref": "#/definitions/PluginAuthPolicy" }, "enabled": { "type": "boolean" @@ -9378,14 +9365,7 @@ "type": "string" }, "installPolicy": { - "anyOf": [ - { - "$ref": "#/definitions/PluginInstallPolicy" - }, - { - "type": "null" - } - ] + "$ref": "#/definitions/PluginInstallPolicy" }, "installed": { "type": "boolean" @@ -9408,8 +9388,10 @@ } }, "required": [ + "authPolicy", "enabled", "id", + "installPolicy", "installed", "name", "source" diff --git a/codex-rs/app-server-protocol/schema/json/v2/PluginInstallResponse.json b/codex-rs/app-server-protocol/schema/json/v2/PluginInstallResponse.json index daa8326443..a95f47cd3e 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/PluginInstallResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/PluginInstallResponse.json @@ -45,18 +45,12 @@ "type": "array" }, "authPolicy": { - "anyOf": [ - { - "$ref": "#/definitions/PluginAuthPolicy" - }, - { - "type": "null" - } - ] + "$ref": "#/definitions/PluginAuthPolicy" } }, "required": [ - "appsNeedingAuth" + "appsNeedingAuth", + "authPolicy" ], "title": "PluginInstallResponse", "type": "object" diff --git a/codex-rs/app-server-protocol/schema/json/v2/PluginListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/PluginListResponse.json index 39a0b659c6..cbd7e04c2e 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/PluginListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/PluginListResponse.json @@ -170,14 +170,7 @@ "PluginSummary": { "properties": { "authPolicy": { - "anyOf": [ - { - "$ref": "#/definitions/PluginAuthPolicy" - }, - { - "type": "null" - } - ] + "$ref": "#/definitions/PluginAuthPolicy" }, "enabled": { "type": "boolean" @@ -186,14 +179,7 @@ "type": "string" }, "installPolicy": { - "anyOf": [ - { - "$ref": "#/definitions/PluginInstallPolicy" - }, - { - "type": "null" - } - ] + "$ref": "#/definitions/PluginInstallPolicy" }, "installed": { "type": "boolean" @@ -216,8 +202,10 @@ } }, "required": [ + "authPolicy", "enabled", "id", + "installPolicy", "installed", "name", "source" diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/PluginInstallResponse.ts b/codex-rs/app-server-protocol/schema/typescript/v2/PluginInstallResponse.ts index d4ea0afbb4..b88119d44c 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/PluginInstallResponse.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/PluginInstallResponse.ts @@ -4,4 +4,4 @@ import type { AppSummary } from "./AppSummary"; import type { PluginAuthPolicy } from "./PluginAuthPolicy"; -export type PluginInstallResponse = { authPolicy: PluginAuthPolicy | null, appsNeedingAuth: Array, }; +export type PluginInstallResponse = { authPolicy: PluginAuthPolicy, appsNeedingAuth: Array, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/PluginSummary.ts b/codex-rs/app-server-protocol/schema/typescript/v2/PluginSummary.ts index 358914cae7..1eb443c592 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/PluginSummary.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/PluginSummary.ts @@ -6,4 +6,4 @@ import type { PluginInstallPolicy } from "./PluginInstallPolicy"; import type { PluginInterface } from "./PluginInterface"; import type { PluginSource } from "./PluginSource"; -export type PluginSummary = { id: string, name: string, source: PluginSource, installed: boolean, enabled: boolean, installPolicy: PluginInstallPolicy | null, authPolicy: PluginAuthPolicy | null, interface: PluginInterface | null, }; +export type PluginSummary = { id: string, name: string, source: PluginSource, installed: boolean, enabled: boolean, installPolicy: PluginInstallPolicy, authPolicy: PluginAuthPolicy, interface: PluginInterface | null, }; diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index e2603de9a4..e5daf85684 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -3087,8 +3087,8 @@ pub struct PluginSummary { pub source: PluginSource, pub installed: bool, pub enabled: bool, - pub install_policy: Option, - pub auth_policy: Option, + pub install_policy: PluginInstallPolicy, + pub auth_policy: PluginAuthPolicy, pub interface: Option, } @@ -3149,7 +3149,7 @@ pub struct PluginInstallParams { #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct PluginInstallResponse { - pub auth_policy: Option, + pub auth_policy: PluginAuthPolicy, pub apps_needing_auth: Vec, } diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index 8d545ddda7..338a7a7142 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -157,13 +157,13 @@ Example with notification opt-out: - `experimentalFeature/list` — list feature flags with stage metadata (`beta`, `underDevelopment`, `stable`, etc.), enabled/default-enabled state, and cursor pagination. For non-beta flags, `displayName`/`description`/`announcement` are `null`. - `collaborationMode/list` — list available collaboration mode presets (experimental, no pagination). This response omits built-in developer instructions; clients should either pass `settings.developer_instructions: null` when setting a mode to use Codex's built-in instructions, or provide their own instructions explicitly. - `skills/list` — list skills for one or more `cwd` values (optional `forceReload`). -- `plugin/list` — list discovered plugin marketplaces and plugin state, including marketplace install/auth policy metadata. `interface.category` uses the marketplace category when present; otherwise it falls back to the plugin manifest category. Pass `forceRemoteSync: true` to refresh curated plugin state before listing (**under development; do not call from production clients yet**). +- `plugin/list` — list discovered plugin marketplaces and plugin state, including effective marketplace install/auth policy metadata. `interface.category` uses the marketplace category when present; otherwise it falls back to the plugin manifest category. Pass `forceRemoteSync: true` to refresh curated plugin state before listing (**under development; do not call from production clients yet**). - `skills/changed` — notification emitted when watched local skill files change. - `skills/remote/list` — list public remote skills (**under development; do not call from production clients yet**). - `skills/remote/export` — download a remote skill by `hazelnutId` into `skills` under `codex_home` (**under development; do not call from production clients yet**). - `app/list` — list available apps. - `skills/config/write` — write user-level skill config by path. -- `plugin/install` — install a plugin from a discovered marketplace entry, rejecting marketplace entries marked unavailable for install, and return the plugin auth policy plus any apps that still need auth (**under development; do not call from production clients yet**). +- `plugin/install` — install a plugin from a discovered marketplace entry, rejecting marketplace entries marked unavailable for install, and return the effective plugin auth policy plus any apps that still need auth (**under development; do not call from production clients yet**). - `plugin/uninstall` — uninstall a plugin by id by removing its cached files and clearing its user-level config entry (**under development; do not call from production clients yet**). - `mcpServer/oauth/login` — start an OAuth login for a configured MCP server; returns an `authorization_url` and later emits `mcpServer/oauthLogin/completed` once the browser flow finishes. - `tool/requestUserInput` — prompt the user with 1–3 short questions for a tool call and return their answers (experimental). diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index fdd05b4502..6a22d899b3 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -5465,8 +5465,8 @@ impl CodexMessageProcessor { PluginSource::Local { path } } }, - install_policy: plugin.install_policy.map(Into::into), - auth_policy: plugin.auth_policy.map(Into::into), + install_policy: plugin.install_policy.into(), + auth_policy: plugin.auth_policy.into(), interface: plugin.interface.map(|interface| PluginInterface { display_name: interface.display_name, short_description: interface.short_description, @@ -5719,7 +5719,7 @@ impl CodexMessageProcessor { .send_response( request_id, PluginInstallResponse { - auth_policy: result.auth_policy.map(Into::into), + auth_policy: result.auth_policy.into(), apps_needing_auth, }, ) diff --git a/codex-rs/app-server/tests/suite/v2/plugin_install.rs b/codex-rs/app-server/tests/suite/v2/plugin_install.rs index 2a76f6addb..6b9ed72c0d 100644 --- a/codex-rs/app-server/tests/suite/v2/plugin_install.rs +++ b/codex-rs/app-server/tests/suite/v2/plugin_install.rs @@ -191,7 +191,7 @@ async fn plugin_install_returns_apps_needing_auth() -> Result<()> { "sample-plugin", "./sample-plugin", None, - Some("ON_INSTALL"), + None, )?; write_plugin_source(repo_root.path(), "sample-plugin", &["alpha", "beta"])?; let marketplace_path = @@ -217,7 +217,7 @@ async fn plugin_install_returns_apps_needing_auth() -> Result<()> { assert_eq!( response, PluginInstallResponse { - auth_policy: Some(PluginAuthPolicy::OnInstall), + auth_policy: PluginAuthPolicy::OnInstall, apps_needing_auth: vec![AppSummary { id: "alpha".to_string(), name: "Alpha".to_string(), @@ -299,7 +299,7 @@ async fn plugin_install_filters_disallowed_apps_needing_auth() -> Result<()> { assert_eq!( response, PluginInstallResponse { - auth_policy: Some(PluginAuthPolicy::OnUse), + auth_policy: PluginAuthPolicy::OnUse, apps_needing_auth: vec![AppSummary { id: "alpha".to_string(), name: "Alpha".to_string(), diff --git a/codex-rs/app-server/tests/suite/v2/plugin_list.rs b/codex-rs/app-server/tests/suite/v2/plugin_list.rs index dfbd88e08f..eefed46958 100644 --- a/codex-rs/app-server/tests/suite/v2/plugin_list.rs +++ b/codex-rs/app-server/tests/suite/v2/plugin_list.rs @@ -224,10 +224,26 @@ enabled = false assert_eq!(marketplace.plugins[0].name, "enabled-plugin"); assert_eq!(marketplace.plugins[0].installed, true); assert_eq!(marketplace.plugins[0].enabled, true); + assert_eq!( + marketplace.plugins[0].install_policy, + PluginInstallPolicy::Available + ); + assert_eq!( + marketplace.plugins[0].auth_policy, + PluginAuthPolicy::OnInstall + ); assert_eq!(marketplace.plugins[1].id, "disabled-plugin@codex-curated"); assert_eq!(marketplace.plugins[1].name, "disabled-plugin"); assert_eq!(marketplace.plugins[1].installed, true); assert_eq!(marketplace.plugins[1].enabled, false); + assert_eq!( + marketplace.plugins[1].install_policy, + PluginInstallPolicy::Available + ); + assert_eq!( + marketplace.plugins[1].auth_policy, + PluginAuthPolicy::OnInstall + ); assert_eq!( marketplace.plugins[2].id, "uninstalled-plugin@codex-curated" @@ -235,6 +251,14 @@ enabled = false assert_eq!(marketplace.plugins[2].name, "uninstalled-plugin"); assert_eq!(marketplace.plugins[2].installed, false); assert_eq!(marketplace.plugins[2].enabled, false); + assert_eq!( + marketplace.plugins[2].install_policy, + PluginInstallPolicy::Available + ); + assert_eq!( + marketplace.plugins[2].auth_policy, + PluginAuthPolicy::OnInstall + ); Ok(()) } @@ -418,8 +442,8 @@ async fn plugin_list_returns_plugin_interface_with_absolute_asset_paths() -> Res assert_eq!(plugin.id, "demo-plugin@codex-curated"); assert_eq!(plugin.installed, false); assert_eq!(plugin.enabled, false); - assert_eq!(plugin.install_policy, Some(PluginInstallPolicy::Available)); - assert_eq!(plugin.auth_policy, Some(PluginAuthPolicy::OnInstall)); + assert_eq!(plugin.install_policy, PluginInstallPolicy::Available); + assert_eq!(plugin.auth_policy, PluginAuthPolicy::OnInstall); let interface = plugin .interface .as_ref() diff --git a/codex-rs/core/src/plugins/manager.rs b/codex-rs/core/src/plugins/manager.rs index 770512b2f8..91d4045ae4 100644 --- a/codex-rs/core/src/plugins/manager.rs +++ b/codex-rs/core/src/plugins/manager.rs @@ -75,7 +75,7 @@ pub struct PluginInstallOutcome { pub plugin_id: PluginId, pub plugin_version: String, pub installed_path: AbsolutePathBuf, - pub auth_policy: Option, + pub auth_policy: MarketplacePluginAuthPolicy, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -90,8 +90,8 @@ pub struct ConfiguredMarketplacePluginSummary { pub id: String, pub name: String, pub source: MarketplacePluginSourceSummary, - pub install_policy: Option, - pub auth_policy: Option, + pub install_policy: MarketplacePluginInstallPolicy, + pub auth_policy: MarketplacePluginAuthPolicy, pub interface: Option, pub installed: bool, pub enabled: bool, @@ -1972,7 +1972,7 @@ mod tests { plugin_id: PluginId::new("sample-plugin".to_string(), "debug".to_string()).unwrap(), plugin_version: "local".to_string(), installed_path: AbsolutePathBuf::try_from(installed_path).unwrap(), - auth_policy: Some(MarketplacePluginAuthPolicy::OnUse), + auth_policy: MarketplacePluginAuthPolicy::OnUse, } ); @@ -2102,8 +2102,8 @@ enabled = false path: AbsolutePathBuf::try_from(tmp.path().join("repo/enabled-plugin")) .unwrap(), }, - install_policy: None, - auth_policy: None, + install_policy: MarketplacePluginInstallPolicy::Available, + auth_policy: MarketplacePluginAuthPolicy::OnInstall, interface: None, installed: true, enabled: true, @@ -2117,8 +2117,8 @@ enabled = false ) .unwrap(), }, - install_policy: None, - auth_policy: None, + install_policy: MarketplacePluginInstallPolicy::Available, + auth_policy: MarketplacePluginAuthPolicy::OnInstall, interface: None, installed: true, enabled: false, @@ -2184,8 +2184,8 @@ enabled = false path: AbsolutePathBuf::try_from(curated_root.join("plugins/linear")) .unwrap(), }, - install_policy: None, - auth_policy: None, + install_policy: MarketplacePluginInstallPolicy::Available, + auth_policy: MarketplacePluginAuthPolicy::OnInstall, interface: None, installed: false, enabled: false, @@ -2284,8 +2284,8 @@ enabled = false source: MarketplacePluginSourceSummary::Local { path: AbsolutePathBuf::try_from(tmp.path().join("repo-a/from-a")).unwrap(), }, - install_policy: None, - auth_policy: None, + install_policy: MarketplacePluginInstallPolicy::Available, + auth_policy: MarketplacePluginAuthPolicy::OnInstall, interface: None, installed: false, enabled: true, @@ -2310,8 +2310,8 @@ enabled = false source: MarketplacePluginSourceSummary::Local { path: AbsolutePathBuf::try_from(tmp.path().join("repo-b/from-b-only")).unwrap(), }, - install_policy: None, - auth_policy: None, + install_policy: MarketplacePluginInstallPolicy::Available, + auth_policy: MarketplacePluginAuthPolicy::OnInstall, interface: None, installed: false, enabled: false, @@ -2389,8 +2389,8 @@ enabled = true path: AbsolutePathBuf::try_from(tmp.path().join("repo/sample-plugin")) .unwrap(), }, - install_policy: None, - auth_policy: None, + install_policy: MarketplacePluginInstallPolicy::Available, + auth_policy: MarketplacePluginAuthPolicy::OnInstall, interface: None, installed: false, enabled: true, diff --git a/codex-rs/core/src/plugins/marketplace.rs b/codex-rs/core/src/plugins/marketplace.rs index fb0abd205f..754d70f89b 100644 --- a/codex-rs/core/src/plugins/marketplace.rs +++ b/codex-rs/core/src/plugins/marketplace.rs @@ -21,7 +21,7 @@ const MARKETPLACE_RELATIVE_PATH: &str = ".agents/plugins/marketplace.json"; pub struct ResolvedMarketplacePlugin { pub plugin_id: PluginId, pub source_path: AbsolutePathBuf, - pub auth_policy: Option, + pub auth_policy: MarketplacePluginAuthPolicy, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -35,8 +35,8 @@ pub struct MarketplaceSummary { pub struct MarketplacePluginSummary { pub name: String, pub source: MarketplacePluginSourceSummary, - pub install_policy: Option, - pub auth_policy: Option, + pub install_policy: MarketplacePluginInstallPolicy, + pub auth_policy: MarketplacePluginAuthPolicy, pub interface: Option, } @@ -45,18 +45,20 @@ pub enum MarketplacePluginSourceSummary { Local { path: AbsolutePathBuf }, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize)] pub enum MarketplacePluginInstallPolicy { #[serde(rename = "NOT_AVAILABLE")] NotAvailable, + #[default] #[serde(rename = "AVAILABLE")] Available, #[serde(rename = "INSTALLED_BY_DEFAULT")] InstalledByDefault, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize)] pub enum MarketplacePluginAuthPolicy { + #[default] #[serde(rename = "ON_INSTALL")] OnInstall, #[serde(rename = "ON_USE")] @@ -148,7 +150,7 @@ pub fn resolve_marketplace_plugin( auth_policy, .. } = plugin; - if install_policy == Some(MarketplacePluginInstallPolicy::NotAvailable) { + if install_policy == MarketplacePluginInstallPolicy::NotAvailable { return Err(MarketplaceError::PluginNotAvailable { plugin_name: name, marketplace_name, @@ -365,9 +367,9 @@ struct MarketplacePlugin { name: String, source: MarketplacePluginSource, #[serde(default)] - install_policy: Option, + install_policy: MarketplacePluginInstallPolicy, #[serde(default)] - auth_policy: Option, + auth_policy: MarketplacePluginAuthPolicy, #[serde(default)] category: Option, } @@ -420,7 +422,7 @@ mod tests { plugin_id: PluginId::new("local-plugin".to_string(), "codex-curated".to_string()) .unwrap(), source_path: AbsolutePathBuf::try_from(repo_root.join("plugin-1")).unwrap(), - auth_policy: None, + auth_policy: MarketplacePluginAuthPolicy::OnInstall, } ); } @@ -527,8 +529,8 @@ mod tests { path: AbsolutePathBuf::try_from(home_root.join("home-shared")) .unwrap(), }, - install_policy: None, - auth_policy: None, + install_policy: MarketplacePluginInstallPolicy::Available, + auth_policy: MarketplacePluginAuthPolicy::OnInstall, interface: None, }, MarketplacePluginSummary { @@ -537,8 +539,8 @@ mod tests { path: AbsolutePathBuf::try_from(home_root.join("home-only")) .unwrap(), }, - install_policy: None, - auth_policy: None, + install_policy: MarketplacePluginInstallPolicy::Available, + auth_policy: MarketplacePluginAuthPolicy::OnInstall, interface: None, }, ], @@ -556,8 +558,8 @@ mod tests { path: AbsolutePathBuf::try_from(repo_root.join("repo-shared")) .unwrap(), }, - install_policy: None, - auth_policy: None, + install_policy: MarketplacePluginInstallPolicy::Available, + auth_policy: MarketplacePluginAuthPolicy::OnInstall, interface: None, }, MarketplacePluginSummary { @@ -566,8 +568,8 @@ mod tests { path: AbsolutePathBuf::try_from(repo_root.join("repo-only")) .unwrap(), }, - install_policy: None, - auth_policy: None, + install_policy: MarketplacePluginInstallPolicy::Available, + auth_policy: MarketplacePluginAuthPolicy::OnInstall, interface: None, }, ], @@ -638,8 +640,8 @@ mod tests { source: MarketplacePluginSourceSummary::Local { path: AbsolutePathBuf::try_from(home_root.join("home-plugin")).unwrap(), }, - install_policy: None, - auth_policy: None, + install_policy: MarketplacePluginInstallPolicy::Available, + auth_policy: MarketplacePluginAuthPolicy::OnInstall, interface: None, }], }, @@ -651,8 +653,8 @@ mod tests { source: MarketplacePluginSourceSummary::Local { path: AbsolutePathBuf::try_from(repo_root.join("repo-plugin")).unwrap(), }, - install_policy: None, - auth_policy: None, + install_policy: MarketplacePluginInstallPolicy::Available, + auth_policy: MarketplacePluginAuthPolicy::OnInstall, interface: None, }], }, @@ -717,8 +719,8 @@ mod tests { source: MarketplacePluginSourceSummary::Local { path: AbsolutePathBuf::try_from(repo_root.join("plugin")).unwrap(), }, - install_policy: None, - auth_policy: None, + install_policy: MarketplacePluginInstallPolicy::Available, + auth_policy: MarketplacePluginAuthPolicy::OnInstall, interface: None, }], }] @@ -774,11 +776,11 @@ mod tests { assert_eq!( marketplaces[0].plugins[0].install_policy, - Some(MarketplacePluginInstallPolicy::Available) + MarketplacePluginInstallPolicy::Available ); assert_eq!( marketplaces[0].plugins[0].auth_policy, - Some(MarketplacePluginAuthPolicy::OnInstall) + MarketplacePluginAuthPolicy::OnInstall ); assert_eq!( marketplaces[0].plugins[0].interface, @@ -868,8 +870,14 @@ mod tests { screenshots: Vec::new(), }) ); - assert_eq!(marketplaces[0].plugins[0].install_policy, None); - assert_eq!(marketplaces[0].plugins[0].auth_policy, None); + assert_eq!( + marketplaces[0].plugins[0].install_policy, + MarketplacePluginInstallPolicy::Available + ); + assert_eq!( + marketplaces[0].plugins[0].auth_policy, + MarketplacePluginAuthPolicy::OnInstall + ); } #[test] From ba5b94287e21bbe3da565d2f070a14f8d4971328 Mon Sep 17 00:00:00 2001 From: Matthew Zeng Date: Wed, 11 Mar 2026 22:06:59 -0700 Subject: [PATCH 2/6] [apps] Add tool_suggest tool. (#14287) - [x] Add tool_suggest tool. - [x] Move chatgpt/src/connectors.rs and core/src/connectors.rs into a dedicated mod so that we have all the logic and global cache in one place. - [x] Update TUI app link view to support rendering the installation view for mcp elicitation. --------- Co-authored-by: Shaqayeq Co-authored-by: Eric Traut Co-authored-by: pakrym-oai Co-authored-by: Ahmed Ibrahim Co-authored-by: guinness-oai Co-authored-by: Eugene Brevdo Co-authored-by: Charlie Guo Co-authored-by: Fouad Matin Co-authored-by: Fouad Matin <169186268+fouad-openai@users.noreply.github.com> Co-authored-by: xl-openai Co-authored-by: alexsong-oai Co-authored-by: Owen Lin Co-authored-by: sdcoffey Co-authored-by: Codex Co-authored-by: Won Park Co-authored-by: Dylan Hurd Co-authored-by: celia-oai Co-authored-by: gabec-openai Co-authored-by: joeytrasatti-openai Co-authored-by: Leo Shimonaka Co-authored-by: Rasmus Rygaard Co-authored-by: maja-openai <163171781+maja-openai@users.noreply.github.com> Co-authored-by: pash-openai Co-authored-by: Josh McKinney --- codex-rs/Cargo.lock | 15 +- codex-rs/Cargo.toml | 2 + codex-rs/chatgpt/Cargo.toml | 2 +- codex-rs/chatgpt/src/connectors.rs | 386 +------------ codex-rs/connectors/BUILD.bazel | 6 + codex-rs/connectors/Cargo.toml | 18 + codex-rs/connectors/src/lib.rs | 534 ++++++++++++++++++ codex-rs/core/Cargo.toml | 1 + codex-rs/core/config.schema.json | 6 + codex-rs/core/src/codex.rs | 62 +- codex-rs/core/src/codex_tests.rs | 28 +- codex-rs/core/src/connectors.rs | 341 ++++++++++- codex-rs/core/src/features.rs | 14 + codex-rs/core/src/mcp_connection_manager.rs | 12 +- codex-rs/core/src/tools/code_mode.rs | 10 +- codex-rs/core/src/tools/discoverable.rs | 111 ++++ codex-rs/core/src/tools/handlers/mod.rs | 3 + .../src/tools/handlers/search_tool_bm25.rs | 0 .../core/src/tools/handlers/tool_suggest.rs | 465 +++++++++++++++ codex-rs/core/src/tools/js_repl/mod.rs | 19 +- codex-rs/core/src/tools/mod.rs | 1 + codex-rs/core/src/tools/router.rs | 70 ++- codex-rs/core/src/tools/spec.rs | 301 +++++++++- .../search_tool/tool_suggest_description.md | 25 + .../core/tests/common/apps_test_server.rs | 34 ++ codex-rs/tui/src/app.rs | 3 + codex-rs/tui/src/bottom_pane/app_link_view.rs | 348 ++++++++++++ .../src/bottom_pane/mcp_server_elicitation.rs | 128 ++++- codex-rs/tui/src/bottom_pane/mod.rs | 48 ++ ...nk_view_enable_suggestion_with_reason.snap | 20 + ...k_view_install_suggestion_with_reason.snap | 18 + 31 files changed, 2594 insertions(+), 437 deletions(-) create mode 100644 codex-rs/connectors/BUILD.bazel create mode 100644 codex-rs/connectors/Cargo.toml create mode 100644 codex-rs/connectors/src/lib.rs create mode 100644 codex-rs/core/src/tools/discoverable.rs delete mode 100644 codex-rs/core/src/tools/handlers/search_tool_bm25.rs create mode 100644 codex-rs/core/src/tools/handlers/tool_suggest.rs create mode 100644 codex-rs/core/templates/search_tool/tool_suggest_description.md create mode 100644 codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__app_link_view__tests__app_link_view_enable_suggestion_with_reason.snap create mode 100644 codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__app_link_view__tests__app_link_view_install_suggestion_with_reason.snap diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index addd5ea835..097fef2af1 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1619,6 +1619,7 @@ version = "0.0.0" dependencies = [ "anyhow", "clap", + "codex-connectors", "codex-core", "codex-git", "codex-utils-cargo-bin", @@ -1628,7 +1629,6 @@ dependencies = [ "serde_json", "tempfile", "tokio", - "urlencoding", ] [[package]] @@ -1790,6 +1790,18 @@ dependencies = [ "tracing", ] +[[package]] +name = "codex-connectors" +version = "0.0.0" +dependencies = [ + "anyhow", + "codex-app-server-protocol", + "pretty_assertions", + "serde", + "tokio", + "urlencoding", +] + [[package]] name = "codex-core" version = "0.0.0" @@ -1814,6 +1826,7 @@ dependencies = [ "codex-async-utils", "codex-client", "codex-config", + "codex-connectors", "codex-execpolicy", "codex-file-search", "codex-git", diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index 681487c099..77ffb61204 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -16,6 +16,7 @@ members = [ "cloud-tasks", "cloud-tasks-client", "cli", + "connectors", "config", "shell-command", "shell-escalation", @@ -98,6 +99,7 @@ codex-chatgpt = { path = "chatgpt" } codex-cli = { path = "cli" } codex-client = { path = "codex-client" } codex-cloud-requirements = { path = "cloud-requirements" } +codex-connectors = { path = "connectors" } codex-config = { path = "config" } codex-core = { path = "core" } codex-exec = { path = "exec" } diff --git a/codex-rs/chatgpt/Cargo.toml b/codex-rs/chatgpt/Cargo.toml index 823b63cad2..17a6f97ad3 100644 --- a/codex-rs/chatgpt/Cargo.toml +++ b/codex-rs/chatgpt/Cargo.toml @@ -10,6 +10,7 @@ workspace = true [dependencies] anyhow = { workspace = true } clap = { workspace = true, features = ["derive"] } +codex-connectors = { workspace = true } codex-core = { workspace = true } codex-utils-cli = { workspace = true } codex-utils-cargo-bin = { workspace = true } @@ -17,7 +18,6 @@ serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } tokio = { workspace = true, features = ["full"] } codex-git = { workspace = true } -urlencoding = { workspace = true } [dev-dependencies] pretty_assertions = { workspace = true } diff --git a/codex-rs/chatgpt/src/connectors.rs b/codex-rs/chatgpt/src/connectors.rs index dfc05fe317..54e2590c65 100644 --- a/codex-rs/chatgpt/src/connectors.rs +++ b/codex-rs/chatgpt/src/connectors.rs @@ -1,25 +1,18 @@ -use std::collections::HashMap; -use std::collections::HashSet; -use std::sync::LazyLock; -use std::sync::Mutex as StdMutex; - use codex_core::AuthManager; use codex_core::config::Config; use codex_core::token_data::TokenData; -use serde::Deserialize; +use std::collections::HashSet; use std::time::Duration; -use std::time::Instant; use crate::chatgpt_client::chatgpt_get_request_with_timeout; use crate::chatgpt_token::get_chatgpt_token_data; use crate::chatgpt_token::init_chatgpt_token_from_auth; -use codex_core::connectors::AppBranding; +use codex_connectors::AllConnectorsCacheKey; +use codex_connectors::DirectoryListResponse; + pub use codex_core::connectors::AppInfo; -use codex_core::connectors::AppMetadata; -use codex_core::connectors::CONNECTORS_CACHE_TTL; pub use codex_core::connectors::connector_display_label; -use codex_core::connectors::connector_install_url; use codex_core::connectors::filter_disallowed_connectors; pub use codex_core::connectors::list_accessible_connectors_from_mcp_tools; pub use codex_core::connectors::list_accessible_connectors_from_mcp_tools_with_options; @@ -30,51 +23,8 @@ use codex_core::connectors::merge_plugin_apps; pub use codex_core::connectors::with_app_enabled_state; use codex_core::plugins::PluginsManager; -#[derive(Debug, Deserialize)] -struct DirectoryListResponse { - apps: Vec, - #[serde(alias = "nextToken")] - next_token: Option, -} - -#[derive(Debug, Deserialize, Clone)] -struct DirectoryApp { - id: String, - name: String, - description: Option, - #[serde(alias = "appMetadata")] - app_metadata: Option, - branding: Option, - labels: Option>, - #[serde(alias = "logoUrl")] - logo_url: Option, - #[serde(alias = "logoUrlDark")] - logo_url_dark: Option, - #[serde(alias = "distributionChannel")] - distribution_channel: Option, - visibility: Option, -} - const DIRECTORY_CONNECTORS_TIMEOUT: Duration = Duration::from_secs(60); -#[derive(Clone, PartialEq, Eq)] -struct AllConnectorsCacheKey { - chatgpt_base_url: String, - account_id: Option, - chatgpt_user_id: Option, - is_workspace_account: bool, -} - -#[derive(Clone)] -struct CachedAllConnectors { - key: AllConnectorsCacheKey, - expires_at: Instant, - connectors: Vec, -} - -static ALL_CONNECTORS_CACHE: LazyLock>> = - LazyLock::new(|| StdMutex::new(None)); - async fn apps_enabled(config: &Config) -> bool { let auth_manager = AuthManager::shared( config.codex_home.clone(), @@ -83,7 +33,6 @@ async fn apps_enabled(config: &Config) -> bool { ); config.features.apps_enabled(Some(&auth_manager)).await } - pub async fn list_connectors(config: &Config) -> anyhow::Result> { if !apps_enabled(config).await { return Ok(Vec::new()); @@ -117,7 +66,7 @@ pub async fn list_cached_all_connectors(config: &Config) -> Option> } let token_data = get_chatgpt_token_data()?; let cache_key = all_connectors_cache_key(config, &token_data); - read_cached_all_connectors(&cache_key).map(|connectors| { + codex_connectors::cached_all_connectors(&cache_key).map(|connectors| { let connectors = merge_plugin_apps(connectors, plugin_apps_for_config(config)); filter_disallowed_connectors(connectors) }) @@ -136,76 +85,31 @@ pub async fn list_all_connectors_with_options( let token_data = get_chatgpt_token_data().ok_or_else(|| anyhow::anyhow!("ChatGPT token not available"))?; let cache_key = all_connectors_cache_key(config, &token_data); - if !force_refetch && let Some(cached_connectors) = read_cached_all_connectors(&cache_key) { - let connectors = merge_plugin_apps(cached_connectors, plugin_apps_for_config(config)); - return Ok(filter_disallowed_connectors(connectors)); - } - - let mut apps = list_directory_connectors(config).await?; - if token_data.id_token.is_workspace_account() { - apps.extend(list_workspace_connectors(config).await?); - } - let mut connectors = merge_directory_apps(apps) - .into_iter() - .map(directory_app_to_app_info) - .collect::>(); - for connector in &mut connectors { - let install_url = match connector.install_url.take() { - Some(install_url) => install_url, - None => connector_install_url(&connector.name, &connector.id), - }; - connector.name = normalize_connector_name(&connector.name, &connector.id); - connector.description = normalize_connector_value(connector.description.as_deref()); - connector.install_url = Some(install_url); - connector.is_accessible = false; - } - connectors.sort_by(|left, right| { - left.name - .cmp(&right.name) - .then_with(|| left.id.cmp(&right.id)) - }); - let connectors = filter_disallowed_connectors(connectors); - write_cached_all_connectors(cache_key, &connectors); + let connectors = codex_connectors::list_all_connectors_with_options( + cache_key, + token_data.id_token.is_workspace_account(), + force_refetch, + |path| async move { + chatgpt_get_request_with_timeout::( + config, + path, + Some(DIRECTORY_CONNECTORS_TIMEOUT), + ) + .await + }, + ) + .await?; let connectors = merge_plugin_apps(connectors, plugin_apps_for_config(config)); Ok(filter_disallowed_connectors(connectors)) } fn all_connectors_cache_key(config: &Config, token_data: &TokenData) -> AllConnectorsCacheKey { - AllConnectorsCacheKey { - chatgpt_base_url: config.chatgpt_base_url.clone(), - account_id: token_data.account_id.clone(), - chatgpt_user_id: token_data.id_token.chatgpt_user_id.clone(), - is_workspace_account: token_data.id_token.is_workspace_account(), - } -} - -fn read_cached_all_connectors(cache_key: &AllConnectorsCacheKey) -> Option> { - let mut cache_guard = ALL_CONNECTORS_CACHE - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let now = Instant::now(); - - if let Some(cached) = cache_guard.as_ref() { - if now < cached.expires_at && cached.key == *cache_key { - return Some(cached.connectors.clone()); - } - if now >= cached.expires_at { - *cache_guard = None; - } - } - - None -} - -fn write_cached_all_connectors(cache_key: AllConnectorsCacheKey, connectors: &[AppInfo]) { - let mut cache_guard = ALL_CONNECTORS_CACHE - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - *cache_guard = Some(CachedAllConnectors { - key: cache_key, - expires_at: Instant::now() + CONNECTORS_CACHE_TTL, - connectors: connectors.to_vec(), - }); + AllConnectorsCacheKey::new( + config.chatgpt_base_url.clone(), + token_data.account_id.clone(), + token_data.id_token.chatgpt_user_id.clone(), + token_data.id_token.is_workspace_account(), + ) } fn plugin_apps_for_config(config: &Config) -> Vec { @@ -235,248 +139,10 @@ pub fn merge_connectors_with_accessible( filter_disallowed_connectors(merged) } -async fn list_directory_connectors(config: &Config) -> anyhow::Result> { - let mut apps = Vec::new(); - let mut next_token: Option = None; - loop { - let path = match next_token.as_deref() { - Some(token) => { - let encoded_token = urlencoding::encode(token); - format!( - "/connectors/directory/list?tier=categorized&token={encoded_token}&external_logos=true" - ) - } - None => "/connectors/directory/list?tier=categorized&external_logos=true".to_string(), - }; - let response: DirectoryListResponse = - chatgpt_get_request_with_timeout(config, path, Some(DIRECTORY_CONNECTORS_TIMEOUT)) - .await?; - apps.extend( - response - .apps - .into_iter() - .filter(|app| !is_hidden_directory_app(app)), - ); - next_token = response - .next_token - .map(|token| token.trim().to_string()) - .filter(|token| !token.is_empty()); - if next_token.is_none() { - break; - } - } - Ok(apps) -} - -async fn list_workspace_connectors(config: &Config) -> anyhow::Result> { - let response: anyhow::Result = chatgpt_get_request_with_timeout( - config, - "/connectors/directory/list_workspace?external_logos=true".to_string(), - Some(DIRECTORY_CONNECTORS_TIMEOUT), - ) - .await; - match response { - Ok(response) => Ok(response - .apps - .into_iter() - .filter(|app| !is_hidden_directory_app(app)) - .collect()), - Err(_) => Ok(Vec::new()), - } -} - -fn merge_directory_apps(apps: Vec) -> Vec { - let mut merged: HashMap = HashMap::new(); - for app in apps { - if let Some(existing) = merged.get_mut(&app.id) { - merge_directory_app(existing, app); - } else { - merged.insert(app.id.clone(), app); - } - } - merged.into_values().collect() -} - -fn merge_directory_app(existing: &mut DirectoryApp, incoming: DirectoryApp) { - let DirectoryApp { - id: _, - name, - description, - app_metadata, - branding, - labels, - logo_url, - logo_url_dark, - distribution_channel, - visibility: _, - } = incoming; - - let incoming_name_is_empty = name.trim().is_empty(); - if existing.name.trim().is_empty() && !incoming_name_is_empty { - existing.name = name; - } - - let incoming_description_present = description - .as_deref() - .map(|value| !value.trim().is_empty()) - .unwrap_or(false); - if incoming_description_present { - existing.description = description; - } - - if existing.logo_url.is_none() && logo_url.is_some() { - existing.logo_url = logo_url; - } - if existing.logo_url_dark.is_none() && logo_url_dark.is_some() { - existing.logo_url_dark = logo_url_dark; - } - if existing.distribution_channel.is_none() && distribution_channel.is_some() { - existing.distribution_channel = distribution_channel; - } - - if let Some(incoming_branding) = branding { - if let Some(existing_branding) = existing.branding.as_mut() { - if existing_branding.category.is_none() && incoming_branding.category.is_some() { - existing_branding.category = incoming_branding.category; - } - if existing_branding.developer.is_none() && incoming_branding.developer.is_some() { - existing_branding.developer = incoming_branding.developer; - } - if existing_branding.website.is_none() && incoming_branding.website.is_some() { - existing_branding.website = incoming_branding.website; - } - if existing_branding.privacy_policy.is_none() - && incoming_branding.privacy_policy.is_some() - { - existing_branding.privacy_policy = incoming_branding.privacy_policy; - } - if existing_branding.terms_of_service.is_none() - && incoming_branding.terms_of_service.is_some() - { - existing_branding.terms_of_service = incoming_branding.terms_of_service; - } - if !existing_branding.is_discoverable_app && incoming_branding.is_discoverable_app { - existing_branding.is_discoverable_app = true; - } - } else { - existing.branding = Some(incoming_branding); - } - } - - if let Some(incoming_app_metadata) = app_metadata { - if let Some(existing_app_metadata) = existing.app_metadata.as_mut() { - if existing_app_metadata.review.is_none() && incoming_app_metadata.review.is_some() { - existing_app_metadata.review = incoming_app_metadata.review; - } - if existing_app_metadata.categories.is_none() - && incoming_app_metadata.categories.is_some() - { - existing_app_metadata.categories = incoming_app_metadata.categories; - } - if existing_app_metadata.sub_categories.is_none() - && incoming_app_metadata.sub_categories.is_some() - { - existing_app_metadata.sub_categories = incoming_app_metadata.sub_categories; - } - if existing_app_metadata.seo_description.is_none() - && incoming_app_metadata.seo_description.is_some() - { - existing_app_metadata.seo_description = incoming_app_metadata.seo_description; - } - if existing_app_metadata.screenshots.is_none() - && incoming_app_metadata.screenshots.is_some() - { - existing_app_metadata.screenshots = incoming_app_metadata.screenshots; - } - if existing_app_metadata.developer.is_none() - && incoming_app_metadata.developer.is_some() - { - existing_app_metadata.developer = incoming_app_metadata.developer; - } - if existing_app_metadata.version.is_none() && incoming_app_metadata.version.is_some() { - existing_app_metadata.version = incoming_app_metadata.version; - } - if existing_app_metadata.version_id.is_none() - && incoming_app_metadata.version_id.is_some() - { - existing_app_metadata.version_id = incoming_app_metadata.version_id; - } - if existing_app_metadata.version_notes.is_none() - && incoming_app_metadata.version_notes.is_some() - { - existing_app_metadata.version_notes = incoming_app_metadata.version_notes; - } - if existing_app_metadata.first_party_type.is_none() - && incoming_app_metadata.first_party_type.is_some() - { - existing_app_metadata.first_party_type = incoming_app_metadata.first_party_type; - } - if existing_app_metadata.first_party_requires_install.is_none() - && incoming_app_metadata.first_party_requires_install.is_some() - { - existing_app_metadata.first_party_requires_install = - incoming_app_metadata.first_party_requires_install; - } - if existing_app_metadata - .show_in_composer_when_unlinked - .is_none() - && incoming_app_metadata - .show_in_composer_when_unlinked - .is_some() - { - existing_app_metadata.show_in_composer_when_unlinked = - incoming_app_metadata.show_in_composer_when_unlinked; - } - } else { - existing.app_metadata = Some(incoming_app_metadata); - } - } - - if existing.labels.is_none() && labels.is_some() { - existing.labels = labels; - } -} - -fn is_hidden_directory_app(app: &DirectoryApp) -> bool { - matches!(app.visibility.as_deref(), Some("HIDDEN")) -} - -fn directory_app_to_app_info(app: DirectoryApp) -> AppInfo { - AppInfo { - id: app.id, - name: app.name, - description: app.description, - logo_url: app.logo_url, - logo_url_dark: app.logo_url_dark, - distribution_channel: app.distribution_channel, - branding: app.branding, - app_metadata: app.app_metadata, - labels: app.labels, - install_url: None, - is_accessible: false, - is_enabled: true, - plugin_display_names: Vec::new(), - } -} - -fn normalize_connector_name(name: &str, connector_id: &str) -> String { - let trimmed = name.trim(); - if trimmed.is_empty() { - connector_id.to_string() - } else { - trimmed.to_string() - } -} - -fn normalize_connector_value(value: Option<&str>) -> Option { - value - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_string) -} #[cfg(test)] mod tests { use super::*; + use codex_core::connectors::connector_install_url; use pretty_assertions::assert_eq; fn app(id: &str) -> AppInfo { diff --git a/codex-rs/connectors/BUILD.bazel b/codex-rs/connectors/BUILD.bazel new file mode 100644 index 0000000000..c4cb9ebde8 --- /dev/null +++ b/codex-rs/connectors/BUILD.bazel @@ -0,0 +1,6 @@ +load("//:defs.bzl", "codex_rust_crate") + +codex_rust_crate( + name = "connectors", + crate_name = "codex_connectors", +) diff --git a/codex-rs/connectors/Cargo.toml b/codex-rs/connectors/Cargo.toml new file mode 100644 index 0000000000..9cd2428a71 --- /dev/null +++ b/codex-rs/connectors/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "codex-connectors" +version.workspace = true +edition.workspace = true +license.workspace = true + +[lints] +workspace = true + +[dependencies] +anyhow = { workspace = true } +codex-app-server-protocol = { workspace = true } +serde = { workspace = true, features = ["derive"] } +urlencoding = { workspace = true } + +[dev-dependencies] +pretty_assertions = { workspace = true } +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/codex-rs/connectors/src/lib.rs b/codex-rs/connectors/src/lib.rs new file mode 100644 index 0000000000..1d3a729233 --- /dev/null +++ b/codex-rs/connectors/src/lib.rs @@ -0,0 +1,534 @@ +use std::collections::HashMap; +use std::future::Future; +use std::sync::LazyLock; +use std::sync::Mutex as StdMutex; +use std::time::Duration; +use std::time::Instant; + +use codex_app_server_protocol::AppBranding; +use codex_app_server_protocol::AppInfo; +use codex_app_server_protocol::AppMetadata; +use serde::Deserialize; + +pub const CONNECTORS_CACHE_TTL: Duration = Duration::from_secs(3600); + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AllConnectorsCacheKey { + chatgpt_base_url: String, + account_id: Option, + chatgpt_user_id: Option, + is_workspace_account: bool, +} + +impl AllConnectorsCacheKey { + pub fn new( + chatgpt_base_url: String, + account_id: Option, + chatgpt_user_id: Option, + is_workspace_account: bool, + ) -> Self { + Self { + chatgpt_base_url, + account_id, + chatgpt_user_id, + is_workspace_account, + } + } +} + +#[derive(Clone)] +struct CachedAllConnectors { + key: AllConnectorsCacheKey, + expires_at: Instant, + connectors: Vec, +} + +static ALL_CONNECTORS_CACHE: LazyLock>> = + LazyLock::new(|| StdMutex::new(None)); + +#[derive(Debug, Deserialize)] +pub struct DirectoryListResponse { + apps: Vec, + #[serde(alias = "nextToken")] + next_token: Option, +} + +#[derive(Debug, Deserialize, Clone)] +pub struct DirectoryApp { + id: String, + name: String, + description: Option, + #[serde(alias = "appMetadata")] + app_metadata: Option, + branding: Option, + labels: Option>, + #[serde(alias = "logoUrl")] + logo_url: Option, + #[serde(alias = "logoUrlDark")] + logo_url_dark: Option, + #[serde(alias = "distributionChannel")] + distribution_channel: Option, + visibility: Option, +} + +pub fn cached_all_connectors(cache_key: &AllConnectorsCacheKey) -> Option> { + let mut cache_guard = ALL_CONNECTORS_CACHE + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let now = Instant::now(); + + if let Some(cached) = cache_guard.as_ref() { + if now < cached.expires_at && cached.key == *cache_key { + return Some(cached.connectors.clone()); + } + if now >= cached.expires_at { + *cache_guard = None; + } + } + + None +} + +pub async fn list_all_connectors_with_options( + cache_key: AllConnectorsCacheKey, + is_workspace_account: bool, + force_refetch: bool, + mut fetch_page: F, +) -> anyhow::Result> +where + F: FnMut(String) -> Fut, + Fut: Future>, +{ + if !force_refetch && let Some(cached_connectors) = cached_all_connectors(&cache_key) { + return Ok(cached_connectors); + } + + let mut apps = list_directory_connectors(&mut fetch_page).await?; + if is_workspace_account { + apps.extend(list_workspace_connectors(&mut fetch_page).await?); + } + + let mut connectors = merge_directory_apps(apps) + .into_iter() + .map(directory_app_to_app_info) + .collect::>(); + for connector in &mut connectors { + let install_url = match connector.install_url.take() { + Some(install_url) => install_url, + None => connector_install_url(&connector.name, &connector.id), + }; + connector.name = normalize_connector_name(&connector.name, &connector.id); + connector.description = normalize_connector_value(connector.description.as_deref()); + connector.install_url = Some(install_url); + connector.is_accessible = false; + } + connectors.sort_by(|left, right| { + left.name + .cmp(&right.name) + .then_with(|| left.id.cmp(&right.id)) + }); + write_cached_all_connectors(cache_key, &connectors); + Ok(connectors) +} + +fn write_cached_all_connectors(cache_key: AllConnectorsCacheKey, connectors: &[AppInfo]) { + let mut cache_guard = ALL_CONNECTORS_CACHE + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + *cache_guard = Some(CachedAllConnectors { + key: cache_key, + expires_at: Instant::now() + CONNECTORS_CACHE_TTL, + connectors: connectors.to_vec(), + }); +} + +async fn list_directory_connectors(fetch_page: &mut F) -> anyhow::Result> +where + F: FnMut(String) -> Fut, + Fut: Future>, +{ + let mut apps = Vec::new(); + let mut next_token: Option = None; + loop { + let path = match next_token.as_deref() { + Some(token) => { + let encoded_token = urlencoding::encode(token); + format!( + "/connectors/directory/list?tier=categorized&token={encoded_token}&external_logos=true" + ) + } + None => "/connectors/directory/list?tier=categorized&external_logos=true".to_string(), + }; + let response = fetch_page(path).await?; + apps.extend( + response + .apps + .into_iter() + .filter(|app| !is_hidden_directory_app(app)), + ); + next_token = response + .next_token + .map(|token| token.trim().to_string()) + .filter(|token| !token.is_empty()); + if next_token.is_none() { + break; + } + } + Ok(apps) +} + +async fn list_workspace_connectors(fetch_page: &mut F) -> anyhow::Result> +where + F: FnMut(String) -> Fut, + Fut: Future>, +{ + let response = + fetch_page("/connectors/directory/list_workspace?external_logos=true".to_string()).await; + match response { + Ok(response) => Ok(response + .apps + .into_iter() + .filter(|app| !is_hidden_directory_app(app)) + .collect()), + Err(_) => Ok(Vec::new()), + } +} + +fn merge_directory_apps(apps: Vec) -> Vec { + let mut merged: HashMap = HashMap::new(); + for app in apps { + if let Some(existing) = merged.get_mut(&app.id) { + merge_directory_app(existing, app); + } else { + merged.insert(app.id.clone(), app); + } + } + merged.into_values().collect() +} + +fn merge_directory_app(existing: &mut DirectoryApp, incoming: DirectoryApp) { + let DirectoryApp { + id: _, + name, + description, + app_metadata, + branding, + labels, + logo_url, + logo_url_dark, + distribution_channel, + visibility: _, + } = incoming; + + let incoming_name_is_empty = name.trim().is_empty(); + if existing.name.trim().is_empty() && !incoming_name_is_empty { + existing.name = name; + } + + let incoming_description_present = description + .as_deref() + .map(|value| !value.trim().is_empty()) + .unwrap_or(false); + if incoming_description_present { + existing.description = description; + } + + if existing.logo_url.is_none() && logo_url.is_some() { + existing.logo_url = logo_url; + } + if existing.logo_url_dark.is_none() && logo_url_dark.is_some() { + existing.logo_url_dark = logo_url_dark; + } + if existing.distribution_channel.is_none() && distribution_channel.is_some() { + existing.distribution_channel = distribution_channel; + } + + if let Some(incoming_branding) = branding { + if let Some(existing_branding) = existing.branding.as_mut() { + if existing_branding.category.is_none() && incoming_branding.category.is_some() { + existing_branding.category = incoming_branding.category; + } + if existing_branding.developer.is_none() && incoming_branding.developer.is_some() { + existing_branding.developer = incoming_branding.developer; + } + if existing_branding.website.is_none() && incoming_branding.website.is_some() { + existing_branding.website = incoming_branding.website; + } + if existing_branding.privacy_policy.is_none() + && incoming_branding.privacy_policy.is_some() + { + existing_branding.privacy_policy = incoming_branding.privacy_policy; + } + if existing_branding.terms_of_service.is_none() + && incoming_branding.terms_of_service.is_some() + { + existing_branding.terms_of_service = incoming_branding.terms_of_service; + } + if !existing_branding.is_discoverable_app && incoming_branding.is_discoverable_app { + existing_branding.is_discoverable_app = true; + } + } else { + existing.branding = Some(incoming_branding); + } + } + + if let Some(incoming_app_metadata) = app_metadata { + if let Some(existing_app_metadata) = existing.app_metadata.as_mut() { + if existing_app_metadata.review.is_none() && incoming_app_metadata.review.is_some() { + existing_app_metadata.review = incoming_app_metadata.review; + } + if existing_app_metadata.categories.is_none() + && incoming_app_metadata.categories.is_some() + { + existing_app_metadata.categories = incoming_app_metadata.categories; + } + if existing_app_metadata.sub_categories.is_none() + && incoming_app_metadata.sub_categories.is_some() + { + existing_app_metadata.sub_categories = incoming_app_metadata.sub_categories; + } + if existing_app_metadata.seo_description.is_none() + && incoming_app_metadata.seo_description.is_some() + { + existing_app_metadata.seo_description = incoming_app_metadata.seo_description; + } + if existing_app_metadata.screenshots.is_none() + && incoming_app_metadata.screenshots.is_some() + { + existing_app_metadata.screenshots = incoming_app_metadata.screenshots; + } + if existing_app_metadata.developer.is_none() + && incoming_app_metadata.developer.is_some() + { + existing_app_metadata.developer = incoming_app_metadata.developer; + } + if existing_app_metadata.version.is_none() && incoming_app_metadata.version.is_some() { + existing_app_metadata.version = incoming_app_metadata.version; + } + if existing_app_metadata.version_id.is_none() + && incoming_app_metadata.version_id.is_some() + { + existing_app_metadata.version_id = incoming_app_metadata.version_id; + } + if existing_app_metadata.version_notes.is_none() + && incoming_app_metadata.version_notes.is_some() + { + existing_app_metadata.version_notes = incoming_app_metadata.version_notes; + } + if existing_app_metadata.first_party_type.is_none() + && incoming_app_metadata.first_party_type.is_some() + { + existing_app_metadata.first_party_type = incoming_app_metadata.first_party_type; + } + if existing_app_metadata.first_party_requires_install.is_none() + && incoming_app_metadata.first_party_requires_install.is_some() + { + existing_app_metadata.first_party_requires_install = + incoming_app_metadata.first_party_requires_install; + } + if existing_app_metadata + .show_in_composer_when_unlinked + .is_none() + && incoming_app_metadata + .show_in_composer_when_unlinked + .is_some() + { + existing_app_metadata.show_in_composer_when_unlinked = + incoming_app_metadata.show_in_composer_when_unlinked; + } + } else { + existing.app_metadata = Some(incoming_app_metadata); + } + } + + if existing.labels.is_none() && labels.is_some() { + existing.labels = labels; + } +} + +fn is_hidden_directory_app(app: &DirectoryApp) -> bool { + matches!(app.visibility.as_deref(), Some("HIDDEN")) +} + +fn directory_app_to_app_info(app: DirectoryApp) -> AppInfo { + AppInfo { + id: app.id, + name: app.name, + description: app.description, + logo_url: app.logo_url, + logo_url_dark: app.logo_url_dark, + distribution_channel: app.distribution_channel, + branding: app.branding, + app_metadata: app.app_metadata, + labels: app.labels, + install_url: None, + is_accessible: false, + is_enabled: true, + plugin_display_names: Vec::new(), + } +} + +fn connector_install_url(name: &str, connector_id: &str) -> String { + let slug = connector_name_slug(name); + format!("https://chatgpt.com/apps/{slug}/{connector_id}") +} + +fn connector_name_slug(name: &str) -> String { + let mut normalized = String::with_capacity(name.len()); + for character in name.chars() { + if character.is_ascii_alphanumeric() { + normalized.push(character.to_ascii_lowercase()); + } else { + normalized.push('-'); + } + } + let normalized = normalized.trim_matches('-'); + if normalized.is_empty() { + "app".to_string() + } else { + normalized.to_string() + } +} + +fn normalize_connector_name(name: &str, connector_id: &str) -> String { + let trimmed = name.trim(); + if trimmed.is_empty() { + connector_id.to_string() + } else { + trimmed.to_string() + } +} + +fn normalize_connector_value(value: Option<&str>) -> Option { + value + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_string) +} + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + use std::sync::Arc; + use std::sync::atomic::AtomicUsize; + use std::sync::atomic::Ordering; + + fn cache_key(id: &str) -> AllConnectorsCacheKey { + AllConnectorsCacheKey::new( + "https://chatgpt.example".to_string(), + Some(format!("account-{id}")), + Some(format!("user-{id}")), + true, + ) + } + + fn app(id: &str, name: &str) -> DirectoryApp { + DirectoryApp { + id: id.to_string(), + name: name.to_string(), + description: None, + app_metadata: None, + branding: None, + labels: None, + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + visibility: None, + } + } + + #[tokio::test] + async fn list_all_connectors_uses_shared_cache() -> anyhow::Result<()> { + let calls = Arc::new(AtomicUsize::new(0)); + let call_counter = Arc::clone(&calls); + let key = cache_key("shared"); + + let first = list_all_connectors_with_options(key.clone(), false, false, move |_path| { + let call_counter = Arc::clone(&call_counter); + async move { + call_counter.fetch_add(1, Ordering::SeqCst); + Ok(DirectoryListResponse { + apps: vec![app("alpha", "Alpha")], + next_token: None, + }) + } + }) + .await?; + + let second = list_all_connectors_with_options(key, false, false, move |_path| async move { + anyhow::bail!("cache should have been used"); + }) + .await?; + + assert_eq!(calls.load(Ordering::SeqCst), 1); + assert_eq!(first, second); + Ok(()) + } + + #[tokio::test] + async fn list_all_connectors_merges_and_normalizes_directory_apps() -> anyhow::Result<()> { + let key = cache_key("merged"); + let calls = Arc::new(AtomicUsize::new(0)); + let call_counter = Arc::clone(&calls); + + let connectors = list_all_connectors_with_options(key, true, true, move |path| { + let call_counter = Arc::clone(&call_counter); + async move { + call_counter.fetch_add(1, Ordering::SeqCst); + if path.starts_with("/connectors/directory/list_workspace") { + Ok(DirectoryListResponse { + apps: vec![ + DirectoryApp { + description: Some("Merged description".to_string()), + branding: Some(AppBranding { + category: Some("calendar".to_string()), + developer: None, + website: None, + privacy_policy: None, + terms_of_service: None, + is_discoverable_app: true, + }), + ..app("alpha", "") + }, + DirectoryApp { + visibility: Some("HIDDEN".to_string()), + ..app("hidden", "Hidden") + }, + ], + next_token: None, + }) + } else { + Ok(DirectoryListResponse { + apps: vec![app("alpha", " Alpha "), app("beta", "Beta")], + next_token: None, + }) + } + } + }) + .await?; + + assert_eq!(calls.load(Ordering::SeqCst), 2); + assert_eq!(connectors.len(), 2); + assert_eq!(connectors[0].id, "alpha"); + assert_eq!(connectors[0].name, "Alpha"); + assert_eq!( + connectors[0].description.as_deref(), + Some("Merged description") + ); + assert_eq!( + connectors[0].install_url.as_deref(), + Some("https://chatgpt.com/apps/alpha/alpha") + ); + assert_eq!( + connectors[0] + .branding + .as_ref() + .and_then(|branding| branding.category.as_deref()), + Some("calendar") + ); + assert_eq!(connectors[1].id, "beta"); + assert_eq!(connectors[1].name, "Beta"); + Ok(()) + } +} diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index e406586524..ef6b8a0132 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -32,6 +32,7 @@ codex-app-server-protocol = { workspace = true } codex-apply-patch = { workspace = true } codex-async-utils = { workspace = true } codex-client = { workspace = true } +codex-connectors = { workspace = true } codex-config = { workspace = true } codex-shell-command = { workspace = true } codex-skills = { workspace = true } diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.json index 79632344e5..3b7395f489 100644 --- a/codex-rs/core/config.schema.json +++ b/codex-rs/core/config.schema.json @@ -471,6 +471,9 @@ "tool_call_mcp_elicitation": { "type": "boolean" }, + "tool_suggest": { + "type": "boolean" + }, "undo": { "type": "boolean" }, @@ -1973,6 +1976,9 @@ "tool_call_mcp_elicitation": { "type": "boolean" }, + "tool_suggest": { + "type": "boolean" + }, "undo": { "type": "boolean" }, diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index d7070981c1..5fa4cffa3c 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -287,12 +287,14 @@ use crate::tasks::SessionTask; use crate::tasks::SessionTaskContext; use crate::tools::ToolRouter; use crate::tools::context::SharedTurnDiffTracker; +use crate::tools::discoverable::DiscoverableTool; use crate::tools::js_repl::JsReplHandle; use crate::tools::js_repl::resolve_compatible_node; use crate::tools::network_approval::NetworkApprovalService; use crate::tools::network_approval::build_blocked_request_observer; use crate::tools::network_approval::build_network_policy_decider; use crate::tools::parallel::ToolCallRuntime; +use crate::tools::router::ToolRouterParams; use crate::tools::sandboxing::ApprovalStore; use crate::tools::spec::ToolsConfig; use crate::tools::spec::ToolsConfigParams; @@ -6246,7 +6248,7 @@ async fn run_sampling_request( } } -async fn built_tools( +pub(crate) async fn built_tools( sess: &Session, turn_context: &TurnContext, input: &[ResponseItem], @@ -6269,10 +6271,17 @@ async fn built_tools( let mut effective_explicitly_enabled_connectors = explicitly_enabled_connectors.clone(); effective_explicitly_enabled_connectors.extend(sess.get_connector_selection().await); - let connectors = if turn_context.apps_enabled() { + let apps_enabled = turn_context.apps_enabled(); + let accessible_connectors = + apps_enabled.then(|| connectors::accessible_connectors_from_mcp_tools(&mcp_tools)); + let accessible_connectors_with_enabled_state = + accessible_connectors.as_ref().map(|connectors| { + connectors::with_app_enabled_state(connectors.clone(), &turn_context.config) + }); + let connectors = if apps_enabled { let connectors = connectors::merge_plugin_apps_with_accessible( loaded_plugins.effective_apps(), - connectors::accessible_connectors_from_mcp_tools(&mcp_tools), + accessible_connectors.clone().unwrap_or_default(), ); Some(connectors::with_app_enabled_state( connectors, @@ -6281,6 +6290,34 @@ async fn built_tools( } else { None }; + let auth = sess.services.auth_manager.auth().await; + let discoverable_tools = if apps_enabled + && turn_context.tools_config.search_tool + && turn_context.tools_config.tool_suggest + { + if let Some(accessible_connectors) = accessible_connectors_with_enabled_state.as_ref() { + match connectors::list_tool_suggest_discoverable_tools_with_auth( + &turn_context.config, + auth.as_ref(), + accessible_connectors.as_slice(), + ) + .await + { + Ok(connectors) if connectors.is_empty() => None, + Ok(connectors) => { + Some(connectors.into_iter().map(DiscoverableTool::from).collect()) + } + Err(err) => { + warn!("failed to load discoverable tool suggestions: {err:#}"); + None + } + } + } else { + None + } + } else { + None + }; // Keep the connector-grouped app view around for the router even though // app tools only become prompt-visible after explicit selection/discovery. @@ -6312,14 +6349,17 @@ async fn built_tools( Ok(Arc::new(ToolRouter::from_config( &turn_context.tools_config, - has_mcp_servers.then(|| { - mcp_tools - .into_iter() - .map(|(name, tool)| (name, tool.tool)) - .collect() - }), - app_tools, - turn_context.dynamic_tools.as_slice(), + ToolRouterParams { + mcp_tools: has_mcp_servers.then(|| { + mcp_tools + .into_iter() + .map(|(name, tool)| (name, tool.tool)) + .collect() + }), + app_tools, + discoverable_tools, + dynamic_tools: turn_context.dynamic_tools.as_slice(), + }, ))) } diff --git a/codex-rs/core/src/codex_tests.rs b/codex-rs/core/src/codex_tests.rs index 07881b153a..f1892449b1 100644 --- a/codex-rs/core/src/codex_tests.rs +++ b/codex-rs/core/src/codex_tests.rs @@ -165,9 +165,12 @@ fn default_image_save_developer_message_text() -> String { fn test_tool_runtime(session: Arc, turn_context: Arc) -> ToolCallRuntime { let router = Arc::new(ToolRouter::from_config( &turn_context.tools_config, - None, - None, - turn_context.dynamic_tools.as_slice(), + crate::tools::router::ToolRouterParams { + mcp_tools: None, + app_tools: None, + discoverable_tools: None, + dynamic_tools: turn_context.dynamic_tools.as_slice(), + }, )); let tracker = Arc::new(tokio::sync::Mutex::new(TurnDiffTracker::new())); ToolCallRuntime::new(router, session, turn_context, tracker) @@ -3954,14 +3957,17 @@ async fn fatal_tool_error_stops_turn_and_reports_error() { let app_tools = Some(tools.clone()); let router = ToolRouter::from_config( &turn_context.tools_config, - Some( - tools - .into_iter() - .map(|(name, tool)| (name, tool.tool)) - .collect(), - ), - app_tools, - turn_context.dynamic_tools.as_slice(), + crate::tools::router::ToolRouterParams { + mcp_tools: Some( + tools + .into_iter() + .map(|(name, tool)| (name, tool.tool)) + .collect(), + ), + app_tools, + discoverable_tools: None, + dynamic_tools: turn_context.dynamic_tools.as_slice(), + }, ); let item = ResponseItem::CustomToolCall { id: None, diff --git a/codex-rs/core/src/connectors.rs b/codex-rs/core/src/connectors.rs index 405eb18685..47d61fe984 100644 --- a/codex-rs/core/src/connectors.rs +++ b/codex-rs/core/src/connectors.rs @@ -9,13 +9,17 @@ use std::sync::Mutex as StdMutex; use std::time::Duration; use std::time::Instant; +use anyhow::Context; use async_channel::unbounded; pub use codex_app_server_protocol::AppBranding; pub use codex_app_server_protocol::AppInfo; pub use codex_app_server_protocol::AppMetadata; +use codex_connectors::AllConnectorsCacheKey; +use codex_connectors::DirectoryListResponse; use codex_protocol::protocol::SandboxPolicy; use rmcp::model::ToolAnnotations; use serde::Deserialize; +use serde::de::DeserializeOwned; use tracing::warn; use crate::AuthManager; @@ -24,6 +28,7 @@ use crate::SandboxState; use crate::config::Config; use crate::config::types::AppToolApproval; use crate::config::types::AppsConfigToml; +use crate::default_client::create_client; use crate::default_client::is_first_party_chat_originator; use crate::default_client::originator; use crate::features::Feature; @@ -38,8 +43,22 @@ use crate::plugins::AppConnectorId; use crate::plugins::PluginsManager; use crate::token_data::TokenData; -pub const CONNECTORS_CACHE_TTL: Duration = Duration::from_secs(3600); +pub use codex_connectors::CONNECTORS_CACHE_TTL; const CONNECTORS_READY_TIMEOUT_ON_EMPTY_TOOLS: Duration = Duration::from_secs(30); +const DIRECTORY_CONNECTORS_TIMEOUT: Duration = Duration::from_secs(60); +const TOOL_SUGGEST_DISCOVERABLE_CONNECTOR_IDS: &[&str] = &[ + "connector_2128aebfecb84f64a069897515042a44", + "connector_68df038e0ba48191908c8434991bbac2", + "asdk_app_69a1d78e929881919bba0dbda1f6436d", + "connector_4964e3b22e3e427e9b4ae1acf2c1fa34", + "connector_9d7cfa34e6654a5f98d3387af34b2e1c", + "connector_6f1ec045b8fa4ced8738e32c7f74514b", + "connector_947e0d954944416db111db556030eea6", + "connector_5f3c8c41a1e54ad7a76272c89e2554fa", + "connector_686fad9b54914a35b75be6d06a0f6f31", + "connector_76869538009648d5b282a4bb21c3d157", + "connector_37316be7febe4224b3d31465bae4dbd7", +]; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) struct AppToolPolicy { @@ -90,6 +109,19 @@ pub async fn list_accessible_connectors_from_mcp_tools( ) } +pub(crate) async fn list_tool_suggest_discoverable_tools_with_auth( + config: &Config, + auth: Option<&CodexAuth>, + accessible_connectors: &[AppInfo], +) -> anyhow::Result> { + let directory_connectors = + list_directory_connectors_for_tool_suggest_with_auth(config, auth).await?; + Ok(filter_tool_suggest_discoverable_tools( + directory_connectors, + accessible_connectors, + )) +} + pub async fn list_cached_accessible_connectors_from_mcp_tools( config: &Config, ) -> Option> { @@ -102,6 +134,21 @@ pub async fn list_cached_accessible_connectors_from_mcp_tools( read_cached_accessible_connectors(&cache_key).map(filter_disallowed_connectors) } +pub(crate) fn refresh_accessible_connectors_cache_from_mcp_tools( + config: &Config, + auth: Option<&CodexAuth>, + mcp_tools: &HashMap, +) { + if !config.features.enabled(Feature::Apps) { + return; + } + + let cache_key = accessible_connectors_cache_key(config, auth); + let accessible_connectors = + filter_disallowed_connectors(accessible_connectors_from_mcp_tools(mcp_tools)); + write_cached_accessible_connectors(cache_key, &accessible_connectors); +} + pub async fn list_accessible_connectors_from_mcp_tools_with_options( config: &Config, force_refetch: bool, @@ -172,19 +219,33 @@ pub async fn list_accessible_connectors_from_mcp_tools_with_options_and_status( ) .await; - if force_refetch - && let Err(err) = mcp_connection_manager + let refreshed_tools = if force_refetch { + match mcp_connection_manager .hard_refresh_codex_apps_tools_cache() .await - { - warn!( - "failed to force-refresh tools for MCP server '{CODEX_APPS_MCP_SERVER_NAME}', using cached/startup tools: {err:#}" - ); - } + { + Ok(tools) => Some(tools), + Err(err) => { + warn!( + "failed to force-refresh tools for MCP server '{CODEX_APPS_MCP_SERVER_NAME}', using cached/startup tools: {err:#}" + ); + None + } + } + } else { + None + }; + let refreshed_tools_succeeded = refreshed_tools.is_some(); - let mut tools = mcp_connection_manager.list_all_tools().await; + let mut tools = if let Some(tools) = refreshed_tools { + tools + } else { + mcp_connection_manager.list_all_tools().await + }; let mut should_reload_tools = false; - let codex_apps_ready = if let Some(cfg) = mcp_servers.get(CODEX_APPS_MCP_SERVER_NAME) { + let codex_apps_ready = if refreshed_tools_succeeded { + true + } else if let Some(cfg) = mcp_servers.get(CODEX_APPS_MCP_SERVER_NAME) { let immediate_ready = mcp_connection_manager .wait_for_server_ready(CODEX_APPS_MCP_SERVER_NAME, Duration::ZERO) .await; @@ -281,6 +342,119 @@ fn write_cached_accessible_connectors( }); } +fn filter_tool_suggest_discoverable_tools( + directory_connectors: Vec, + accessible_connectors: &[AppInfo], +) -> Vec { + let accessible_connector_ids: HashSet<&str> = accessible_connectors + .iter() + .filter(|connector| connector.is_accessible && connector.is_enabled) + .map(|connector| connector.id.as_str()) + .collect(); + let allowed_connector_ids: HashSet<&str> = TOOL_SUGGEST_DISCOVERABLE_CONNECTOR_IDS + .iter() + .copied() + .collect(); + + let mut connectors = filter_disallowed_connectors(directory_connectors) + .into_iter() + .filter(|connector| !accessible_connector_ids.contains(connector.id.as_str())) + .filter(|connector| allowed_connector_ids.contains(connector.id.as_str())) + .collect::>(); + connectors.sort_by(|left, right| { + left.name + .cmp(&right.name) + .then_with(|| left.id.cmp(&right.id)) + }); + connectors +} + +async fn list_directory_connectors_for_tool_suggest_with_auth( + config: &Config, + auth: Option<&CodexAuth>, +) -> anyhow::Result> { + if !config.features.enabled(Feature::Apps) { + return Ok(Vec::new()); + } + + let token_data = if let Some(auth) = auth { + auth.get_token_data().ok() + } else { + let auth_manager = auth_manager_from_config(config); + auth_manager + .auth() + .await + .and_then(|auth| auth.get_token_data().ok()) + }; + let Some(token_data) = token_data else { + return Ok(Vec::new()); + }; + + let account_id = match token_data.account_id.as_deref() { + Some(account_id) if !account_id.is_empty() => account_id, + _ => return Ok(Vec::new()), + }; + let access_token = token_data.access_token.clone(); + let account_id = account_id.to_string(); + let is_workspace_account = token_data.id_token.is_workspace_account(); + let cache_key = AllConnectorsCacheKey::new( + config.chatgpt_base_url.clone(), + Some(account_id.clone()), + token_data.id_token.chatgpt_user_id.clone(), + is_workspace_account, + ); + + codex_connectors::list_all_connectors_with_options( + cache_key, + is_workspace_account, + false, + |path| { + let access_token = access_token.clone(); + let account_id = account_id.clone(); + async move { + chatgpt_get_request_with_token::( + config, + path, + access_token.as_str(), + account_id.as_str(), + ) + .await + } + }, + ) + .await +} + +async fn chatgpt_get_request_with_token( + config: &Config, + path: String, + access_token: &str, + account_id: &str, +) -> anyhow::Result { + let client = create_client(); + let url = format!("{}{}", config.chatgpt_base_url, path); + let response = client + .get(&url) + .bearer_auth(access_token) + .header("chatgpt-account-id", account_id) + .header("Content-Type", "application/json") + .timeout(DIRECTORY_CONNECTORS_TIMEOUT) + .send() + .await + .context("failed to send request")?; + + if response.status().is_success() { + response + .json() + .await + .context("failed to parse JSON response") + } else { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + anyhow::bail!("request failed with status {status}: {body}"); + } +} + fn auth_manager_from_config(config: &Config) -> std::sync::Arc { AuthManager::shared( config.codex_home.clone(), @@ -719,10 +893,12 @@ fn format_connector_label(name: &str, _id: &str) -> String { #[cfg(test)] mod tests { use super::*; + use crate::config::ConfigBuilder; use crate::config::types::AppConfig; use crate::config::types::AppToolConfig; use crate::config::types::AppToolsConfig; use crate::config::types::AppsDefaultConfig; + use crate::features::Feature; use crate::mcp::CODEX_APPS_MCP_SERVER_NAME; use crate::mcp_connection_manager::ToolInfo; use pretty_assertions::assert_eq; @@ -730,6 +906,7 @@ mod tests { use rmcp::model::Tool; use std::collections::HashMap; use std::sync::Arc; + use tempfile::tempdir; fn annotations( destructive_hint: Option, @@ -762,6 +939,15 @@ mod tests { } } + fn named_app(id: &str, name: &str) -> AppInfo { + AppInfo { + id: id.to_string(), + name: name.to_string(), + install_url: Some(connector_install_url(name, id)), + ..app(id) + } + } + fn plugin_names(names: &[&str]) -> Vec { names.iter().map(ToString::to_string).collect() } @@ -821,6 +1007,21 @@ mod tests { } } + fn with_accessible_connectors_cache_cleared(f: impl FnOnce() -> R) -> R { + let previous = { + let mut cache_guard = ACCESSIBLE_CONNECTORS_CACHE + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + cache_guard.take() + }; + let result = f(); + let mut cache_guard = ACCESSIBLE_CONNECTORS_CACHE + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + *cache_guard = previous; + result + } + #[test] fn merge_connectors_replaces_plugin_placeholder_name_with_accessible_name() { let plugin = plugin_app_to_app_info(AppConnectorId("calendar".to_string())); @@ -907,6 +1108,62 @@ mod tests { ); } + #[tokio::test] + async fn refresh_accessible_connectors_cache_from_mcp_tools_writes_latest_installed_apps() { + let codex_home = tempdir().expect("tempdir should succeed"); + let mut config = ConfigBuilder::default() + .codex_home(codex_home.path().to_path_buf()) + .build() + .await + .expect("config should load"); + let _ = config.features.set_enabled(Feature::Apps, true); + let cache_key = accessible_connectors_cache_key(&config, None); + let tools = HashMap::from([ + ( + "mcp__codex_apps__calendar_list_events".to_string(), + codex_app_tool( + "calendar_list_events", + "calendar", + Some("Google Calendar"), + &["calendar-plugin"], + ), + ), + ( + "mcp__codex_apps__openai_hidden".to_string(), + codex_app_tool( + "openai_hidden", + "connector_openai_hidden", + Some("Hidden"), + &[], + ), + ), + ]); + + let cached = with_accessible_connectors_cache_cleared(|| { + refresh_accessible_connectors_cache_from_mcp_tools(&config, None, &tools); + read_cached_accessible_connectors(&cache_key).expect("cache should be populated") + }); + + assert_eq!( + cached, + vec![AppInfo { + id: "calendar".to_string(), + name: "Google Calendar".to_string(), + description: None, + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + install_url: Some(connector_install_url("Google Calendar", "calendar")), + branding: None, + app_metadata: None, + labels: None, + is_accessible: true, + is_enabled: true, + plugin_display_names: plugin_names(&["calendar-plugin"]), + }] + ); + } + #[test] fn merge_connectors_unions_and_dedupes_plugin_display_names() { let mut plugin = plugin_app_to_app_info(AppConnectorId("calendar".to_string())); @@ -1344,4 +1601,68 @@ mod tests { vec![app("asdk_app_6938a94a61d881918ef32cb999ff937c")] ); } + + #[test] + fn filter_tool_suggest_discoverable_tools_keeps_only_allowlisted_uninstalled_apps() { + let filtered = filter_tool_suggest_discoverable_tools( + vec![ + named_app( + "connector_2128aebfecb84f64a069897515042a44", + "Google Calendar", + ), + named_app("connector_68df038e0ba48191908c8434991bbac2", "Gmail"), + named_app("connector_other", "Other"), + ], + &[AppInfo { + is_accessible: true, + ..named_app( + "connector_2128aebfecb84f64a069897515042a44", + "Google Calendar", + ) + }], + ); + + assert_eq!( + filtered, + vec![named_app( + "connector_68df038e0ba48191908c8434991bbac2", + "Gmail", + )] + ); + } + + #[test] + fn filter_tool_suggest_discoverable_tools_keeps_disabled_accessible_apps() { + let filtered = filter_tool_suggest_discoverable_tools( + vec![ + named_app( + "connector_2128aebfecb84f64a069897515042a44", + "Google Calendar", + ), + named_app("connector_68df038e0ba48191908c8434991bbac2", "Gmail"), + ], + &[ + AppInfo { + is_accessible: true, + ..named_app( + "connector_2128aebfecb84f64a069897515042a44", + "Google Calendar", + ) + }, + AppInfo { + is_accessible: true, + is_enabled: false, + ..named_app("connector_68df038e0ba48191908c8434991bbac2", "Gmail") + }, + ], + ); + + assert_eq!( + filtered, + vec![named_app( + "connector_68df038e0ba48191908c8434991bbac2", + "Gmail" + )] + ); + } } diff --git a/codex-rs/core/src/features.rs b/codex-rs/core/src/features.rs index fb08e157b3..76da67c323 100644 --- a/codex-rs/core/src/features.rs +++ b/codex-rs/core/src/features.rs @@ -142,6 +142,8 @@ pub enum Feature { SpawnCsv, /// Enable apps. Apps, + /// Enable discoverable tool suggestions for apps. + ToolSuggest, /// Enable plugins. Plugins, /// Allow the model to invoke the built-in image generation tool. @@ -714,6 +716,12 @@ pub const FEATURES: &[FeatureSpec] = &[ }, default_enabled: false, }, + FeatureSpec { + id: Feature::ToolSuggest, + key: "tool_suggest", + stage: Stage::UnderDevelopment, + default_enabled: false, + }, FeatureSpec { id: Feature::Plugins, key: "plugins", @@ -996,6 +1004,12 @@ mod tests { assert_eq!(Feature::RequestPermissionsTool.default_enabled(), false); } + #[test] + fn tool_suggest_is_under_development() { + assert_eq!(Feature::ToolSuggest.stage(), Stage::UnderDevelopment); + assert_eq!(Feature::ToolSuggest.default_enabled(), false); + } + #[test] fn image_generation_is_under_development() { assert_eq!(Feature::ImageGeneration.stage(), Stage::UnderDevelopment); diff --git a/codex-rs/core/src/mcp_connection_manager.rs b/codex-rs/core/src/mcp_connection_manager.rs index 009b85d831..de8ad51430 100644 --- a/codex-rs/core/src/mcp_connection_manager.rs +++ b/codex-rs/core/src/mcp_connection_manager.rs @@ -827,9 +827,10 @@ impl McpConnectionManager { /// Force-refresh codex apps tools by bypassing the in-process cache. /// - /// On success, the refreshed tools replace the cache contents. On failure, - /// the existing cache remains unchanged. - pub async fn hard_refresh_codex_apps_tools_cache(&self) -> Result<()> { + /// On success, the refreshed tools replace the cache contents and the + /// latest filtered tool map is returned directly to the caller. On + /// failure, the existing cache remains unchanged. + pub async fn hard_refresh_codex_apps_tools_cache(&self) -> Result> { let managed_client = self .clients .get(CODEX_APPS_MCP_SERVER_NAME) @@ -865,7 +866,10 @@ impl McpConnectionManager { list_start.elapsed(), &[("cache", "miss")], ); - Ok(()) + Ok(qualify_tools(filter_tools( + tools, + &managed_client.tool_filter, + ))) } /// Returns a single map that contains all resources. Each key is the diff --git a/codex-rs/core/src/tools/code_mode.rs b/codex-rs/core/src/tools/code_mode.rs index fd0587c71f..110588469b 100644 --- a/codex-rs/core/src/tools/code_mode.rs +++ b/codex-rs/core/src/tools/code_mode.rs @@ -18,6 +18,7 @@ use crate::tools::context::ToolPayload; use crate::tools::js_repl::resolve_compatible_node; use crate::tools::router::ToolCall; use crate::tools::router::ToolCallSource; +use crate::tools::router::ToolRouterParams; use crate::truncate::TruncationPolicy; use crate::truncate::formatted_truncate_text_content_items_with_policy; use crate::truncate::truncate_function_output_items_with_policy; @@ -408,9 +409,12 @@ async fn build_nested_router(exec: &ExecContext) -> ToolRouter { ToolRouter::from_config( &nested_tools_config, - Some(mcp_tools), - None, - exec.turn.dynamic_tools.as_slice(), + ToolRouterParams { + mcp_tools: Some(mcp_tools), + app_tools: None, + discoverable_tools: None, + dynamic_tools: exec.turn.dynamic_tools.as_slice(), + }, ) } diff --git a/codex-rs/core/src/tools/discoverable.rs b/codex-rs/core/src/tools/discoverable.rs new file mode 100644 index 0000000000..75de51b150 --- /dev/null +++ b/codex-rs/core/src/tools/discoverable.rs @@ -0,0 +1,111 @@ +use crate::plugins::PluginCapabilitySummary; +use codex_app_server_protocol::AppInfo; +use serde::Deserialize; +use serde::Serialize; + +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub(crate) enum DiscoverableToolType { + Connector, + Plugin, +} + +impl DiscoverableToolType { + pub(crate) fn as_str(self) -> &'static str { + match self { + Self::Connector => "connector", + Self::Plugin => "plugin", + } + } +} + +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub(crate) enum DiscoverableToolAction { + Install, + Enable, +} + +impl DiscoverableToolAction { + pub(crate) fn as_str(self) -> &'static str { + match self { + Self::Install => "install", + Self::Enable => "enable", + } + } +} + +#[derive(Clone, Debug, PartialEq)] +pub(crate) enum DiscoverableTool { + Connector(Box), + Plugin(Box), +} + +impl DiscoverableTool { + pub(crate) fn tool_type(&self) -> DiscoverableToolType { + match self { + Self::Connector(_) => DiscoverableToolType::Connector, + Self::Plugin(_) => DiscoverableToolType::Plugin, + } + } + + pub(crate) fn id(&self) -> &str { + match self { + Self::Connector(connector) => connector.id.as_str(), + Self::Plugin(plugin) => plugin.id.as_str(), + } + } + + pub(crate) fn name(&self) -> &str { + match self { + Self::Connector(connector) => connector.name.as_str(), + Self::Plugin(plugin) => plugin.name.as_str(), + } + } + + pub(crate) fn description(&self) -> Option<&str> { + match self { + Self::Connector(connector) => connector.description.as_deref(), + Self::Plugin(plugin) => plugin.description.as_deref(), + } + } +} + +impl From for DiscoverableTool { + fn from(value: AppInfo) -> Self { + Self::Connector(Box::new(value)) + } +} + +impl From for DiscoverableTool { + fn from(value: DiscoverablePluginInfo) -> Self { + Self::Plugin(Box::new(value)) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct DiscoverablePluginInfo { + pub(crate) id: String, + pub(crate) name: String, + pub(crate) description: Option, + pub(crate) has_skills: bool, + pub(crate) mcp_server_names: Vec, + pub(crate) app_connector_ids: Vec, +} + +impl From for DiscoverablePluginInfo { + fn from(value: PluginCapabilitySummary) -> Self { + Self { + id: value.config_name, + name: value.display_name, + description: value.description, + has_skills: value.has_skills, + mcp_server_names: value.mcp_server_names, + app_connector_ids: value + .app_connector_ids + .into_iter() + .map(|connector_id| connector_id.0) + .collect(), + } + } +} diff --git a/codex-rs/core/src/tools/handlers/mod.rs b/codex-rs/core/src/tools/handlers/mod.rs index 1bb9c7acc8..2177800463 100644 --- a/codex-rs/core/src/tools/handlers/mod.rs +++ b/codex-rs/core/src/tools/handlers/mod.rs @@ -16,6 +16,7 @@ mod request_user_input; mod shell; mod test_sync; mod tool_search; +mod tool_suggest; pub(crate) mod unified_exec; mod view_image; @@ -56,6 +57,8 @@ pub use test_sync::TestSyncHandler; pub(crate) use tool_search::DEFAULT_LIMIT as TOOL_SEARCH_DEFAULT_LIMIT; pub(crate) use tool_search::TOOL_SEARCH_TOOL_NAME; pub use tool_search::ToolSearchHandler; +pub(crate) use tool_suggest::TOOL_SUGGEST_TOOL_NAME; +pub use tool_suggest::ToolSuggestHandler; pub use unified_exec::UnifiedExecHandler; pub use view_image::ViewImageHandler; diff --git a/codex-rs/core/src/tools/handlers/search_tool_bm25.rs b/codex-rs/core/src/tools/handlers/search_tool_bm25.rs deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/codex-rs/core/src/tools/handlers/tool_suggest.rs b/codex-rs/core/src/tools/handlers/tool_suggest.rs new file mode 100644 index 0000000000..5483cac043 --- /dev/null +++ b/codex-rs/core/src/tools/handlers/tool_suggest.rs @@ -0,0 +1,465 @@ +use std::collections::BTreeMap; +use std::collections::HashSet; + +use async_trait::async_trait; +use codex_app_server_protocol::AppInfo; +use codex_app_server_protocol::McpElicitationObjectType; +use codex_app_server_protocol::McpElicitationSchema; +use codex_app_server_protocol::McpServerElicitationRequest; +use codex_app_server_protocol::McpServerElicitationRequestParams; +use codex_rmcp_client::ElicitationAction; +use rmcp::model::RequestId; +use serde::Deserialize; +use serde::Serialize; +use serde_json::json; +use tracing::warn; + +use crate::connectors; +use crate::function_tool::FunctionCallError; +use crate::mcp::CODEX_APPS_MCP_SERVER_NAME; +use crate::tools::context::FunctionToolOutput; +use crate::tools::context::ToolInvocation; +use crate::tools::context::ToolPayload; +use crate::tools::discoverable::DiscoverableTool; +use crate::tools::discoverable::DiscoverableToolAction; +use crate::tools::discoverable::DiscoverableToolType; +use crate::tools::handlers::parse_arguments; +use crate::tools::registry::ToolHandler; +use crate::tools::registry::ToolKind; + +pub struct ToolSuggestHandler; + +pub(crate) const TOOL_SUGGEST_TOOL_NAME: &str = "tool_suggest"; +const TOOL_SUGGEST_APPROVAL_KIND_VALUE: &str = "tool_suggestion"; + +#[derive(Debug, Deserialize)] +struct ToolSuggestArgs { + tool_type: DiscoverableToolType, + action_type: DiscoverableToolAction, + tool_id: String, + suggest_reason: String, +} + +#[derive(Debug, Serialize, PartialEq, Eq)] +struct ToolSuggestResult { + completed: bool, + user_confirmed: bool, + tool_type: DiscoverableToolType, + action_type: DiscoverableToolAction, + tool_id: String, + tool_name: String, + suggest_reason: String, +} + +#[derive(Debug, Serialize, PartialEq, Eq)] +struct ToolSuggestMeta<'a> { + codex_approval_kind: &'static str, + tool_type: DiscoverableToolType, + suggest_type: DiscoverableToolAction, + suggest_reason: &'a str, + tool_id: &'a str, + tool_name: &'a str, + install_url: &'a str, +} + +#[async_trait] +impl ToolHandler for ToolSuggestHandler { + type Output = FunctionToolOutput; + + fn kind(&self) -> ToolKind { + ToolKind::Function + } + + async fn handle(&self, invocation: ToolInvocation) -> Result { + let ToolInvocation { + payload, + session, + turn, + call_id, + .. + } = invocation; + + let arguments = match payload { + ToolPayload::Function { arguments } => arguments, + _ => { + return Err(FunctionCallError::Fatal(format!( + "{TOOL_SUGGEST_TOOL_NAME} handler received unsupported payload" + ))); + } + }; + + let args: ToolSuggestArgs = parse_arguments(&arguments)?; + let suggest_reason = args.suggest_reason.trim(); + if suggest_reason.is_empty() { + return Err(FunctionCallError::RespondToModel( + "suggest_reason must not be empty".to_string(), + )); + } + if args.tool_type == DiscoverableToolType::Plugin { + return Err(FunctionCallError::RespondToModel( + "plugin tool suggestions are not currently available".to_string(), + )); + } + if args.action_type != DiscoverableToolAction::Install { + return Err(FunctionCallError::RespondToModel( + "connector tool suggestions currently support only action_type=\"install\"" + .to_string(), + )); + } + + let auth = session.services.auth_manager.auth().await; + let manager = session.services.mcp_connection_manager.read().await; + let mcp_tools = manager.list_all_tools().await; + drop(manager); + let accessible_connectors = connectors::with_app_enabled_state( + connectors::accessible_connectors_from_mcp_tools(&mcp_tools), + &turn.config, + ); + let discoverable_tools = connectors::list_tool_suggest_discoverable_tools_with_auth( + &turn.config, + auth.as_ref(), + &accessible_connectors, + ) + .await + .map(|connectors| { + connectors + .into_iter() + .map(DiscoverableTool::from) + .collect::>() + }) + .map_err(|err| { + FunctionCallError::RespondToModel(format!( + "tool suggestions are unavailable right now: {err}" + )) + })?; + + let connector = discoverable_tools + .into_iter() + .find_map(|tool| match tool { + DiscoverableTool::Connector(connector) if connector.id == args.tool_id => { + Some(*connector) + } + DiscoverableTool::Connector(_) | DiscoverableTool::Plugin(_) => None, + }) + .ok_or_else(|| { + FunctionCallError::RespondToModel(format!( + "tool_id must match one of the discoverable tools exposed by {TOOL_SUGGEST_TOOL_NAME}" + )) + })?; + + let request_id = RequestId::String(format!("tool_suggestion_{call_id}").into()); + let params = build_tool_suggestion_elicitation_request( + session.conversation_id.to_string(), + turn.sub_id.clone(), + &args, + suggest_reason, + &connector, + ); + let response = session + .request_mcp_server_elicitation(turn.as_ref(), request_id, params) + .await; + let user_confirmed = response + .as_ref() + .is_some_and(|response| response.action == ElicitationAction::Accept); + + let completed = if user_confirmed { + let manager = session.services.mcp_connection_manager.read().await; + match manager.hard_refresh_codex_apps_tools_cache().await { + Ok(mcp_tools) => { + let accessible_connectors = connectors::with_app_enabled_state( + connectors::accessible_connectors_from_mcp_tools(&mcp_tools), + &turn.config, + ); + connectors::refresh_accessible_connectors_cache_from_mcp_tools( + &turn.config, + auth.as_ref(), + &mcp_tools, + ); + verified_connector_suggestion_completed( + args.action_type, + connector.id.as_str(), + &accessible_connectors, + ) + } + Err(err) => { + warn!( + "failed to refresh codex apps tools cache after tool suggestion for {}: {err:#}", + connector.id + ); + false + } + } + } else { + false + }; + + if completed { + session + .merge_connector_selection(HashSet::from([connector.id.clone()])) + .await; + } + + let content = serde_json::to_string(&ToolSuggestResult { + completed, + user_confirmed, + tool_type: args.tool_type, + action_type: args.action_type, + tool_id: connector.id, + tool_name: connector.name, + suggest_reason: suggest_reason.to_string(), + }) + .map_err(|err| { + FunctionCallError::Fatal(format!( + "failed to serialize {TOOL_SUGGEST_TOOL_NAME} response: {err}" + )) + })?; + + Ok(FunctionToolOutput::from_text(content, Some(true))) + } +} + +fn build_tool_suggestion_elicitation_request( + thread_id: String, + turn_id: String, + args: &ToolSuggestArgs, + suggest_reason: &str, + connector: &AppInfo, +) -> McpServerElicitationRequestParams { + let tool_name = connector.name.clone(); + let install_url = connector + .install_url + .clone() + .unwrap_or_else(|| connectors::connector_install_url(&tool_name, &connector.id)); + + let message = format!( + "{tool_name} could help with this request.\n\n{suggest_reason}\n\nOpen ChatGPT to {} it, then confirm here if you finish.", + args.action_type.as_str() + ); + + McpServerElicitationRequestParams { + thread_id, + turn_id: Some(turn_id), + server_name: CODEX_APPS_MCP_SERVER_NAME.to_string(), + request: McpServerElicitationRequest::Form { + meta: Some(json!(build_tool_suggestion_meta( + args.tool_type, + args.action_type, + suggest_reason, + connector.id.as_str(), + tool_name.as_str(), + install_url.as_str(), + ))), + message, + requested_schema: McpElicitationSchema { + schema_uri: None, + type_: McpElicitationObjectType::Object, + properties: BTreeMap::new(), + required: None, + }, + }, + } +} + +fn build_tool_suggestion_meta<'a>( + tool_type: DiscoverableToolType, + action_type: DiscoverableToolAction, + suggest_reason: &'a str, + tool_id: &'a str, + tool_name: &'a str, + install_url: &'a str, +) -> ToolSuggestMeta<'a> { + ToolSuggestMeta { + codex_approval_kind: TOOL_SUGGEST_APPROVAL_KIND_VALUE, + tool_type, + suggest_type: action_type, + suggest_reason, + tool_id, + tool_name, + install_url, + } +} + +fn verified_connector_suggestion_completed( + action_type: DiscoverableToolAction, + tool_id: &str, + accessible_connectors: &[AppInfo], +) -> bool { + accessible_connectors + .iter() + .find(|connector| connector.id == tool_id) + .is_some_and(|connector| match action_type { + DiscoverableToolAction::Install => connector.is_accessible, + DiscoverableToolAction::Enable => connector.is_accessible && connector.is_enabled, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + + #[test] + fn build_tool_suggestion_elicitation_request_uses_expected_shape() { + let args = ToolSuggestArgs { + tool_type: DiscoverableToolType::Connector, + action_type: DiscoverableToolAction::Install, + tool_id: "connector_2128aebfecb84f64a069897515042a44".to_string(), + suggest_reason: "Plan and reference events from your calendar".to_string(), + }; + let connector = AppInfo { + id: "connector_2128aebfecb84f64a069897515042a44".to_string(), + name: "Google Calendar".to_string(), + description: Some("Plan events and schedules.".to_string()), + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + branding: None, + app_metadata: None, + labels: None, + install_url: Some( + "https://chatgpt.com/apps/google-calendar/connector_2128aebfecb84f64a069897515042a44" + .to_string(), + ), + is_accessible: false, + is_enabled: true, + plugin_display_names: Vec::new(), + }; + + let request = build_tool_suggestion_elicitation_request( + "thread-1".to_string(), + "turn-1".to_string(), + &args, + "Plan and reference events from your calendar", + &connector, + ); + + assert_eq!( + request, + McpServerElicitationRequestParams { + thread_id: "thread-1".to_string(), + turn_id: Some("turn-1".to_string()), + server_name: CODEX_APPS_MCP_SERVER_NAME.to_string(), + request: McpServerElicitationRequest::Form { + meta: Some(json!(ToolSuggestMeta { + codex_approval_kind: TOOL_SUGGEST_APPROVAL_KIND_VALUE, + tool_type: DiscoverableToolType::Connector, + suggest_type: DiscoverableToolAction::Install, + suggest_reason: "Plan and reference events from your calendar", + tool_id: "connector_2128aebfecb84f64a069897515042a44", + tool_name: "Google Calendar", + install_url: "https://chatgpt.com/apps/google-calendar/connector_2128aebfecb84f64a069897515042a44", + })), + message: "Google Calendar could help with this request.\n\nPlan and reference events from your calendar\n\nOpen ChatGPT to install it, then confirm here if you finish.".to_string(), + requested_schema: McpElicitationSchema { + schema_uri: None, + type_: McpElicitationObjectType::Object, + properties: BTreeMap::new(), + required: None, + }, + }, + } + ); + } + + #[test] + fn build_tool_suggestion_meta_uses_expected_shape() { + let meta = build_tool_suggestion_meta( + DiscoverableToolType::Connector, + DiscoverableToolAction::Install, + "Find and reference emails from your inbox", + "connector_68df038e0ba48191908c8434991bbac2", + "Gmail", + "https://chatgpt.com/apps/gmail/connector_68df038e0ba48191908c8434991bbac2", + ); + + assert_eq!( + meta, + ToolSuggestMeta { + codex_approval_kind: TOOL_SUGGEST_APPROVAL_KIND_VALUE, + tool_type: DiscoverableToolType::Connector, + suggest_type: DiscoverableToolAction::Install, + suggest_reason: "Find and reference emails from your inbox", + tool_id: "connector_68df038e0ba48191908c8434991bbac2", + tool_name: "Gmail", + install_url: "https://chatgpt.com/apps/gmail/connector_68df038e0ba48191908c8434991bbac2", + } + ); + } + + #[test] + fn verified_connector_suggestion_completed_requires_installed_connector() { + let accessible_connectors = vec![AppInfo { + id: "calendar".to_string(), + name: "Google Calendar".to_string(), + description: None, + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + branding: None, + app_metadata: None, + labels: None, + install_url: None, + is_accessible: true, + is_enabled: true, + plugin_display_names: Vec::new(), + }]; + + assert!(verified_connector_suggestion_completed( + DiscoverableToolAction::Install, + "calendar", + &accessible_connectors, + )); + assert!(!verified_connector_suggestion_completed( + DiscoverableToolAction::Install, + "gmail", + &accessible_connectors, + )); + } + + #[test] + fn verified_connector_suggestion_completed_requires_enabled_connector_for_enable() { + let accessible_connectors = vec![ + AppInfo { + id: "calendar".to_string(), + name: "Google Calendar".to_string(), + description: None, + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + branding: None, + app_metadata: None, + labels: None, + install_url: None, + is_accessible: true, + is_enabled: false, + plugin_display_names: Vec::new(), + }, + AppInfo { + id: "gmail".to_string(), + name: "Gmail".to_string(), + description: None, + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + branding: None, + app_metadata: None, + labels: None, + install_url: None, + is_accessible: true, + is_enabled: true, + plugin_display_names: Vec::new(), + }, + ]; + + assert!(!verified_connector_suggestion_completed( + DiscoverableToolAction::Enable, + "calendar", + &accessible_connectors, + )); + assert!(verified_connector_suggestion_completed( + DiscoverableToolAction::Enable, + "gmail", + &accessible_connectors, + )); + } +} diff --git a/codex-rs/core/src/tools/js_repl/mod.rs b/codex-rs/core/src/tools/js_repl/mod.rs index 6fe78a728b..1fb3d528fe 100644 --- a/codex-rs/core/src/tools/js_repl/mod.rs +++ b/codex-rs/core/src/tools/js_repl/mod.rs @@ -1360,14 +1360,17 @@ impl JsReplManager { let router = ToolRouter::from_config( &exec.turn.tools_config, - Some( - mcp_tools - .into_iter() - .map(|(name, tool)| (name, tool.tool)) - .collect(), - ), - None, - exec.turn.dynamic_tools.as_slice(), + crate::tools::router::ToolRouterParams { + mcp_tools: Some( + mcp_tools + .into_iter() + .map(|(name, tool)| (name, tool.tool)) + .collect(), + ), + app_tools: None, + discoverable_tools: None, + dynamic_tools: exec.turn.dynamic_tools.as_slice(), + }, ); let payload = if let Some((server, tool)) = exec diff --git a/codex-rs/core/src/tools/mod.rs b/codex-rs/core/src/tools/mod.rs index 20808325b2..4e495190ec 100644 --- a/codex-rs/core/src/tools/mod.rs +++ b/codex-rs/core/src/tools/mod.rs @@ -1,6 +1,7 @@ pub mod code_mode; pub(crate) mod code_mode_description; pub mod context; +pub(crate) mod discoverable; pub mod events; pub(crate) mod handlers; pub mod js_repl; diff --git a/codex-rs/core/src/tools/router.rs b/codex-rs/core/src/tools/router.rs index 4482c34bb4..d311d00702 100644 --- a/codex-rs/core/src/tools/router.rs +++ b/codex-rs/core/src/tools/router.rs @@ -9,11 +9,12 @@ use crate::tools::context::SharedTurnDiffTracker; use crate::tools::context::ToolInvocation; use crate::tools::context::ToolPayload; use crate::tools::context::ToolSearchOutput; +use crate::tools::discoverable::DiscoverableTool; use crate::tools::registry::AnyToolResult; use crate::tools::registry::ConfiguredToolSpec; use crate::tools::registry::ToolRegistry; use crate::tools::spec::ToolsConfig; -use crate::tools::spec::build_specs; +use crate::tools::spec::build_specs_with_discoverable_tools; use codex_protocol::dynamic_tools::DynamicToolSpec; use codex_protocol::models::LocalShellAction; use codex_protocol::models::ResponseInputItem; @@ -40,14 +41,28 @@ pub struct ToolRouter { specs: Vec, } +pub(crate) struct ToolRouterParams<'a> { + pub(crate) mcp_tools: Option>, + pub(crate) app_tools: Option>, + pub(crate) discoverable_tools: Option>, + pub(crate) dynamic_tools: &'a [DynamicToolSpec], +} + impl ToolRouter { - pub fn from_config( - config: &ToolsConfig, - mcp_tools: Option>, - app_tools: Option>, - dynamic_tools: &[DynamicToolSpec], - ) -> Self { - let builder = build_specs(config, mcp_tools, app_tools, dynamic_tools); + pub fn from_config(config: &ToolsConfig, params: ToolRouterParams<'_>) -> Self { + let ToolRouterParams { + mcp_tools, + app_tools, + discoverable_tools, + dynamic_tools, + } = params; + let builder = build_specs_with_discoverable_tools( + config, + mcp_tools, + app_tools, + discoverable_tools, + dynamic_tools, + ); let (specs, registry) = builder.build(); Self { registry, specs } @@ -287,6 +302,7 @@ mod tests { use super::ToolCall; use super::ToolCallSource; use super::ToolRouter; + use super::ToolRouterParams; #[tokio::test] async fn js_repl_tools_only_blocks_direct_tool_calls() -> anyhow::Result<()> { @@ -305,14 +321,17 @@ mod tests { let app_tools = Some(mcp_tools.clone()); let router = ToolRouter::from_config( &turn.tools_config, - Some( - mcp_tools - .into_iter() - .map(|(name, tool)| (name, tool.tool)) - .collect(), - ), - app_tools, - turn.dynamic_tools.as_slice(), + ToolRouterParams { + mcp_tools: Some( + mcp_tools + .into_iter() + .map(|(name, tool)| (name, tool.tool)) + .collect(), + ), + app_tools, + discoverable_tools: None, + dynamic_tools: turn.dynamic_tools.as_slice(), + }, ); let call = ToolCall { @@ -359,14 +378,17 @@ mod tests { let app_tools = Some(mcp_tools.clone()); let router = ToolRouter::from_config( &turn.tools_config, - Some( - mcp_tools - .into_iter() - .map(|(name, tool)| (name, tool.tool)) - .collect(), - ), - app_tools, - turn.dynamic_tools.as_slice(), + ToolRouterParams { + mcp_tools: Some( + mcp_tools + .into_iter() + .map(|(name, tool)| (name, tool.tool)) + .collect(), + ), + app_tools, + discoverable_tools: None, + dynamic_tools: turn.dynamic_tools.as_slice(), + }, ); let call = ToolCall { diff --git a/codex-rs/core/src/tools/spec.rs b/codex-rs/core/src/tools/spec.rs index 748333ea9a..3364d19e84 100644 --- a/codex-rs/core/src/tools/spec.rs +++ b/codex-rs/core/src/tools/spec.rs @@ -10,9 +10,14 @@ use crate::models_manager::collaboration_mode_presets::CollaborationModesConfig; use crate::original_image_detail::can_request_original_image_detail; use crate::tools::code_mode::PUBLIC_TOOL_NAME; use crate::tools::code_mode_description::augment_tool_spec_for_code_mode; +use crate::tools::discoverable::DiscoverablePluginInfo; +use crate::tools::discoverable::DiscoverableTool; +use crate::tools::discoverable::DiscoverableToolAction; +use crate::tools::discoverable::DiscoverableToolType; use crate::tools::handlers::PLAN_TOOL; use crate::tools::handlers::TOOL_SEARCH_DEFAULT_LIMIT; use crate::tools::handlers::TOOL_SEARCH_TOOL_NAME; +use crate::tools::handlers::TOOL_SUGGEST_TOOL_NAME; use crate::tools::handlers::agent_jobs::BatchJobHandler; use crate::tools::handlers::apply_patch::create_apply_patch_freeform_tool; use crate::tools::handlers::apply_patch::create_apply_patch_json_tool; @@ -44,6 +49,8 @@ use std::collections::HashMap; const TOOL_SEARCH_DESCRIPTION_TEMPLATE: &str = include_str!("../../templates/search_tool/tool_description.md"); +const TOOL_SUGGEST_DESCRIPTION_TEMPLATE: &str = + include_str!("../../templates/search_tool/tool_suggest_description.md"); const WEB_SEARCH_CONTENT_TYPES: [&str; 2] = ["text", "image"]; fn unified_exec_output_schema() -> JsonValue { @@ -105,6 +112,7 @@ pub(crate) struct ToolsConfig { pub image_gen_tool: bool, pub agent_roles: BTreeMap, pub search_tool: bool, + pub tool_suggest: bool, pub request_permission_enabled: bool, pub request_permissions_tool_enabled: bool, pub code_mode_enabled: bool, @@ -148,6 +156,7 @@ impl ToolsConfig { let include_default_mode_request_user_input = include_request_user_input && features.enabled(Feature::DefaultModeRequestUserInput); let include_search_tool = features.enabled(Feature::Apps); + let include_tool_suggest = include_search_tool && features.enabled(Feature::ToolSuggest); let include_original_image_detail = can_request_original_image_detail(features, model_info); let include_artifact_tools = features.enabled(Feature::Artifact) && codex_artifacts::can_manage_artifact_runtime(); @@ -215,6 +224,7 @@ impl ToolsConfig { image_gen_tool: include_image_gen_tool, agent_roles: BTreeMap::new(), search_tool: include_search_tool, + tool_suggest: include_tool_suggest, request_permission_enabled, request_permissions_tool_enabled, code_mode_enabled: include_code_mode, @@ -1451,6 +1461,133 @@ fn create_tool_search_tool(app_tools: &HashMap) -> ToolSpec { } } +fn create_tool_suggest_tool(discoverable_tools: &[DiscoverableTool]) -> ToolSpec { + let discoverable_tool_ids = discoverable_tools + .iter() + .map(DiscoverableTool::id) + .collect::>() + .join(", "); + let properties = BTreeMap::from([ + ( + "tool_type".to_string(), + JsonSchema::String { + description: Some( + "Type of discoverable tool to suggest. Use \"connector\" or \"plugin\"." + .to_string(), + ), + }, + ), + ( + "action_type".to_string(), + JsonSchema::String { + description: Some( + "Suggested action for the tool. Use \"install\" or \"enable\".".to_string(), + ), + }, + ), + ( + "tool_id".to_string(), + JsonSchema::String { + description: Some(format!( + "Connector or plugin id to suggest. Must be one of: {discoverable_tool_ids}." + )), + }, + ), + ( + "suggest_reason".to_string(), + JsonSchema::String { + description: Some( + "Concise one-line user-facing reason why this tool can help with the current request." + .to_string(), + ), + }, + ), + ]); + let description = TOOL_SUGGEST_DESCRIPTION_TEMPLATE.replace( + "{{discoverable_tools}}", + format_discoverable_tools(discoverable_tools).as_str(), + ); + + ToolSpec::Function(ResponsesApiTool { + name: TOOL_SUGGEST_TOOL_NAME.to_string(), + description, + strict: false, + defer_loading: None, + parameters: JsonSchema::Object { + properties, + required: Some(vec![ + "tool_type".to_string(), + "action_type".to_string(), + "tool_id".to_string(), + "suggest_reason".to_string(), + ]), + additional_properties: Some(false.into()), + }, + output_schema: None, + }) +} + +fn format_discoverable_tools(discoverable_tools: &[DiscoverableTool]) -> String { + let mut discoverable_tools = discoverable_tools.to_vec(); + discoverable_tools.sort_by(|left, right| { + left.name() + .cmp(right.name()) + .then_with(|| left.id().cmp(right.id())) + }); + + discoverable_tools + .into_iter() + .map(|tool| { + let description = tool + .description() + .filter(|description| !description.trim().is_empty()) + .map(ToString::to_string) + .unwrap_or_else(|| match &tool { + DiscoverableTool::Connector(_) => "No description provided.".to_string(), + DiscoverableTool::Plugin(plugin) => format_plugin_summary(plugin.as_ref()), + }); + let default_action = match tool.tool_type() { + DiscoverableToolType::Connector => DiscoverableToolAction::Install, + DiscoverableToolType::Plugin => DiscoverableToolAction::Enable, + }; + format!( + "- {} (id: `{}`, type: {}, action: {}): {}", + tool.name(), + tool.id(), + tool.tool_type().as_str(), + default_action.as_str(), + description + ) + }) + .collect::>() + .join("\n") +} + +fn format_plugin_summary(plugin: &DiscoverablePluginInfo) -> String { + let mut details = Vec::new(); + if plugin.has_skills { + details.push("skills".to_string()); + } + if !plugin.mcp_server_names.is_empty() { + details.push(format!( + "MCP servers: {}", + plugin.mcp_server_names.join(", ") + )); + } + if !plugin.app_connector_ids.is_empty() { + details.push(format!( + "app connectors: {}", + plugin.app_connector_ids.join(", ") + )); + } + + if details.is_empty() { + "No description provided.".to_string() + } else { + details.join("; ") + } +} + fn create_read_file_tool() -> ToolSpec { let indentation_properties = BTreeMap::from([ ( @@ -2083,11 +2220,22 @@ fn sanitize_json_schema(value: &mut JsonValue) { } /// Builds the tool registry builder while collecting tool specs for later serialization. +#[cfg(test)] pub(crate) fn build_specs( config: &ToolsConfig, mcp_tools: Option>, app_tools: Option>, dynamic_tools: &[DynamicToolSpec], +) -> ToolRegistryBuilder { + build_specs_with_discoverable_tools(config, mcp_tools, app_tools, None, dynamic_tools) +} + +pub(crate) fn build_specs_with_discoverable_tools( + config: &ToolsConfig, + mcp_tools: Option>, + app_tools: Option>, + discoverable_tools: Option>, + dynamic_tools: &[DynamicToolSpec], ) -> ToolRegistryBuilder { use crate::tools::handlers::ApplyPatchHandler; use crate::tools::handlers::ArtifactsHandler; @@ -2108,6 +2256,7 @@ pub(crate) fn build_specs( use crate::tools::handlers::ShellHandler; use crate::tools::handlers::TestSyncHandler; use crate::tools::handlers::ToolSearchHandler; + use crate::tools::handlers::ToolSuggestHandler; use crate::tools::handlers::UnifiedExecHandler; use crate::tools::handlers::ViewImageHandler; use std::sync::Arc; @@ -2127,6 +2276,7 @@ pub(crate) fn build_specs( let request_user_input_handler = Arc::new(RequestUserInputHandler { default_mode_request_user_input: config.default_mode_request_user_input, }); + let tool_suggest_handler = Arc::new(ToolSuggestHandler); let code_mode_handler = Arc::new(CodeModeHandler); let js_repl_handler = Arc::new(JsReplHandler); let js_repl_reset_handler = Arc::new(JsReplResetHandler); @@ -2135,10 +2285,11 @@ pub(crate) fn build_specs( if config.code_mode_enabled { let nested_config = config.for_code_mode_nested_tools(); - let (nested_specs, _) = build_specs( + let (nested_specs, _) = build_specs_with_discoverable_tools( &nested_config, mcp_tools.clone(), app_tools.clone(), + None, dynamic_tools, ) .build(); @@ -2304,6 +2455,15 @@ pub(crate) fn build_specs( } } + if config.tool_suggest + && let Some(discoverable_tools) = discoverable_tools + .as_ref() + .filter(|tools| !tools.is_empty()) + { + builder.push_spec_with_parallel_support(create_tool_suggest_tool(discoverable_tools), true); + builder.register_handler(TOOL_SUGGEST_TOOL_NAME, tool_suggest_handler); + } + if let Some(apply_patch_tool_type) = &config.apply_patch_tool_type { match apply_patch_tool_type { ApplyPatchToolType::Freeform => { @@ -2565,6 +2725,7 @@ mod tests { use crate::models_manager::manager::ModelsManager; use crate::models_manager::model_info::with_config_overrides; use crate::tools::registry::ConfiguredToolSpec; + use codex_app_server_protocol::AppInfo; use codex_protocol::openai_models::InputModality; use codex_protocol::openai_models::ModelInfo; use codex_protocol::openai_models::ModelsResponse; @@ -2590,6 +2751,25 @@ mod tests { } } + fn discoverable_connector(id: &str, name: &str, description: &str) -> DiscoverableTool { + let slug = name.replace(' ', "-").to_lowercase(); + DiscoverableTool::Connector(Box::new(AppInfo { + id: id.to_string(), + name: name.to_string(), + description: Some(description.to_string()), + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + branding: None, + app_metadata: None, + labels: None, + install_url: Some(format!("https://chatgpt.com/apps/{slug}/{id}")), + is_accessible: false, + is_enabled: true, + plugin_display_names: Vec::new(), + })) + } + #[test] fn mcp_tool_to_openai_tool_inserts_empty_properties() { let mut schema = rmcp::model::JsonObject::new(); @@ -4147,7 +4327,6 @@ mod tests { }); let (tools, _) = build_specs(&tools_config, None, app_tools.clone(), &[]).build(); assert_lacks_tool_name(&tools, TOOL_SEARCH_TOOL_NAME); - let mut features = Features::with_defaults(); features.enable(Feature::Apps); let available_models = Vec::new(); @@ -4162,6 +4341,41 @@ mod tests { assert_contains_tool_names(&tools, &[TOOL_SEARCH_TOOL_NAME]); } + #[test] + fn tool_suggest_is_not_registered_without_feature_flag() { + let config = test_config(); + let model_info = + ModelsManager::construct_model_info_offline_for_tests("gpt-5-codex", &config); + let mut features = Features::with_defaults(); + features.enable(Feature::Apps); + let available_models = Vec::new(); + let tools_config = ToolsConfig::new(&ToolsConfigParams { + model_info: &model_info, + available_models: &available_models, + features: &features, + web_search_mode: Some(WebSearchMode::Cached), + session_source: SessionSource::Cli, + }); + let (tools, _) = build_specs_with_discoverable_tools( + &tools_config, + None, + None, + Some(vec![discoverable_connector( + "connector_2128aebfecb84f64a069897515042a44", + "Google Calendar", + "Plan events and schedules.", + )]), + &[], + ) + .build(); + + assert!( + !tools + .iter() + .any(|tool| tool_name(&tool.spec) == TOOL_SUGGEST_TOOL_NAME) + ); + } + #[test] fn search_tool_description_handles_no_enabled_apps() { let config = test_config(); @@ -4253,6 +4467,89 @@ mod tests { assert!(registry.has_handler(alias.as_str(), None)); } + #[test] + fn tool_suggest_description_lists_discoverable_tools() { + let config = test_config(); + let model_info = + ModelsManager::construct_model_info_offline_for_tests("gpt-5-codex", &config); + let mut features = Features::with_defaults(); + features.enable(Feature::Apps); + features.enable(Feature::ToolSuggest); + let available_models = Vec::new(); + let tools_config = ToolsConfig::new(&ToolsConfigParams { + model_info: &model_info, + available_models: &available_models, + features: &features, + web_search_mode: Some(WebSearchMode::Cached), + session_source: SessionSource::Cli, + }); + + let discoverable_tools = vec![ + discoverable_connector( + "connector_2128aebfecb84f64a069897515042a44", + "Google Calendar", + "Plan events and schedules.", + ), + discoverable_connector( + "connector_68df038e0ba48191908c8434991bbac2", + "Gmail", + "Find and summarize email threads.", + ), + DiscoverableTool::Plugin(Box::new(DiscoverablePluginInfo { + id: "sample@test".to_string(), + name: "Sample Plugin".to_string(), + description: None, + has_skills: true, + mcp_server_names: vec!["sample-docs".to_string()], + app_connector_ids: vec!["connector_sample".to_string()], + })), + ]; + + let (tools, _) = build_specs_with_discoverable_tools( + &tools_config, + None, + None, + Some(discoverable_tools), + &[], + ) + .build(); + + let tool_suggest = find_tool(&tools, TOOL_SUGGEST_TOOL_NAME); + let ToolSpec::Function(ResponsesApiTool { + description, + parameters, + .. + }) = &tool_suggest.spec + else { + panic!("expected function tool"); + }; + assert!(description.contains("Google Calendar")); + assert!(description.contains("Gmail")); + assert!(description.contains("Sample Plugin")); + assert!(description.contains("Plan events and schedules.")); + assert!(description.contains("Find and summarize email threads.")); + assert!(description.contains("id: `sample@test`, type: plugin, action: enable")); + assert!( + description + .contains("skills; MCP servers: sample-docs; app connectors: connector_sample") + ); + assert!( + description.contains("DO NOT explore or recommend tools that are not on this list.") + ); + let JsonSchema::Object { required, .. } = parameters else { + panic!("expected object parameters"); + }; + assert_eq!( + required.as_ref(), + Some(&vec![ + "tool_type".to_string(), + "action_type".to_string(), + "tool_id".to_string(), + "suggest_reason".to_string(), + ]) + ); + } + #[test] fn test_mcp_tool_property_missing_type_defaults_to_string() { let config = test_config(); diff --git a/codex-rs/core/templates/search_tool/tool_suggest_description.md b/codex-rs/core/templates/search_tool/tool_suggest_description.md new file mode 100644 index 0000000000..fcf599c397 --- /dev/null +++ b/codex-rs/core/templates/search_tool/tool_suggest_description.md @@ -0,0 +1,25 @@ +# Tool suggestion discovery + +Suggests a discoverable connector or plugin when the user clearly wants a capability that is not currently available in the active `tools` list. + +Use this ONLY when: +- There's no available tool to handle the user's request +- And tool_search fails to find a good match +- AND the user's request strongly matches one of the discoverable tools listed below. + +Tool suggestions should only use the discoverable tools listed here. DO NOT explore or recommend tools that are not on this list. + +Discoverable tools: +{{discoverable_tools}} + +Workflow: + +1. Match the user's request against the discoverable tools list above. +2. If one tool clearly fits, call `tool_suggest` with: + - `tool_type`: `connector` or `plugin` + - `action_type`: `install` or `enable` + - `tool_id`: exact id from the discoverable tools list above + - `suggest_reason`: concise one-line user-facing reason this tool can help with the current request +3. After the suggestion flow completes: + - if the user finished the install or enable flow, continue by searching again or using the newly available tool + - if the user did not finish, continue without that tool, and don't suggest that tool again unless the user explicitly asks you to. diff --git a/codex-rs/core/tests/common/apps_test_server.rs b/codex-rs/core/tests/common/apps_test_server.rs index c8ef0bd1c3..83ce020bef 100644 --- a/codex-rs/core/tests/common/apps_test_server.rs +++ b/codex-rs/core/tests/common/apps_test_server.rs @@ -12,6 +12,8 @@ use wiremock::matchers::path_regex; const CONNECTOR_ID: &str = "calendar"; const CONNECTOR_NAME: &str = "Calendar"; +const DISCOVERABLE_CALENDAR_ID: &str = "connector_2128aebfecb84f64a069897515042a44"; +const DISCOVERABLE_GMAIL_ID: &str = "connector_68df038e0ba48191908c8434991bbac2"; const CONNECTOR_DESCRIPTION: &str = "Plan events and manage your calendar."; const PROTOCOL_VERSION: &str = "2025-11-25"; const SERVER_NAME: &str = "codex-apps-test"; @@ -32,6 +34,7 @@ impl AppsTestServer { connector_name: &str, ) -> Result { mount_oauth_metadata(server).await; + mount_connectors_directory(server).await; mount_streamable_http_json_rpc( server, connector_name.to_string(), @@ -56,6 +59,37 @@ async fn mount_oauth_metadata(server: &MockServer) { .await; } +async fn mount_connectors_directory(server: &MockServer) { + Mock::given(method("GET")) + .and(path("/connectors/directory/list")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "apps": [ + { + "id": DISCOVERABLE_CALENDAR_ID, + "name": "Google Calendar", + "description": "Plan events and schedules.", + }, + { + "id": DISCOVERABLE_GMAIL_ID, + "name": "Gmail", + "description": "Find and summarize email threads.", + } + ], + "nextToken": null + }))) + .mount(server) + .await; + + Mock::given(method("GET")) + .and(path("/connectors/directory/list_workspace")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "apps": [], + "nextToken": null + }))) + .mount(server) + .await; +} + async fn mount_streamable_http_json_rpc( server: &MockServer, connector_name: String, diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index 3ad7c579cd..87b2dc795c 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -2422,6 +2422,9 @@ impl App { url, is_installed, is_enabled, + suggest_reason: None, + suggestion_type: None, + elicitation_target: None, }); } AppEvent::OpenUrlInBrowser { url } => { diff --git a/codex-rs/tui/src/bottom_pane/app_link_view.rs b/codex-rs/tui/src/bottom_pane/app_link_view.rs index 2e6fab3d24..15596a9d7f 100644 --- a/codex-rs/tui/src/bottom_pane/app_link_view.rs +++ b/codex-rs/tui/src/bottom_pane/app_link_view.rs @@ -1,3 +1,7 @@ +use codex_protocol::ThreadId; +use codex_protocol::approvals::ElicitationAction; +use codex_protocol::mcp::RequestId as McpRequestId; +use codex_protocol::protocol::Op; use crossterm::event::KeyCode; use crossterm::event::KeyEvent; use crossterm::event::KeyModifiers; @@ -34,6 +38,19 @@ enum AppLinkScreen { InstallConfirmation, } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum AppLinkSuggestionType { + Install, + Enable, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct AppLinkElicitationTarget { + pub(crate) thread_id: ThreadId, + pub(crate) server_name: String, + pub(crate) request_id: McpRequestId, +} + pub(crate) struct AppLinkViewParams { pub(crate) app_id: String, pub(crate) title: String, @@ -42,6 +59,9 @@ pub(crate) struct AppLinkViewParams { pub(crate) url: String, pub(crate) is_installed: bool, pub(crate) is_enabled: bool, + pub(crate) suggest_reason: Option, + pub(crate) suggestion_type: Option, + pub(crate) elicitation_target: Option, } pub(crate) struct AppLinkView { @@ -52,6 +72,9 @@ pub(crate) struct AppLinkView { url: String, is_installed: bool, is_enabled: bool, + suggest_reason: Option, + suggestion_type: Option, + elicitation_target: Option, app_event_tx: AppEventSender, screen: AppLinkScreen, selected_action: usize, @@ -68,6 +91,9 @@ impl AppLinkView { url, is_installed, is_enabled, + suggest_reason, + suggestion_type, + elicitation_target, } = params; Self { app_id, @@ -77,6 +103,9 @@ impl AppLinkView { url, is_installed, is_enabled, + suggest_reason, + suggestion_type, + elicitation_target, app_event_tx, screen: AppLinkScreen::Link, selected_action: 0, @@ -113,6 +142,31 @@ impl AppLinkView { self.selected_action = (self.selected_action + 1).min(self.action_labels().len() - 1); } + fn is_tool_suggestion(&self) -> bool { + self.elicitation_target.is_some() + } + + fn resolve_elicitation(&self, decision: ElicitationAction) { + let Some(target) = self.elicitation_target.as_ref() else { + return; + }; + self.app_event_tx.send(AppEvent::SubmitThreadOp { + thread_id: target.thread_id, + op: Op::ResolveElicitation { + server_name: target.server_name.clone(), + request_id: target.request_id.clone(), + decision, + content: None, + meta: None, + }, + }); + } + + fn decline_tool_suggestion(&mut self) { + self.resolve_elicitation(ElicitationAction::Decline); + self.complete = true; + } + fn open_chatgpt_link(&mut self) { self.app_event_tx.send(AppEvent::OpenUrlInBrowser { url: self.url.clone(), @@ -127,6 +181,9 @@ impl AppLinkView { self.app_event_tx.send(AppEvent::RefreshConnectors { force_refetch: true, }); + if self.is_tool_suggestion() { + self.resolve_elicitation(ElicitationAction::Accept); + } self.complete = true; } @@ -141,9 +198,40 @@ impl AppLinkView { id: self.app_id.clone(), enabled: self.is_enabled, }); + if self.is_tool_suggestion() { + self.resolve_elicitation(ElicitationAction::Accept); + self.complete = true; + } } fn activate_selected_action(&mut self) { + if self.is_tool_suggestion() { + match self.suggestion_type { + Some(AppLinkSuggestionType::Enable) => match self.screen { + AppLinkScreen::Link => match self.selected_action { + 0 => self.open_chatgpt_link(), + 1 if self.is_installed => self.toggle_enabled(), + _ => self.decline_tool_suggestion(), + }, + AppLinkScreen::InstallConfirmation => match self.selected_action { + 0 => self.refresh_connectors_and_close(), + _ => self.decline_tool_suggestion(), + }, + }, + Some(AppLinkSuggestionType::Install) | None => match self.screen { + AppLinkScreen::Link => match self.selected_action { + 0 => self.open_chatgpt_link(), + _ => self.decline_tool_suggestion(), + }, + AppLinkScreen::InstallConfirmation => match self.selected_action { + 0 => self.refresh_connectors_and_close(), + _ => self.decline_tool_suggestion(), + }, + }, + } + return; + } + match self.screen { AppLinkScreen::Link => match self.selected_action { 0 => self.open_chatgpt_link(), @@ -181,6 +269,17 @@ impl AppLinkView { } lines.push(Line::from("")); + if let Some(suggest_reason) = self + .suggest_reason + .as_deref() + .map(str::trim) + .filter(|suggest_reason| !suggest_reason.is_empty()) + { + for line in wrap(suggest_reason, usable_width) { + lines.push(Line::from(line.into_owned().italic())); + } + lines.push(Line::from("")); + } if self.is_installed { for line in wrap("Use $ to insert this app into the prompt.", usable_width) { lines.push(Line::from(line.into_owned())); @@ -366,6 +465,9 @@ impl BottomPaneView for AppLinkView { } fn on_ctrl_c(&mut self) -> CancellationEvent { + if self.is_tool_suggestion() { + self.resolve_elicitation(ElicitationAction::Decline); + } self.complete = true; CancellationEvent::Handled } @@ -447,8 +549,40 @@ mod tests { use super::*; use crate::app_event::AppEvent; use crate::render::renderable::Renderable; + use insta::assert_snapshot; use tokio::sync::mpsc::unbounded_channel; + fn suggestion_target() -> AppLinkElicitationTarget { + AppLinkElicitationTarget { + thread_id: ThreadId::try_from("00000000-0000-0000-0000-000000000001") + .expect("valid thread id"), + server_name: "codex_apps".to_string(), + request_id: McpRequestId::String("request-1".to_string()), + } + } + + fn render_snapshot(view: &AppLinkView, area: Rect) -> String { + let mut buf = Buffer::empty(area); + view.render(area, &mut buf); + (0..area.height) + .map(|y| { + (0..area.width) + .map(|x| { + let symbol = buf[(x, y)].symbol(); + if symbol.is_empty() { + ' ' + } else { + symbol.chars().next().unwrap_or(' ') + } + }) + .collect::() + .trim_end() + .to_string() + }) + .collect::>() + .join("\n") + } + #[test] fn installed_app_has_toggle_action() { let (tx_raw, _rx) = unbounded_channel::(); @@ -462,6 +596,9 @@ mod tests { url: "https://example.test/notion".to_string(), is_installed: true, is_enabled: true, + suggest_reason: None, + suggestion_type: None, + elicitation_target: None, }, tx, ); @@ -485,6 +622,9 @@ mod tests { url: "https://example.test/notion".to_string(), is_installed: true, is_enabled: true, + suggest_reason: None, + suggestion_type: None, + elicitation_target: None, }, tx, ); @@ -521,6 +661,9 @@ mod tests { url: url_like.to_string(), is_installed: true, is_enabled: true, + suggest_reason: None, + suggestion_type: None, + elicitation_target: None, }, tx, ); @@ -561,6 +704,9 @@ mod tests { url: url.to_string(), is_installed: true, is_enabled: true, + suggest_reason: None, + suggestion_type: None, + elicitation_target: None, }, tx, ); @@ -593,4 +739,206 @@ mod tests { "expected wrapped setup URL tail to remain visible in narrow pane, got:\n{rendered_blob}" ); } + + #[test] + fn install_tool_suggestion_resolves_elicitation_after_confirmation() { + let (tx_raw, mut rx) = unbounded_channel::(); + let tx = AppEventSender::new(tx_raw); + let mut view = AppLinkView::new( + AppLinkViewParams { + app_id: "connector_google_calendar".to_string(), + title: "Google Calendar".to_string(), + description: Some("Plan events and schedules.".to_string()), + instructions: "Install this app in your browser, then return here.".to_string(), + url: "https://example.test/google-calendar".to_string(), + is_installed: false, + is_enabled: false, + suggest_reason: Some("Plan and reference events from your calendar".to_string()), + suggestion_type: Some(AppLinkSuggestionType::Install), + elicitation_target: Some(suggestion_target()), + }, + tx, + ); + + view.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); + match rx.try_recv() { + Ok(AppEvent::OpenUrlInBrowser { url }) => { + assert_eq!(url, "https://example.test/google-calendar".to_string()); + } + Ok(other) => panic!("unexpected app event: {other:?}"), + Err(err) => panic!("missing app event: {err}"), + } + assert_eq!(view.screen, AppLinkScreen::InstallConfirmation); + + view.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); + match rx.try_recv() { + Ok(AppEvent::RefreshConnectors { force_refetch }) => { + assert!(force_refetch); + } + Ok(other) => panic!("unexpected app event: {other:?}"), + Err(err) => panic!("missing app event: {err}"), + } + match rx.try_recv() { + Ok(AppEvent::SubmitThreadOp { thread_id, op }) => { + assert_eq!(thread_id, suggestion_target().thread_id); + assert_eq!( + op, + Op::ResolveElicitation { + server_name: "codex_apps".to_string(), + request_id: McpRequestId::String("request-1".to_string()), + decision: ElicitationAction::Accept, + content: None, + meta: None, + } + ); + } + Ok(other) => panic!("unexpected app event: {other:?}"), + Err(err) => panic!("missing app event: {err}"), + } + assert!(view.is_complete()); + } + + #[test] + fn declined_tool_suggestion_resolves_elicitation_decline() { + let (tx_raw, mut rx) = unbounded_channel::(); + let tx = AppEventSender::new(tx_raw); + let mut view = AppLinkView::new( + AppLinkViewParams { + app_id: "connector_google_calendar".to_string(), + title: "Google Calendar".to_string(), + description: None, + instructions: "Install this app in your browser, then return here.".to_string(), + url: "https://example.test/google-calendar".to_string(), + is_installed: false, + is_enabled: false, + suggest_reason: Some("Plan and reference events from your calendar".to_string()), + suggestion_type: Some(AppLinkSuggestionType::Install), + elicitation_target: Some(suggestion_target()), + }, + tx, + ); + + view.handle_key_event(KeyEvent::new(KeyCode::Char('2'), KeyModifiers::NONE)); + + match rx.try_recv() { + Ok(AppEvent::SubmitThreadOp { thread_id, op }) => { + assert_eq!(thread_id, suggestion_target().thread_id); + assert_eq!( + op, + Op::ResolveElicitation { + server_name: "codex_apps".to_string(), + request_id: McpRequestId::String("request-1".to_string()), + decision: ElicitationAction::Decline, + content: None, + meta: None, + } + ); + } + Ok(other) => panic!("unexpected app event: {other:?}"), + Err(err) => panic!("missing app event: {err}"), + } + assert!(view.is_complete()); + } + + #[test] + fn enable_tool_suggestion_resolves_elicitation_after_enable() { + let (tx_raw, mut rx) = unbounded_channel::(); + let tx = AppEventSender::new(tx_raw); + let mut view = AppLinkView::new( + AppLinkViewParams { + app_id: "connector_google_calendar".to_string(), + title: "Google Calendar".to_string(), + description: Some("Plan events and schedules.".to_string()), + instructions: "Enable this app to use it for the current request.".to_string(), + url: "https://example.test/google-calendar".to_string(), + is_installed: true, + is_enabled: false, + suggest_reason: Some("Plan and reference events from your calendar".to_string()), + suggestion_type: Some(AppLinkSuggestionType::Enable), + elicitation_target: Some(suggestion_target()), + }, + tx, + ); + + view.handle_key_event(KeyEvent::new(KeyCode::Char('2'), KeyModifiers::NONE)); + + match rx.try_recv() { + Ok(AppEvent::SetAppEnabled { id, enabled }) => { + assert_eq!(id, "connector_google_calendar"); + assert!(enabled); + } + Ok(other) => panic!("unexpected app event: {other:?}"), + Err(err) => panic!("missing app event: {err}"), + } + match rx.try_recv() { + Ok(AppEvent::SubmitThreadOp { thread_id, op }) => { + assert_eq!(thread_id, suggestion_target().thread_id); + assert_eq!( + op, + Op::ResolveElicitation { + server_name: "codex_apps".to_string(), + request_id: McpRequestId::String("request-1".to_string()), + decision: ElicitationAction::Accept, + content: None, + meta: None, + } + ); + } + Ok(other) => panic!("unexpected app event: {other:?}"), + Err(err) => panic!("missing app event: {err}"), + } + assert!(view.is_complete()); + } + + #[test] + fn install_suggestion_with_reason_snapshot() { + let (tx_raw, _rx) = unbounded_channel::(); + let tx = AppEventSender::new(tx_raw); + let view = AppLinkView::new( + AppLinkViewParams { + app_id: "connector_google_calendar".to_string(), + title: "Google Calendar".to_string(), + description: Some("Plan events and schedules.".to_string()), + instructions: "Install this app in your browser, then return here.".to_string(), + url: "https://example.test/google-calendar".to_string(), + is_installed: false, + is_enabled: false, + suggest_reason: Some("Plan and reference events from your calendar".to_string()), + suggestion_type: Some(AppLinkSuggestionType::Install), + elicitation_target: Some(suggestion_target()), + }, + tx, + ); + + assert_snapshot!( + "app_link_view_install_suggestion_with_reason", + render_snapshot(&view, Rect::new(0, 0, 72, view.desired_height(72))) + ); + } + + #[test] + fn enable_suggestion_with_reason_snapshot() { + let (tx_raw, _rx) = unbounded_channel::(); + let tx = AppEventSender::new(tx_raw); + let view = AppLinkView::new( + AppLinkViewParams { + app_id: "connector_google_calendar".to_string(), + title: "Google Calendar".to_string(), + description: Some("Plan events and schedules.".to_string()), + instructions: "Enable this app to use it for the current request.".to_string(), + url: "https://example.test/google-calendar".to_string(), + is_installed: true, + is_enabled: false, + suggest_reason: Some("Plan and reference events from your calendar".to_string()), + suggestion_type: Some(AppLinkSuggestionType::Enable), + elicitation_target: Some(suggestion_target()), + }, + tx, + ); + + assert_snapshot!( + "app_link_view_enable_suggestion_with_reason", + render_snapshot(&view, Rect::new(0, 0, 72, view.desired_height(72))) + ); + } } diff --git a/codex-rs/tui/src/bottom_pane/mcp_server_elicitation.rs b/codex-rs/tui/src/bottom_pane/mcp_server_elicitation.rs index 43da1c0b81..4aecbea32d 100644 --- a/codex-rs/tui/src/bottom_pane/mcp_server_elicitation.rs +++ b/codex-rs/tui/src/bottom_pane/mcp_server_elicitation.rs @@ -54,9 +54,16 @@ const APPROVAL_DECLINE_VALUE: &str = "decline"; const APPROVAL_CANCEL_VALUE: &str = "cancel"; const APPROVAL_META_KIND_KEY: &str = "codex_approval_kind"; const APPROVAL_META_KIND_MCP_TOOL_CALL: &str = "mcp_tool_call"; +const APPROVAL_META_KIND_TOOL_SUGGESTION: &str = "tool_suggestion"; const APPROVAL_PERSIST_KEY: &str = "persist"; const APPROVAL_PERSIST_SESSION_VALUE: &str = "session"; const APPROVAL_PERSIST_ALWAYS_VALUE: &str = "always"; +const TOOL_TYPE_KEY: &str = "tool_type"; +const TOOL_ID_KEY: &str = "tool_id"; +const TOOL_NAME_KEY: &str = "tool_name"; +const TOOL_SUGGEST_SUGGEST_TYPE_KEY: &str = "suggest_type"; +const TOOL_SUGGEST_REASON_KEY: &str = "suggest_reason"; +const TOOL_SUGGEST_INSTALL_URL_KEY: &str = "install_url"; #[derive(Clone, PartialEq, Default)] struct ComposerDraft { @@ -117,6 +124,28 @@ enum McpServerElicitationResponseMode { ApprovalAction, } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum ToolSuggestionToolType { + Connector, + Plugin, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum ToolSuggestionType { + Install, + Enable, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct ToolSuggestionRequest { + pub(crate) tool_type: ToolSuggestionToolType, + pub(crate) suggest_type: ToolSuggestionType, + pub(crate) suggest_reason: String, + pub(crate) tool_id: String, + pub(crate) tool_name: String, + pub(crate) install_url: String, +} + #[derive(Clone, Debug, PartialEq)] pub(crate) struct McpServerElicitationFormRequest { thread_id: ThreadId, @@ -125,6 +154,7 @@ pub(crate) struct McpServerElicitationFormRequest { message: String, response_mode: McpServerElicitationResponseMode, fields: Vec, + tool_suggestion: Option, } #[derive(Default)] @@ -170,6 +200,7 @@ impl McpServerElicitationFormRequest { return None; }; + let tool_suggestion = parse_tool_suggestion_request(meta.as_ref()); let is_tool_approval = meta .as_ref() .and_then(Value::as_object) @@ -186,9 +217,11 @@ impl McpServerElicitationFormRequest { let is_tool_approval_action = is_tool_approval && (requested_schema.is_null() || is_empty_object_schema); - let (response_mode, fields) = if requested_schema.is_null() - || (is_tool_approval && is_empty_object_schema) + let (response_mode, fields) = if tool_suggestion.is_some() + && (requested_schema.is_null() || is_empty_object_schema) { + (McpServerElicitationResponseMode::FormContent, Vec::new()) + } else if requested_schema.is_null() || (is_tool_approval && is_empty_object_schema) { let mut options = vec![McpServerElicitationOption { label: "Allow".to_string(), description: Some("Run the tool and continue.".to_string()), @@ -266,8 +299,63 @@ impl McpServerElicitationFormRequest { message, response_mode, fields, + tool_suggestion, }) } + + pub(crate) fn tool_suggestion(&self) -> Option<&ToolSuggestionRequest> { + self.tool_suggestion.as_ref() + } + + pub(crate) fn thread_id(&self) -> ThreadId { + self.thread_id + } + + pub(crate) fn server_name(&self) -> &str { + self.server_name.as_str() + } + + pub(crate) fn request_id(&self) -> &McpRequestId { + &self.request_id + } +} + +fn parse_tool_suggestion_request(meta: Option<&Value>) -> Option { + let meta = meta?.as_object()?; + if meta.get(APPROVAL_META_KIND_KEY).and_then(Value::as_str) + != Some(APPROVAL_META_KIND_TOOL_SUGGESTION) + { + return None; + } + + let tool_type = match meta.get(TOOL_TYPE_KEY).and_then(Value::as_str) { + Some("connector") => ToolSuggestionToolType::Connector, + Some("plugin") => ToolSuggestionToolType::Plugin, + _ => return None, + }; + let suggest_type = match meta + .get(TOOL_SUGGEST_SUGGEST_TYPE_KEY) + .and_then(Value::as_str) + { + Some("install") => ToolSuggestionType::Install, + Some("enable") => ToolSuggestionType::Enable, + _ => return None, + }; + + Some(ToolSuggestionRequest { + tool_type, + suggest_type, + suggest_reason: meta + .get(TOOL_SUGGEST_REASON_KEY) + .and_then(Value::as_str)? + .to_string(), + tool_id: meta.get(TOOL_ID_KEY).and_then(Value::as_str)?.to_string(), + tool_name: meta.get(TOOL_NAME_KEY).and_then(Value::as_str)?.to_string(), + install_url: meta + .get(TOOL_SUGGEST_INSTALL_URL_KEY) + .and_then(Value::as_str)? + .to_string(), + }) } fn tool_approval_supports_persist_mode(meta: Option<&Value>, expected_mode: &str) -> bool { @@ -1550,6 +1638,7 @@ mod tests { default_idx: None, }, }], + tool_suggestion: None, } ); } @@ -1621,6 +1710,7 @@ mod tests { default_idx: Some(0), }, }], + tool_suggestion: None, } ); } @@ -1667,10 +1757,44 @@ mod tests { default_idx: Some(0), }, }], + tool_suggestion: None, } ); } + #[test] + fn tool_suggestion_meta_is_parsed_into_request_payload() { + let request = McpServerElicitationFormRequest::from_event( + ThreadId::default(), + form_request( + "Suggest Google Calendar", + empty_object_schema(), + Some(serde_json::json!({ + "codex_approval_kind": "tool_suggestion", + "tool_type": "connector", + "suggest_type": "install", + "suggest_reason": "Plan and reference events from your calendar", + "tool_id": "connector_2128aebfecb84f64a069897515042a44", + "tool_name": "Google Calendar", + "install_url": "https://example.test/google-calendar", + })), + ), + ) + .expect("expected tool suggestion form"); + + assert_eq!( + request.tool_suggestion(), + Some(&ToolSuggestionRequest { + tool_type: ToolSuggestionToolType::Connector, + suggest_type: ToolSuggestionType::Install, + suggest_reason: "Plan and reference events from your calendar".to_string(), + tool_id: "connector_2128aebfecb84f64a069897515042a44".to_string(), + tool_name: "Google Calendar".to_string(), + install_url: "https://example.test/google-calendar".to_string(), + }) + ); + } + #[test] fn empty_unmarked_schema_falls_back() { let request = McpServerElicitationFormRequest::from_event( diff --git a/codex-rs/tui/src/bottom_pane/mod.rs b/codex-rs/tui/src/bottom_pane/mod.rs index f9c34222fb..2d66002418 100644 --- a/codex-rs/tui/src/bottom_pane/mod.rs +++ b/codex-rs/tui/src/bottom_pane/mod.rs @@ -47,6 +47,8 @@ mod mcp_server_elicitation; mod multi_select_picker; mod request_user_input; mod status_line_setup; +pub(crate) use app_link_view::AppLinkElicitationTarget; +pub(crate) use app_link_view::AppLinkSuggestionType; pub(crate) use app_link_view::AppLinkView; pub(crate) use app_link_view::AppLinkViewParams; pub(crate) use approval_overlay::ApprovalOverlay; @@ -963,6 +965,52 @@ impl BottomPane { request }; + if let Some(tool_suggestion) = request.tool_suggestion() { + let suggestion_type = match tool_suggestion.suggest_type { + mcp_server_elicitation::ToolSuggestionType::Install => { + AppLinkSuggestionType::Install + } + mcp_server_elicitation::ToolSuggestionType::Enable => AppLinkSuggestionType::Enable, + }; + let is_installed = matches!( + tool_suggestion.suggest_type, + mcp_server_elicitation::ToolSuggestionType::Enable + ); + let view = AppLinkView::new( + AppLinkViewParams { + app_id: tool_suggestion.tool_id.clone(), + title: tool_suggestion.tool_name.clone(), + description: None, + instructions: match suggestion_type { + AppLinkSuggestionType::Install => { + "Install this app in your browser, then return here.".to_string() + } + AppLinkSuggestionType::Enable => { + "Enable this app to use it for the current request.".to_string() + } + }, + url: tool_suggestion.install_url.clone(), + is_installed, + is_enabled: false, + suggest_reason: Some(tool_suggestion.suggest_reason.clone()), + suggestion_type: Some(suggestion_type), + elicitation_target: Some(AppLinkElicitationTarget { + thread_id: request.thread_id(), + server_name: request.server_name().to_string(), + request_id: request.request_id().clone(), + }), + }, + self.app_event_tx.clone(), + ); + self.pause_status_timer_for_modal(); + self.set_composer_input_enabled( + false, + Some("Respond to the tool suggestion to continue.".to_string()), + ); + self.push_view(Box::new(view)); + return; + } + let modal = McpServerElicitationOverlay::new( request, self.app_event_tx.clone(), diff --git a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__app_link_view__tests__app_link_view_enable_suggestion_with_reason.snap b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__app_link_view__tests__app_link_view_enable_suggestion_with_reason.snap new file mode 100644 index 0000000000..94980ff650 --- /dev/null +++ b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__app_link_view__tests__app_link_view_enable_suggestion_with_reason.snap @@ -0,0 +1,20 @@ +--- +source: tui/src/bottom_pane/app_link_view.rs +expression: "render_snapshot(&view, Rect::new(0, 0, 72, view.desired_height(72)))" +--- + + Google Calendar + Plan events and schedules. + + Plan and reference events from your calendar + + Use $ to insert this app into the prompt. + + Enable this app to use it for the current request. + Newly installed apps can take a few minutes to appear in /apps. + + + › 1. Manage on ChatGPT + 2. Enable app + 3. Back + Use tab / ↑ ↓ to move, enter to select, esc to close diff --git a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__app_link_view__tests__app_link_view_install_suggestion_with_reason.snap b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__app_link_view__tests__app_link_view_install_suggestion_with_reason.snap new file mode 100644 index 0000000000..ac47f87418 --- /dev/null +++ b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__app_link_view__tests__app_link_view_install_suggestion_with_reason.snap @@ -0,0 +1,18 @@ +--- +source: tui/src/bottom_pane/app_link_view.rs +expression: "render_snapshot(&view, Rect::new(0, 0, 72, view.desired_height(72)))" +--- + + Google Calendar + Plan events and schedules. + + Plan and reference events from your calendar + + Install this app in your browser, then return here. + Newly installed apps can take a few minutes to appear in /apps. + After installed, use $ to insert this app into the prompt. + + + › 1. Install on ChatGPT + 2. Back + Use tab / ↑ ↓ to move, enter to select, esc to close From 367a8a2210e7b7a8937b39a838d1ba03451c6213 Mon Sep 17 00:00:00 2001 From: Ahmed Ibrahim Date: Wed, 11 Mar 2026 23:03:07 -0700 Subject: [PATCH 3/6] Clarify spawn agent authorization (#14432) - Clarify that spawn_agent requires explicit user permission for delegation or parallel agent work. - Add a regression test covering the new description text. --- codex-rs/core/src/tools/spec.rs | 5 ++++- .../tests/suite/spawn_agent_description.rs | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/codex-rs/core/src/tools/spec.rs b/codex-rs/core/src/tools/spec.rs index 3364d19e84..afa4c18617 100644 --- a/codex-rs/core/src/tools/spec.rs +++ b/codex-rs/core/src/tools/spec.rs @@ -853,7 +853,10 @@ fn create_spawn_agent_tool(config: &ToolsConfig) -> ToolSpec { name: "spawn_agent".to_string(), description: format!( r#" - Only use `spawn_agent` if and only if the user explicitly asked for sub-agents or parallel agent work. Spawn a sub-agent for a well-scoped task. Returns the agent id (and user-facing nickname when available) to use to communicate with this agent. This spawn_agent tool provides you access to smaller but more efficient sub-agents. A mini model can solve many tasks faster than the main model. You should follow the rules and guidelines below to use this tool. + Only use `spawn_agent` if and only if the user explicitly asks for sub-agents, delegation, or parallel agent work. + Requests for depth, thoroughness, research, investigation, or detailed codebase analysis do not count as permission to spawn. + Agent-role guidance below only helps choose which agent to use after spawning is already authorized; it never authorizes spawning by itself. + Spawn a sub-agent for a well-scoped task. Returns the agent id (and user-facing nickname when available) to use to communicate with this agent. This spawn_agent tool provides you access to smaller but more efficient sub-agents. A mini model can solve many tasks faster than the main model. You should follow the rules and guidelines below to use this tool. {available_models_description} ### When to delegate vs. do the subtask yourself diff --git a/codex-rs/core/tests/suite/spawn_agent_description.rs b/codex-rs/core/tests/suite/spawn_agent_description.rs index ad822805a8..117d1d9ef2 100644 --- a/codex-rs/core/tests/suite/spawn_agent_description.rs +++ b/codex-rs/core/tests/suite/spawn_agent_description.rs @@ -180,6 +180,24 @@ async fn spawn_agent_description_lists_visible_models_and_reasoning_efforts() -> !description.contains("Hidden Model"), "hidden picker model should be omitted from spawn_agent description: {description:?}" ); + assert!( + description.contains( + "Only use `spawn_agent` if and only if the user explicitly asks for sub-agents, delegation, or parallel agent work." + ), + "expected explicit authorization rule in spawn_agent description: {description:?}" + ); + assert!( + description.contains( + "Requests for depth, thoroughness, research, investigation, or detailed codebase analysis do not count as permission to spawn." + ), + "expected non-authorization clarification in spawn_agent description: {description:?}" + ); + assert!( + description.contains( + "Agent-role guidance below only helps choose which agent to use after spawning is already authorized; it never authorizes spawning by itself." + ), + "expected agent-role clarification in spawn_agent description: {description:?}" + ); Ok(()) } From f6c6128fc705205b9f1f2bff50cc2710046ce8de Mon Sep 17 00:00:00 2001 From: pakrym-oai Date: Wed, 11 Mar 2026 23:13:54 -0700 Subject: [PATCH 4/6] Support waiting for code_mode sessions (#14295) ## Summary - persist the code mode runner process in the session-scoped code mode store - switch the runner protocol from `init` to `start` with explicit session ids - handle runner-side session processing without the init waiter queue ## Validation - just fmt - cargo check -p codex-core - node --check codex-rs/core/src/tools/code_mode_runner.cjs --- codex-rs/core/src/codex.rs | 4 +- codex-rs/core/src/codex_tests.rs | 8 +- codex-rs/core/src/state/service.rs | 26 +- codex-rs/core/src/tools/code_mode.rs | 569 +++++++--- codex-rs/core/src/tools/code_mode_runner.cjs | 978 ++++++++++++------ codex-rs/core/src/tools/handlers/code_mode.rs | 83 +- codex-rs/core/src/tools/handlers/mod.rs | 1 + codex-rs/core/src/tools/spec.rs | 66 +- codex-rs/core/tests/suite/code_mode.rs | 821 +++++++++++++++ 9 files changed, 2059 insertions(+), 497 deletions(-) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 5fa4cffa3c..b5695a0bff 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -1737,7 +1737,9 @@ impl Session { config.features.enabled(Feature::RuntimeMetrics), Self::build_model_client_beta_features_header(config.as_ref()), ), - code_mode_store: Default::default(), + code_mode_service: crate::tools::code_mode::CodeModeService::new( + config.js_repl_node_path.clone(), + ), }; let js_repl = Arc::new(JsReplHandle::with_node_path( config.js_repl_node_path.clone(), diff --git a/codex-rs/core/src/codex_tests.rs b/codex-rs/core/src/codex_tests.rs index f1892449b1..7e838642af 100644 --- a/codex-rs/core/src/codex_tests.rs +++ b/codex-rs/core/src/codex_tests.rs @@ -2165,7 +2165,9 @@ pub(crate) async fn make_session_and_context() -> (Session, TurnContext) { config.features.enabled(Feature::RuntimeMetrics), Session::build_model_client_beta_features_header(config.as_ref()), ), - code_mode_store: Default::default(), + code_mode_service: crate::tools::code_mode::CodeModeService::new( + config.js_repl_node_path.clone(), + ), }; let js_repl = Arc::new(JsReplHandle::with_node_path( config.js_repl_node_path.clone(), @@ -2802,7 +2804,9 @@ pub(crate) async fn make_session_and_context_with_dynamic_tools_and_rx( config.features.enabled(Feature::RuntimeMetrics), Session::build_model_client_beta_features_header(config.as_ref()), ), - code_mode_store: Default::default(), + code_mode_service: crate::tools::code_mode::CodeModeService::new( + config.js_repl_node_path.clone(), + ), }; let js_repl = Arc::new(JsReplHandle::with_node_path( config.js_repl_node_path.clone(), diff --git a/codex-rs/core/src/state/service.rs b/codex-rs/core/src/state/service.rs index 5c0a741a12..851618c00e 100644 --- a/codex-rs/core/src/state/service.rs +++ b/codex-rs/core/src/state/service.rs @@ -15,6 +15,7 @@ use crate::models_manager::manager::ModelsManager; use crate::plugins::PluginsManager; use crate::skills::SkillsManager; use crate::state_db::StateDbHandle; +use crate::tools::code_mode::CodeModeService; use crate::tools::network_approval::NetworkApprovalService; use crate::tools::runtimes::ExecveSessionApproval; use crate::tools::sandboxing::ApprovalStore; @@ -22,35 +23,12 @@ use crate::unified_exec::UnifiedExecProcessManager; use codex_hooks::Hooks; use codex_otel::SessionTelemetry; use codex_utils_absolute_path::AbsolutePathBuf; -use serde_json::Value as JsonValue; use std::path::PathBuf; use tokio::sync::Mutex; use tokio::sync::RwLock; use tokio::sync::watch; use tokio_util::sync::CancellationToken; -pub(crate) struct CodeModeStoreService { - stored_values: Mutex>, -} - -impl Default for CodeModeStoreService { - fn default() -> Self { - Self { - stored_values: Mutex::new(HashMap::new()), - } - } -} - -impl CodeModeStoreService { - pub(crate) async fn stored_values(&self) -> HashMap { - self.stored_values.lock().await.clone() - } - - pub(crate) async fn replace_stored_values(&self, values: HashMap) { - *self.stored_values.lock().await = values; - } -} - pub(crate) struct SessionServices { pub(crate) mcp_connection_manager: Arc>, pub(crate) mcp_startup_cancellation_token: Mutex, @@ -82,5 +60,5 @@ pub(crate) struct SessionServices { pub(crate) state_db: Option, /// Session-scoped model client shared across turns. pub(crate) model_client: ModelClient, - pub(crate) code_mode_store: CodeModeStoreService, + pub(crate) code_mode_service: CodeModeService, } diff --git a/codex-rs/core/src/tools/code_mode.rs b/codex-rs/core/src/tools/code_mode.rs index 110588469b..a6e6227be5 100644 --- a/codex-rs/core/src/tools/code_mode.rs +++ b/codex-rs/core/src/tools/code_mode.rs @@ -1,4 +1,6 @@ use std::collections::HashMap; +use std::collections::VecDeque; +use std::path::PathBuf; use std::sync::Arc; use std::time::Duration; @@ -6,7 +8,6 @@ use crate::client_common::tools::ToolSpec; use crate::codex::Session; use crate::codex::TurnContext; use crate::config::Config; -use crate::exec_env::create_env; use crate::features::Feature; use crate::function_tool::FunctionCallError; use crate::tools::ToolRouter; @@ -31,10 +32,15 @@ use tokio::io::AsyncBufReadExt; use tokio::io::AsyncReadExt; use tokio::io::AsyncWriteExt; use tokio::io::BufReader; +use tokio::sync::Mutex; +use tokio::task::JoinHandle; +use tracing::warn; const CODE_MODE_RUNNER_SOURCE: &str = include_str!("code_mode_runner.cjs"); const CODE_MODE_BRIDGE_SOURCE: &str = include_str!("code_mode_bridge.js"); pub(crate) const PUBLIC_TOOL_NAME: &str = "exec"; +pub(crate) const WAIT_TOOL_NAME: &str = "exec_wait"; +pub(crate) const DEFAULT_WAIT_YIELD_TIME_MS: u64 = 10_000; #[derive(Clone)] struct ExecContext { @@ -43,6 +49,133 @@ struct ExecContext { tracker: SharedTurnDiffTracker, } +pub(crate) struct CodeModeProcess { + child: tokio::process::Child, + stdin: tokio::process::ChildStdin, + stdout_lines: tokio::io::Lines>, + stderr_task: Option>, + pending_messages: HashMap>, +} + +impl CodeModeProcess { + async fn write(&mut self, message: &HostToNodeMessage) -> Result<(), std::io::Error> { + let line = serde_json::to_string(message).map_err(std::io::Error::other)?; + self.stdin.write_all(line.as_bytes()).await?; + self.stdin.write_all(b"\n").await?; + self.stdin.flush().await?; + Ok(()) + } + + async fn read(&mut self, session_id: i32) -> Result { + if let Some(message) = self + .pending_messages + .get_mut(&session_id) + .and_then(VecDeque::pop_front) + { + return Ok(message); + } + + loop { + let Some(line) = self.stdout_lines.next_line().await? else { + match self.wait_for_exit().await { + Ok(status) => { + self.join_stderr_task().await; + return Err(std::io::Error::other(format!( + "{PUBLIC_TOOL_NAME} runner exited without returning a result (status {status})" + ))); + } + Err(err) => return Err(std::io::Error::other(err)), + } + }; + if line.trim().is_empty() { + continue; + } + let message: NodeToHostMessage = + serde_json::from_str(&line).map_err(std::io::Error::other)?; + let message_session_id = message_session_id(&message); + if message_session_id == session_id { + return Ok(message); + } + self.pending_messages + .entry(message_session_id) + .or_default() + .push_back(message); + } + } + + fn has_exited(&mut self) -> Result { + self.child + .try_wait() + .map(|status| status.is_some()) + .map_err(|err| format!("failed to inspect {PUBLIC_TOOL_NAME} runner: {err}")) + } + + async fn wait_for_exit(&mut self) -> Result { + self.child + .wait() + .await + .map_err(|err| format!("failed to wait for {PUBLIC_TOOL_NAME} runner: {err}")) + } + + async fn join_stderr_task(&mut self) { + let Some(stderr_task) = self.stderr_task.take() else { + return; + }; + if let Err(err) = stderr_task.await { + warn!("failed to join {PUBLIC_TOOL_NAME} stderr task: {err}"); + } + } +} + +pub(crate) struct CodeModeService { + js_repl_node_path: Option, + stored_values: Mutex>, + process: Arc>>, + next_session_id: Mutex, +} + +impl CodeModeService { + pub(crate) fn new(js_repl_node_path: Option) -> Self { + Self { + js_repl_node_path, + stored_values: Mutex::new(HashMap::new()), + process: Arc::new(Mutex::new(None)), + next_session_id: Mutex::new(1), + } + } + + pub(crate) async fn stored_values(&self) -> HashMap { + self.stored_values.lock().await.clone() + } + + pub(crate) async fn replace_stored_values(&self, values: HashMap) { + *self.stored_values.lock().await = values; + } + + async fn ensure_started( + &self, + ) -> Result>, String> { + let mut process_slot = self.process.lock().await; + let needs_spawn = match process_slot.as_mut() { + Some(process) => !matches!(process.has_exited(), Ok(false)), + None => true, + }; + if needs_spawn { + let node_path = resolve_compatible_node(self.js_repl_node_path.as_deref()).await?; + *process_slot = Some(spawn_code_mode_process(&node_path).await?); + } + drop(process_slot); + Ok(self.process.clone().lock_owned().await) + } + + pub(crate) async fn allocate_session_id(&self) -> i32 { + let mut next_session_id = self.next_session_id.lock().await; + let session_id = *next_session_id; + *next_session_id = next_session_id.saturating_add(1); + session_id + } +} + #[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] #[serde(rename_all = "snake_case")] enum CodeModeToolKind { @@ -64,12 +197,21 @@ struct EnabledTool { #[derive(Serialize)] #[serde(tag = "type", rename_all = "snake_case")] enum HostToNodeMessage { - Init { + Start { + session_id: i32, enabled_tools: Vec, stored_values: HashMap, source: String, }, + Poll { + session_id: i32, + yield_time_ms: u64, + }, + Terminate { + session_id: i32, + }, Response { + session_id: i32, id: String, code_mode_result: JsonValue, }, @@ -79,12 +221,22 @@ enum HostToNodeMessage { #[serde(tag = "type", rename_all = "snake_case")] enum NodeToHostMessage { ToolCall { + session_id: i32, id: String, name: String, #[serde(default)] input: Option, }, + Yielded { + session_id: i32, + content_items: Vec, + }, + Terminated { + session_id: i32, + content_items: Vec, + }, Result { + session_id: i32, content_items: Vec, stored_values: HashMap, #[serde(default)] @@ -94,6 +246,18 @@ enum NodeToHostMessage { }, } +enum CodeModeSessionProgress { + Finished(FunctionToolOutput), + Yielded { output: FunctionToolOutput }, +} + +enum CodeModeExecutionStatus { + Completed, + Failed, + Running(i32), + Terminated, +} + pub(crate) fn instructions(config: &Config) -> Option { if !config.features.enabled(Feature::CodeMode) { return None; @@ -114,7 +278,10 @@ pub(crate) fn instructions(config: &Config) -> Option { )); section.push_str("- Import nested tools from `tools.js`, for example `import { exec_command } from \"tools.js\"` or `import { ALL_TOOLS } from \"tools.js\"` to inspect the available `{ module, name, description }` entries. Namespaced tools are also available from `tools/.js`; MCP tools use `tools/mcp/.js`, for example `import { append_notebook_logs_chart } from \"tools/mcp/ologs.js\"`. Nested tool calls resolve to their code-mode result values.\n"); section.push_str(&format!( - "- Import `{{ output_text, output_image, set_max_output_tokens_per_exec_call, store, load }}` from `@openai/code_mode` (or `\"openai/code_mode\"`). `output_text(value)` surfaces text back to the model and stringifies non-string objects with `JSON.stringify(...)` when possible. `output_image(imageUrl)` appends an `input_image` content item for `http(s)` or `data:` URLs. `store(key, value)` persists JSON-serializable values across `{PUBLIC_TOOL_NAME}` calls in the current session, and `load(key)` returns a cloned stored value or `undefined`. `set_max_output_tokens_per_exec_call(value)` sets the token budget used to truncate the final Rust-side result of the current `{PUBLIC_TOOL_NAME}` execution; the default is `10000`. This guards the overall `{PUBLIC_TOOL_NAME}` output, not individual nested tool invocations. The returned content starts with a separate `Script completed` or `Script failed` text item that includes wall time. When truncation happens, the final text may include `Total output lines:` and the usual `…N tokens truncated…` marker.\n", + "- Import `{{ output_text, output_image, set_max_output_tokens_per_exec_call, set_yield_time, store, load }}` from `@openai/code_mode` (or `\"openai/code_mode\"`). `output_text(value)` surfaces text back to the model and stringifies non-string objects with `JSON.stringify(...)` when possible. `output_image(imageUrl)` appends an `input_image` content item for `http(s)` or `data:` URLs. `store(key, value)` persists JSON-serializable values across `{PUBLIC_TOOL_NAME}` calls in the current session, and `load(key)` returns a cloned stored value or `undefined`. `set_max_output_tokens_per_exec_call(value)` sets the token budget used to truncate direct `{PUBLIC_TOOL_NAME}` returns; `{WAIT_TOOL_NAME}` uses its own `max_tokens` argument instead and defaults to `10000`. `set_yield_time(value)` asks `{PUBLIC_TOOL_NAME}` to return early if the script is still running after that many milliseconds so `{WAIT_TOOL_NAME}` can resume it later. The returned content starts with a separate `Script completed`, `Script failed`, or `Script running with session ID …` text item that includes wall time. When truncation happens, the final text may include `Total output lines:` and the usual `…N tokens truncated…` marker.\n", + )); + section.push_str(&format!( + "- If `{PUBLIC_TOOL_NAME}` returns `Script running with session ID …`, call `{WAIT_TOOL_NAME}` with that `session_id` to keep waiting for more output, completion, or termination.\n", )); section.push_str( "- Function tools require JSON object arguments. Freeform tools require raw strings.\n", @@ -137,30 +304,103 @@ pub(crate) async fn execute( tracker, }; let enabled_tools = build_enabled_tools(&exec).await; - let stored_values = exec.session.services.code_mode_store.stored_values().await; + let service = &exec.session.services.code_mode_service; + let stored_values = service.stored_values().await; let source = build_source(&code, &enabled_tools).map_err(FunctionCallError::RespondToModel)?; - execute_node(exec, source, enabled_tools, stored_values) + let session_id = service.allocate_session_id().await; + let process_slot = service + .ensure_started() .await - .map_err(FunctionCallError::RespondToModel) + .map_err(FunctionCallError::RespondToModel)?; + let result = { + let mut process_slot = process_slot; + let Some(process) = process_slot.as_mut() else { + return Err(FunctionCallError::RespondToModel(format!( + "{PUBLIC_TOOL_NAME} runner failed to start" + ))); + }; + drive_code_mode_session( + &exec, + process, + HostToNodeMessage::Start { + session_id, + enabled_tools, + stored_values, + source, + }, + None, + false, + ) + .await + }; + match result { + Ok(CodeModeSessionProgress::Finished(output)) + | Ok(CodeModeSessionProgress::Yielded { output }) => Ok(output), + Err(error) => Err(FunctionCallError::RespondToModel(error)), + } } -async fn execute_node( - exec: ExecContext, - source: String, - enabled_tools: Vec, - stored_values: HashMap, -) -> Result { - let node_path = resolve_compatible_node(exec.turn.config.js_repl_node_path.as_deref()).await?; - let started_at = std::time::Instant::now(); +pub(crate) async fn wait( + session: Arc, + turn: Arc, + tracker: SharedTurnDiffTracker, + session_id: i32, + yield_time_ms: u64, + max_output_tokens: Option, + terminate: bool, +) -> Result { + let exec = ExecContext { + session, + turn, + tracker, + }; + let process_slot = exec + .session + .services + .code_mode_service + .ensure_started() + .await + .map_err(FunctionCallError::RespondToModel)?; + let result = { + let mut process_slot = process_slot; + let Some(process) = process_slot.as_mut() else { + return Err(FunctionCallError::RespondToModel(format!( + "{PUBLIC_TOOL_NAME} runner failed to start" + ))); + }; + if !matches!(process.has_exited(), Ok(false)) { + return Err(FunctionCallError::RespondToModel(format!( + "{PUBLIC_TOOL_NAME} runner failed to start" + ))); + } + drive_code_mode_session( + &exec, + process, + if terminate { + HostToNodeMessage::Terminate { session_id } + } else { + HostToNodeMessage::Poll { + session_id, + yield_time_ms, + } + }, + Some(max_output_tokens), + terminate, + ) + .await + }; + match result { + Ok(CodeModeSessionProgress::Finished(output)) + | Ok(CodeModeSessionProgress::Yielded { output }) => Ok(output), + Err(error) => Err(FunctionCallError::RespondToModel(error)), + } +} - let env = create_env(&exec.turn.shell_environment_policy, None); - let mut cmd = tokio::process::Command::new(&node_path); +async fn spawn_code_mode_process(node_path: &std::path::Path) -> Result { + let mut cmd = tokio::process::Command::new(node_path); cmd.arg("--experimental-vm-modules"); cmd.arg("--eval"); cmd.arg(CODE_MODE_RUNNER_SOURCE); - cmd.current_dir(&exec.turn.cwd); - cmd.env_clear(); - cmd.envs(env); cmd.stdin(std::process::Stdio::piped()) .stdout(std::process::Stdio::piped()) .stderr(std::process::Stdio::piped()) @@ -177,7 +417,7 @@ async fn execute_node( .stderr .take() .ok_or_else(|| format!("{PUBLIC_TOOL_NAME} runner missing stderr"))?; - let mut stdin = child + let stdin = child .stdin .take() .ok_or_else(|| format!("{PUBLIC_TOOL_NAME} runner missing stdin"))?; @@ -185,138 +425,189 @@ async fn execute_node( let stderr_task = tokio::spawn(async move { let mut reader = BufReader::new(stderr); let mut buf = Vec::new(); - let _ = reader.read_to_end(&mut buf).await; - String::from_utf8_lossy(&buf).trim().to_string() + match reader.read_to_end(&mut buf).await { + Ok(_) => { + let stderr = String::from_utf8_lossy(&buf).trim().to_string(); + if !stderr.is_empty() { + warn!("{PUBLIC_TOOL_NAME} runner stderr: {stderr}"); + } + } + Err(err) => { + warn!("failed to read {PUBLIC_TOOL_NAME} stderr: {err}"); + } + } }); - write_message( - &mut stdin, - &HostToNodeMessage::Init { - enabled_tools: enabled_tools.clone(), - stored_values, - source, - }, - ) - .await?; - - let mut stdout_lines = BufReader::new(stdout).lines(); - let mut pending_result = None; - while let Some(line) = stdout_lines - .next_line() - .await - .map_err(|err| format!("failed to read {PUBLIC_TOOL_NAME} runner stdout: {err}"))? - { - if line.trim().is_empty() { - continue; - } - let message: NodeToHostMessage = serde_json::from_str(&line).map_err(|err| { - format!("invalid {PUBLIC_TOOL_NAME} runner message: {err}; line={line}") - })?; - match message { - NodeToHostMessage::ToolCall { id, name, input } => { - let response = HostToNodeMessage::Response { - id, - code_mode_result: call_nested_tool(exec.clone(), name, input).await, - }; - write_message(&mut stdin, &response).await?; - } - NodeToHostMessage::Result { - content_items, - stored_values, - error_text, - max_output_tokens_per_exec_call, - } => { - exec.session - .services - .code_mode_store - .replace_stored_values(stored_values) - .await; - pending_result = Some(( - output_content_items_from_json_values(content_items)?, - error_text, - max_output_tokens_per_exec_call, - )); - break; - } - } - } - - drop(stdin); - - let status = child - .wait() - .await - .map_err(|err| format!("failed to wait for {PUBLIC_TOOL_NAME} runner: {err}"))?; - let stderr = stderr_task - .await - .map_err(|err| format!("failed to collect {PUBLIC_TOOL_NAME} stderr: {err}"))?; - let wall_time = started_at.elapsed(); - let success = status.success(); - - let Some((mut content_items, error_text, max_output_tokens_per_exec_call)) = pending_result - else { - let message = if stderr.is_empty() { - format!("{PUBLIC_TOOL_NAME} runner exited without returning a result (status {status})") - } else { - stderr - }; - return Err(message); - }; - - if !success { - let error_text = error_text.unwrap_or_else(|| { - if stderr.is_empty() { - format!("Process exited with status {status}") - } else { - stderr - } - }); - content_items.push(FunctionCallOutputContentItem::InputText { - text: format!("Script error:\n{error_text}"), - }); - } - - let mut content_items = - truncate_code_mode_result(content_items, max_output_tokens_per_exec_call); - prepend_script_status(&mut content_items, success, wall_time); - Ok(FunctionToolOutput::from_content( - content_items, - Some(success), - )) + Ok(CodeModeProcess { + child, + stdin, + stdout_lines: BufReader::new(stdout).lines(), + stderr_task: Some(stderr_task), + pending_messages: HashMap::new(), + }) } -async fn write_message( - stdin: &mut tokio::process::ChildStdin, - message: &HostToNodeMessage, -) -> Result<(), String> { - let line = serde_json::to_string(message) - .map_err(|err| format!("failed to serialize {PUBLIC_TOOL_NAME} message: {err}"))?; - stdin - .write_all(line.as_bytes()) +async fn drive_code_mode_session( + exec: &ExecContext, + process: &mut CodeModeProcess, + message: HostToNodeMessage, + poll_max_output_tokens: Option>, + is_terminate: bool, +) -> Result { + let started_at = std::time::Instant::now(); + let session_id = match &message { + HostToNodeMessage::Start { session_id, .. } + | HostToNodeMessage::Poll { session_id, .. } + | HostToNodeMessage::Terminate { session_id } + | HostToNodeMessage::Response { session_id, .. } => *session_id, + }; + process + .write(&message) .await - .map_err(|err| format!("failed to write {PUBLIC_TOOL_NAME} message: {err}"))?; - stdin - .write_all(b"\n") - .await - .map_err(|err| format!("failed to write {PUBLIC_TOOL_NAME} message newline: {err}"))?; - stdin - .flush() - .await - .map_err(|err| format!("failed to flush {PUBLIC_TOOL_NAME} message: {err}")) + .map_err(|err| err.to_string())?; + + loop { + let message = process + .read(session_id) + .await + .map_err(|err| err.to_string())?; + if let Some(progress) = handle_node_message( + exec, + process, + session_id, + message, + poll_max_output_tokens, + started_at, + is_terminate, + ) + .await? + { + return Ok(progress); + } + } +} + +async fn handle_node_message( + exec: &ExecContext, + process: &mut CodeModeProcess, + session_id: i32, + message: NodeToHostMessage, + poll_max_output_tokens: Option>, + started_at: std::time::Instant, + is_terminate: bool, +) -> Result, String> { + match message { + NodeToHostMessage::ToolCall { + session_id: message_session_id, + id, + name, + input, + } => { + if is_terminate { + return Ok(None); + } + let response = HostToNodeMessage::Response { + session_id: message_session_id, + id, + code_mode_result: call_nested_tool(exec.clone(), name, input).await, + }; + process + .write(&response) + .await + .map_err(|err| err.to_string())?; + Ok(None) + } + NodeToHostMessage::Yielded { content_items, .. } => { + if is_terminate { + return Ok(None); + } + let mut delta_items = output_content_items_from_json_values(content_items)?; + delta_items = truncate_code_mode_result(delta_items, poll_max_output_tokens.flatten()); + prepend_script_status( + &mut delta_items, + CodeModeExecutionStatus::Running(session_id), + started_at.elapsed(), + ); + Ok(Some(CodeModeSessionProgress::Yielded { + output: FunctionToolOutput::from_content(delta_items, Some(true)), + })) + } + NodeToHostMessage::Terminated { content_items, .. } => { + let mut delta_items = output_content_items_from_json_values(content_items)?; + delta_items = truncate_code_mode_result(delta_items, poll_max_output_tokens.flatten()); + prepend_script_status( + &mut delta_items, + CodeModeExecutionStatus::Terminated, + started_at.elapsed(), + ); + Ok(Some(CodeModeSessionProgress::Finished( + FunctionToolOutput::from_content(delta_items, Some(true)), + ))) + } + NodeToHostMessage::Result { + content_items, + stored_values, + error_text, + max_output_tokens_per_exec_call, + .. + } => { + exec.session + .services + .code_mode_service + .replace_stored_values(stored_values) + .await; + let mut delta_items = output_content_items_from_json_values(content_items)?; + let success = error_text.is_none(); + if let Some(error_text) = error_text { + delta_items.push(FunctionCallOutputContentItem::InputText { + text: format!("Script error:\n{error_text}"), + }); + } + + let mut delta_items = truncate_code_mode_result( + delta_items, + poll_max_output_tokens.unwrap_or(max_output_tokens_per_exec_call), + ); + prepend_script_status( + &mut delta_items, + if success { + CodeModeExecutionStatus::Completed + } else { + CodeModeExecutionStatus::Failed + }, + started_at.elapsed(), + ); + Ok(Some(CodeModeSessionProgress::Finished( + FunctionToolOutput::from_content(delta_items, Some(success)), + ))) + } + } +} + +fn message_session_id(message: &NodeToHostMessage) -> i32 { + match message { + NodeToHostMessage::ToolCall { session_id, .. } + | NodeToHostMessage::Yielded { session_id, .. } + | NodeToHostMessage::Terminated { session_id, .. } + | NodeToHostMessage::Result { session_id, .. } => *session_id, + } } fn prepend_script_status( content_items: &mut Vec, - success: bool, + status: CodeModeExecutionStatus, wall_time: Duration, ) { let wall_time_seconds = ((wall_time.as_secs_f32()) * 10.0).round() / 10.0; let header = format!( "{}\nWall time {wall_time_seconds:.1} seconds\nOutput:\n", - if success { - "Script completed" - } else { - "Script failed" + match status { + CodeModeExecutionStatus::Completed => "Script completed".to_string(), + CodeModeExecutionStatus::Failed => "Script failed".to_string(), + CodeModeExecutionStatus::Running(session_id) => { + format!("Script running with session ID {session_id}") + } + CodeModeExecutionStatus::Terminated => "Script terminated".to_string(), } ); content_items.insert(0, FunctionCallOutputContentItem::InputText { text: header }); @@ -366,7 +657,7 @@ async fn build_enabled_tools(exec: &ExecContext) -> Vec { fn enabled_tool_from_spec(spec: ToolSpec) -> Option { let tool_name = spec.name().to_string(); - if tool_name == PUBLIC_TOOL_NAME { + if tool_name == PUBLIC_TOOL_NAME || tool_name == WAIT_TOOL_NAME { return None; } diff --git a/codex-rs/core/src/tools/code_mode_runner.cjs b/codex-rs/core/src/tools/code_mode_runner.cjs index f36fa6f92e..d64e369f32 100644 --- a/codex-rs/core/src/tools/code_mode_runner.cjs +++ b/codex-rs/core/src/tools/code_mode_runner.cjs @@ -1,9 +1,8 @@ 'use strict'; const readline = require('node:readline'); -const vm = require('node:vm'); +const { Worker } = require('node:worker_threads'); -const { SourceTextModule, SyntheticModule } = vm; const DEFAULT_MAX_OUTPUT_TOKENS_PER_EXEC_CALL = 10000; function normalizeMaxOutputTokensPerExecCall(value) { @@ -13,6 +12,425 @@ function normalizeMaxOutputTokensPerExecCall(value) { return value; } +function normalizeYieldTime(value) { + if (!Number.isSafeInteger(value) || value < 0) { + throw new TypeError('yield_time must be a non-negative safe integer'); + } + return value; +} + +function formatErrorText(error) { + return String(error && error.stack ? error.stack : error); +} + +function cloneJsonValue(value) { + return JSON.parse(JSON.stringify(value)); +} + +function clearTimer(timer) { + if (timer !== null) { + clearTimeout(timer); + } + return null; +} + +function takeContentItems(session) { + const clonedContentItems = cloneJsonValue(session.content_items); + session.content_items.splice(0, session.content_items.length); + return Array.isArray(clonedContentItems) ? clonedContentItems : []; +} + +function codeModeWorkerMain() { + 'use strict'; + + const { parentPort, workerData } = require('node:worker_threads'); + const vm = require('node:vm'); + const { SourceTextModule, SyntheticModule } = vm; + + const DEFAULT_MAX_OUTPUT_TOKENS_PER_EXEC_CALL = 10000; + + function normalizeMaxOutputTokensPerExecCall(value) { + if (!Number.isSafeInteger(value) || value < 0) { + throw new TypeError('max_output_tokens_per_exec_call must be a non-negative safe integer'); + } + return value; + } + + function normalizeYieldTime(value) { + if (!Number.isSafeInteger(value) || value < 0) { + throw new TypeError('yield_time must be a non-negative safe integer'); + } + return value; + } + + function formatErrorText(error) { + return String(error && error.stack ? error.stack : error); + } + + function cloneJsonValue(value) { + return JSON.parse(JSON.stringify(value)); + } + + function createToolCaller() { + let nextId = 0; + const pending = new Map(); + + parentPort.on('message', (message) => { + if (message.type === 'tool_response') { + const entry = pending.get(message.id); + if (!entry) { + return; + } + pending.delete(message.id); + entry.resolve(message.result ?? ''); + return; + } + + if (message.type === 'tool_response_error') { + const entry = pending.get(message.id); + if (!entry) { + return; + } + pending.delete(message.id); + entry.reject(new Error(message.error_text ?? 'tool call failed')); + return; + } + }); + + return (name, input) => { + const id = 'msg-' + ++nextId; + return new Promise((resolve, reject) => { + pending.set(id, { resolve, reject }); + parentPort.postMessage({ + type: 'tool_call', + id, + name: String(name), + input, + }); + }); + }; + } + + function createContentItems() { + const contentItems = []; + const push = contentItems.push.bind(contentItems); + contentItems.push = (...items) => { + for (const item of items) { + parentPort.postMessage({ + type: 'content_item', + item: cloneJsonValue(item), + }); + } + return push(...items); + }; + parentPort.on('message', (message) => { + if (message.type === 'clear_content') { + contentItems.splice(0, contentItems.length); + } + }); + return contentItems; + } + + function createToolsNamespace(callTool, enabledTools) { + const tools = Object.create(null); + + for (const { tool_name } of enabledTools) { + Object.defineProperty(tools, tool_name, { + value: async (args) => callTool(tool_name, args), + configurable: false, + enumerable: true, + writable: false, + }); + } + + return Object.freeze(tools); + } + + function createAllToolsMetadata(enabledTools) { + return Object.freeze( + enabledTools.map(({ module: modulePath, name, description }) => + Object.freeze({ + module: modulePath, + name, + description, + }) + ) + ); + } + + function createToolsModule(context, callTool, enabledTools) { + const tools = createToolsNamespace(callTool, enabledTools); + const allTools = createAllToolsMetadata(enabledTools); + const exportNames = ['ALL_TOOLS']; + + for (const { tool_name } of enabledTools) { + if (tool_name !== 'ALL_TOOLS') { + exportNames.push(tool_name); + } + } + + const uniqueExportNames = [...new Set(exportNames)]; + + return new SyntheticModule( + uniqueExportNames, + function initToolsModule() { + this.setExport('ALL_TOOLS', allTools); + for (const exportName of uniqueExportNames) { + if (exportName !== 'ALL_TOOLS') { + this.setExport(exportName, tools[exportName]); + } + } + }, + { context } + ); + } + + function ensureContentItems(context) { + if (!Array.isArray(context.__codexContentItems)) { + context.__codexContentItems = []; + } + return context.__codexContentItems; + } + + function serializeOutputText(value) { + if (typeof value === 'string') { + return value; + } + if ( + typeof value === 'undefined' || + value === null || + typeof value === 'boolean' || + typeof value === 'number' || + typeof value === 'bigint' + ) { + return String(value); + } + + const serialized = JSON.stringify(value); + if (typeof serialized === 'string') { + return serialized; + } + + return String(value); + } + + function normalizeOutputImageUrl(value) { + if (typeof value !== 'string' || !value) { + throw new TypeError('output_image expects a non-empty image URL string'); + } + if (/^(?:https?:\/\/|data:)/i.test(value)) { + return value; + } + throw new TypeError('output_image expects an http(s) or data URL'); + } + + function createCodeModeModule(context, state) { + const load = (key) => { + if (typeof key !== 'string') { + throw new TypeError('load key must be a string'); + } + if (!Object.prototype.hasOwnProperty.call(state.storedValues, key)) { + return undefined; + } + return cloneJsonValue(state.storedValues[key]); + }; + const store = (key, value) => { + if (typeof key !== 'string') { + throw new TypeError('store key must be a string'); + } + state.storedValues[key] = cloneJsonValue(value); + }; + const outputText = (value) => { + const item = { + type: 'input_text', + text: serializeOutputText(value), + }; + ensureContentItems(context).push(item); + return item; + }; + const outputImage = (value) => { + const item = { + type: 'input_image', + image_url: normalizeOutputImageUrl(value), + }; + ensureContentItems(context).push(item); + return item; + }; + + return new SyntheticModule( + [ + 'load', + 'output_text', + 'output_image', + 'set_max_output_tokens_per_exec_call', + 'set_yield_time', + 'store', + ], + function initCodeModeModule() { + this.setExport('load', load); + this.setExport('output_text', outputText); + this.setExport('output_image', outputImage); + this.setExport('set_max_output_tokens_per_exec_call', (value) => { + const normalized = normalizeMaxOutputTokensPerExecCall(value); + state.maxOutputTokensPerExecCall = normalized; + parentPort.postMessage({ + type: 'set_max_output_tokens_per_exec_call', + value: normalized, + }); + return normalized; + }); + this.setExport('set_yield_time', (value) => { + const normalized = normalizeYieldTime(value); + parentPort.postMessage({ + type: 'set_yield_time', + value: normalized, + }); + return normalized; + }); + this.setExport('store', store); + }, + { context } + ); + } + + function namespacesMatch(left, right) { + if (left.length !== right.length) { + return false; + } + return left.every((segment, index) => segment === right[index]); + } + + function createNamespacedToolsNamespace(callTool, enabledTools, namespace) { + const tools = Object.create(null); + + for (const tool of enabledTools) { + const toolNamespace = Array.isArray(tool.namespace) ? tool.namespace : []; + if (!namespacesMatch(toolNamespace, namespace)) { + continue; + } + + Object.defineProperty(tools, tool.name, { + value: async (args) => callTool(tool.tool_name, args), + configurable: false, + enumerable: true, + writable: false, + }); + } + + return Object.freeze(tools); + } + + function createNamespacedToolsModule(context, callTool, enabledTools, namespace) { + const tools = createNamespacedToolsNamespace(callTool, enabledTools, namespace); + const exportNames = []; + + for (const exportName of Object.keys(tools)) { + if (exportName !== 'ALL_TOOLS') { + exportNames.push(exportName); + } + } + + const uniqueExportNames = [...new Set(exportNames)]; + + return new SyntheticModule( + uniqueExportNames, + function initNamespacedToolsModule() { + for (const exportName of uniqueExportNames) { + this.setExport(exportName, tools[exportName]); + } + }, + { context } + ); + } + + function createModuleResolver(context, callTool, enabledTools, state) { + const toolsModule = createToolsModule(context, callTool, enabledTools); + const codeModeModule = createCodeModeModule(context, state); + const namespacedModules = new Map(); + + return function resolveModule(specifier) { + if (specifier === 'tools.js') { + return toolsModule; + } + if (specifier === '@openai/code_mode' || specifier === 'openai/code_mode') { + return codeModeModule; + } + const namespacedMatch = /^tools\/(.+)\.js$/.exec(specifier); + if (!namespacedMatch) { + throw new Error('Unsupported import in exec: ' + specifier); + } + + const namespace = namespacedMatch[1] + .split('/') + .filter((segment) => segment.length > 0); + if (namespace.length === 0) { + throw new Error('Unsupported import in exec: ' + specifier); + } + + const cacheKey = namespace.join('/'); + if (!namespacedModules.has(cacheKey)) { + namespacedModules.set( + cacheKey, + createNamespacedToolsModule(context, callTool, enabledTools, namespace) + ); + } + return namespacedModules.get(cacheKey); + }; + } + + async function runModule(context, start, state, callTool) { + const resolveModule = createModuleResolver( + context, + callTool, + start.enabled_tools ?? [], + state + ); + const mainModule = new SourceTextModule(start.source, { + context, + identifier: 'exec_main.mjs', + importModuleDynamically: async (specifier) => resolveModule(specifier), + }); + + await mainModule.link(resolveModule); + await mainModule.evaluate(); + } + + async function main() { + const start = workerData ?? {}; + const state = { + maxOutputTokensPerExecCall: DEFAULT_MAX_OUTPUT_TOKENS_PER_EXEC_CALL, + storedValues: cloneJsonValue(start.stored_values ?? {}), + }; + const callTool = createToolCaller(); + const context = vm.createContext({ + __codexContentItems: createContentItems(), + __codex_tool_call: callTool, + }); + + try { + await runModule(context, start, state, callTool); + parentPort.postMessage({ + type: 'result', + stored_values: state.storedValues, + }); + } catch (error) { + parentPort.postMessage({ + type: 'result', + stored_values: state.storedValues, + error_text: formatErrorText(error), + }); + } + } + + void main().catch((error) => { + parentPort.postMessage({ + type: 'result', + stored_values: {}, + error_text: formatErrorText(error), + }); + }); +} + function createProtocol() { const rl = readline.createInterface({ input: process.stdin, @@ -21,11 +439,10 @@ function createProtocol() { let nextId = 0; const pending = new Map(); - let initResolve; - let initReject; - const init = new Promise((resolve, reject) => { - initResolve = resolve; - initReject = reject; + const sessions = new Map(); + let closedResolve; + const closed = new Promise((resolve) => { + closedResolve = resolve; }); rl.on('line', (line) => { @@ -37,40 +454,80 @@ function createProtocol() { try { message = JSON.parse(line); } catch (error) { - initReject(error); + process.stderr.write(formatErrorText(error) + '\n'); return; } - if (message.type === 'init') { - initResolve(message); + if (message.type === 'start') { + startSession(protocol, sessions, message); + return; + } + + if (message.type === 'poll') { + const session = sessions.get(message.session_id); + if (session) { + schedulePollYield(protocol, session, normalizeYieldTime(message.yield_time_ms ?? 0)); + } else { + void protocol.send({ + type: 'result', + session_id: message.session_id, + content_items: [], + stored_values: {}, + error_text: `exec session ${message.session_id} not found`, + max_output_tokens_per_exec_call: DEFAULT_MAX_OUTPUT_TOKENS_PER_EXEC_CALL, + }); + } + return; + } + + if (message.type === 'terminate') { + const session = sessions.get(message.session_id); + if (session) { + void terminateSession(protocol, sessions, session); + } else { + void protocol.send({ + type: 'result', + session_id: message.session_id, + content_items: [], + stored_values: {}, + error_text: `exec session ${message.session_id} not found`, + max_output_tokens_per_exec_call: DEFAULT_MAX_OUTPUT_TOKENS_PER_EXEC_CALL, + }); + } return; } if (message.type === 'response') { - const entry = pending.get(message.id); + const entry = pending.get(message.session_id + ':' + message.id); if (!entry) { return; } - pending.delete(message.id); + pending.delete(message.session_id + ':' + message.id); entry.resolve(message.code_mode_result ?? ''); return; } - initReject(new Error(`Unknown protocol message type: ${message.type}`)); + process.stderr.write('Unknown protocol message type: ' + message.type + '\n'); }); rl.on('close', () => { const error = new Error('stdin closed'); - initReject(error); for (const entry of pending.values()) { entry.reject(error); } pending.clear(); + for (const session of sessions.values()) { + session.initial_yield_timer = clearTimer(session.initial_yield_timer); + session.poll_yield_timer = clearTimer(session.poll_yield_timer); + void session.worker.terminate().catch(() => {}); + } + sessions.clear(); + closedResolve(); }); function send(message) { return new Promise((resolve, reject) => { - process.stdout.write(`${JSON.stringify(message)}\n`, (error) => { + process.stdout.write(JSON.stringify(message) + '\n', (error) => { if (error) { reject(error); } else { @@ -80,328 +537,223 @@ function createProtocol() { }); } - function request(type, payload) { - const id = `msg-${++nextId}`; + function request(sessionId, type, payload) { + const id = 'msg-' + ++nextId; + const pendingKey = sessionId + ':' + id; return new Promise((resolve, reject) => { - pending.set(id, { resolve, reject }); - void send({ type, id, ...payload }).catch((error) => { - pending.delete(id); + pending.set(pendingKey, { resolve, reject }); + void send({ type, session_id: sessionId, id, ...payload }).catch((error) => { + pending.delete(pendingKey); reject(error); }); }); } - return { init, request, send }; + const protocol = { closed, request, send }; + return protocol; } -function readContentItems(context) { - try { - const serialized = vm.runInContext('JSON.stringify(globalThis.__codexContentItems ?? [])', context); - const contentItems = JSON.parse(serialized); - return Array.isArray(contentItems) ? contentItems : []; - } catch { - return []; - } +function sessionWorkerSource() { + return '(' + codeModeWorkerMain.toString() + ')();'; } -function formatErrorText(error) { - return String(error && error.stack ? error.stack : error); -} - -function cloneJsonValue(value) { - return JSON.parse(JSON.stringify(value)); -} - -function createToolCaller(protocol) { - return (name, input) => - protocol.request('tool_call', { - name: String(name), - input, - }); -} - -function createToolsNamespace(callTool, enabledTools) { - const tools = Object.create(null); - - for (const { tool_name } of enabledTools) { - Object.defineProperty(tools, tool_name, { - value: async (args) => callTool(tool_name, args), - configurable: false, - enumerable: true, - writable: false, - }); - } - - return Object.freeze(tools); -} - -function createAllToolsMetadata(enabledTools) { - return Object.freeze( - enabledTools.map(({ module: modulePath, name, description }) => - Object.freeze({ - module: modulePath, - name, - description, - }) - ) - ); -} - -function createToolsModule(context, callTool, enabledTools) { - const tools = createToolsNamespace(callTool, enabledTools); - const allTools = createAllToolsMetadata(enabledTools); - const exportNames = ['ALL_TOOLS']; - - for (const { tool_name } of enabledTools) { - if (tool_name !== 'ALL_TOOLS') { - exportNames.push(tool_name); - } - } - - const uniqueExportNames = [...new Set(exportNames)]; - - return new SyntheticModule( - uniqueExportNames, - function initToolsModule() { - this.setExport('ALL_TOOLS', allTools); - for (const exportName of uniqueExportNames) { - if (exportName !== 'ALL_TOOLS') { - this.setExport(exportName, tools[exportName]); - } - } - }, - { context } - ); -} - -function ensureContentItems(context) { - if (!Array.isArray(context.__codexContentItems)) { - context.__codexContentItems = []; - } - return context.__codexContentItems; -} - -function serializeOutputText(value) { - if (typeof value === 'string') { - return value; - } - if ( - typeof value === 'undefined' || - value === null || - typeof value === 'boolean' || - typeof value === 'number' || - typeof value === 'bigint' - ) { - return String(value); - } - - const serialized = JSON.stringify(value); - if (typeof serialized === 'string') { - return serialized; - } - - return String(value); -} - -function normalizeOutputImageUrl(value) { - if (typeof value !== 'string' || !value) { - throw new TypeError('output_image expects a non-empty image URL string'); - } - if (/^(?:https?:\/\/|data:)/i.test(value)) { - return value; - } - throw new TypeError('output_image expects an http(s) or data URL'); -} - -function createCodeModeModule(context, state) { - const load = (key) => { - if (typeof key !== 'string') { - throw new TypeError('load key must be a string'); - } - if (!Object.prototype.hasOwnProperty.call(state.storedValues, key)) { - return undefined; - } - return cloneJsonValue(state.storedValues[key]); - }; - const store = (key, value) => { - if (typeof key !== 'string') { - throw new TypeError('store key must be a string'); - } - state.storedValues[key] = cloneJsonValue(value); - }; - const outputText = (value) => { - const item = { - type: 'input_text', - text: serializeOutputText(value), - }; - ensureContentItems(context).push(item); - return item; - }; - const outputImage = (value) => { - const item = { - type: 'input_image', - image_url: normalizeOutputImageUrl(value), - }; - ensureContentItems(context).push(item); - return item; +function startSession(protocol, sessions, start) { + const session = { + completed: false, + content_items: [], + id: start.session_id, + initial_yield_timer: null, + initial_yield_triggered: false, + max_output_tokens_per_exec_call: DEFAULT_MAX_OUTPUT_TOKENS_PER_EXEC_CALL, + poll_yield_timer: null, + worker: new Worker(sessionWorkerSource(), { + eval: true, + workerData: start, + }), }; + sessions.set(session.id, session); - return new SyntheticModule( - ['load', 'output_text', 'output_image', 'set_max_output_tokens_per_exec_call', 'store'], - function initCodeModeModule() { - this.setExport('load', load); - this.setExport('output_text', outputText); - this.setExport('output_image', outputImage); - this.setExport('set_max_output_tokens_per_exec_call', (value) => { - const normalized = normalizeMaxOutputTokensPerExecCall(value); - state.maxOutputTokensPerExecCall = normalized; - return normalized; + session.worker.on('message', (message) => { + void handleWorkerMessage(protocol, sessions, session, message).catch((error) => { + void completeSession(protocol, sessions, session, { + type: 'result', + stored_values: {}, + error_text: formatErrorText(error), }); - this.setExport('store', store); - }, - { context } - ); -} - -function namespacesMatch(left, right) { - if (left.length !== right.length) { - return false; - } - return left.every((segment, index) => segment === right[index]); -} - -function createNamespacedToolsNamespace(callTool, enabledTools, namespace) { - const tools = Object.create(null); - - for (const tool of enabledTools) { - const toolNamespace = Array.isArray(tool.namespace) ? tool.namespace : []; - if (!namespacesMatch(toolNamespace, namespace)) { - continue; - } - - Object.defineProperty(tools, tool.name, { - value: async (args) => callTool(tool.tool_name, args), - configurable: false, - enumerable: true, - writable: false, }); - } - - return Object.freeze(tools); -} - -function createNamespacedToolsModule(context, callTool, enabledTools, namespace) { - const tools = createNamespacedToolsNamespace(callTool, enabledTools, namespace); - const exportNames = []; - - for (const exportName of Object.keys(tools)) { - if (exportName !== 'ALL_TOOLS') { - exportNames.push(exportName); - } - } - - const uniqueExportNames = [...new Set(exportNames)]; - - return new SyntheticModule( - uniqueExportNames, - function initNamespacedToolsModule() { - for (const exportName of uniqueExportNames) { - this.setExport(exportName, tools[exportName]); - } - }, - { context } - ); -} - -function createModuleResolver(context, callTool, enabledTools, state) { - const toolsModule = createToolsModule(context, callTool, enabledTools); - const codeModeModule = createCodeModeModule(context, state); - const namespacedModules = new Map(); - - return function resolveModule(specifier) { - if (specifier === 'tools.js') { - return toolsModule; - } - if (specifier === '@openai/code_mode' || specifier === 'openai/code_mode') { - return codeModeModule; - } - const namespacedMatch = /^tools\/(.+)\.js$/.exec(specifier); - if (!namespacedMatch) { - throw new Error(`Unsupported import in exec: ${specifier}`); - } - - const namespace = namespacedMatch[1] - .split('/') - .filter((segment) => segment.length > 0); - if (namespace.length === 0) { - throw new Error(`Unsupported import in exec: ${specifier}`); - } - - const cacheKey = namespace.join('/'); - if (!namespacedModules.has(cacheKey)) { - namespacedModules.set( - cacheKey, - createNamespacedToolsModule(context, callTool, enabledTools, namespace) - ); - } - return namespacedModules.get(cacheKey); - }; -} - -async function runModule(context, request, state, callTool) { - const resolveModule = createModuleResolver( - context, - callTool, - request.enabled_tools ?? [], - state - ); - const mainModule = new SourceTextModule(request.source, { - context, - identifier: 'exec_main.mjs', - importModuleDynamically: async (specifier) => resolveModule(specifier), }); + session.worker.on('error', (error) => { + void completeSession(protocol, sessions, session, { + type: 'result', + stored_values: {}, + error_text: formatErrorText(error), + }); + }); + session.worker.on('exit', (code) => { + if (code !== 0 && !session.completed) { + void completeSession(protocol, sessions, session, { + type: 'result', + stored_values: {}, + error_text: 'exec worker exited with code ' + code, + }); + } + }); +} - await mainModule.link(resolveModule); - await mainModule.evaluate(); +async function handleWorkerMessage(protocol, sessions, session, message) { + if (session.completed) { + return; + } + + if (message.type === 'content_item') { + session.content_items.push(cloneJsonValue(message.item)); + return; + } + + if (message.type === 'set_yield_time') { + scheduleInitialYield(protocol, session, normalizeYieldTime(message.value ?? 0)); + return; + } + + if (message.type === 'set_max_output_tokens_per_exec_call') { + session.max_output_tokens_per_exec_call = normalizeMaxOutputTokensPerExecCall(message.value); + return; + } + + if (message.type === 'tool_call') { + void forwardToolCall(protocol, session, message); + return; + } + + if (message.type === 'result') { + await completeSession(protocol, sessions, session, { + type: 'result', + stored_values: cloneJsonValue(message.stored_values ?? {}), + error_text: + typeof message.error_text === 'string' ? message.error_text : undefined, + }); + return; + } + + process.stderr.write('Unknown worker message type: ' + message.type + '\n'); +} + +async function forwardToolCall(protocol, session, message) { + try { + const result = await protocol.request(session.id, 'tool_call', { + name: String(message.name), + input: message.input, + }); + if (session.completed) { + return; + } + try { + session.worker.postMessage({ + type: 'tool_response', + id: message.id, + result, + }); + } catch {} + } catch (error) { + if (session.completed) { + return; + } + try { + session.worker.postMessage({ + type: 'tool_response_error', + id: message.id, + error_text: formatErrorText(error), + }); + } catch {} + } +} + +async function sendYielded(protocol, session) { + if (session.completed) { + return; + } + const contentItems = takeContentItems(session); + try { + session.worker.postMessage({ type: 'clear_content' }); + } catch {} + await protocol.send({ + type: 'yielded', + session_id: session.id, + content_items: contentItems, + }); +} + +function scheduleInitialYield(protocol, session, yieldTime) { + if (session.completed || session.initial_yield_triggered) { + return yieldTime; + } + session.initial_yield_timer = clearTimer(session.initial_yield_timer); + session.initial_yield_timer = setTimeout(() => { + session.initial_yield_timer = null; + session.initial_yield_triggered = true; + void sendYielded(protocol, session); + }, yieldTime); + return yieldTime; +} + +function schedulePollYield(protocol, session, yieldTime) { + if (session.completed) { + return; + } + session.poll_yield_timer = clearTimer(session.poll_yield_timer); + session.poll_yield_timer = setTimeout(() => { + session.poll_yield_timer = null; + void sendYielded(protocol, session); + }, yieldTime); +} + +async function completeSession(protocol, sessions, session, message) { + if (session.completed) { + return; + } + session.completed = true; + session.initial_yield_timer = clearTimer(session.initial_yield_timer); + session.poll_yield_timer = clearTimer(session.poll_yield_timer); + sessions.delete(session.id); + const contentItems = takeContentItems(session); + try { + session.worker.postMessage({ type: 'clear_content' }); + } catch {} + await protocol.send({ + ...message, + session_id: session.id, + content_items: contentItems, + max_output_tokens_per_exec_call: session.max_output_tokens_per_exec_call, + }); +} + +async function terminateSession(protocol, sessions, session) { + if (session.completed) { + return; + } + session.completed = true; + session.initial_yield_timer = clearTimer(session.initial_yield_timer); + session.poll_yield_timer = clearTimer(session.poll_yield_timer); + sessions.delete(session.id); + const contentItems = takeContentItems(session); + try { + await session.worker.terminate(); + } catch {} + await protocol.send({ + type: 'terminated', + session_id: session.id, + content_items: contentItems, + }); } async function main() { const protocol = createProtocol(); - const request = await protocol.init; - const state = { - maxOutputTokensPerExecCall: DEFAULT_MAX_OUTPUT_TOKENS_PER_EXEC_CALL, - storedValues: cloneJsonValue(request.stored_values ?? {}), - }; - const callTool = createToolCaller(protocol); - const context = vm.createContext({ - __codexContentItems: [], - __codex_tool_call: callTool, - }); - - try { - await runModule(context, request, state, callTool); - await protocol.send({ - type: 'result', - content_items: readContentItems(context), - stored_values: state.storedValues, - max_output_tokens_per_exec_call: state.maxOutputTokensPerExecCall, - }); - process.exit(0); - } catch (error) { - await protocol.send({ - type: 'result', - content_items: readContentItems(context), - stored_values: state.storedValues, - error_text: formatErrorText(error), - max_output_tokens_per_exec_call: state.maxOutputTokensPerExecCall, - }); - process.exit(1); - } + await protocol.closed; } void main().catch(async (error) => { try { - process.stderr.write(`${formatErrorText(error)}\n`); + process.stderr.write(formatErrorText(error) + '\n'); } finally { process.exitCode = 1; } diff --git a/codex-rs/core/src/tools/handlers/code_mode.rs b/codex-rs/core/src/tools/handlers/code_mode.rs index 4763a69b46..fe4a23965d 100644 --- a/codex-rs/core/src/tools/handlers/code_mode.rs +++ b/codex-rs/core/src/tools/handlers/code_mode.rs @@ -1,16 +1,35 @@ use async_trait::async_trait; +use serde::Deserialize; -use crate::features::Feature; use crate::function_tool::FunctionCallError; use crate::tools::code_mode; +use crate::tools::code_mode::DEFAULT_WAIT_YIELD_TIME_MS; use crate::tools::code_mode::PUBLIC_TOOL_NAME; +use crate::tools::code_mode::WAIT_TOOL_NAME; use crate::tools::context::FunctionToolOutput; use crate::tools::context::ToolInvocation; use crate::tools::context::ToolPayload; +use crate::tools::handlers::parse_arguments; use crate::tools::registry::ToolHandler; use crate::tools::registry::ToolKind; pub struct CodeModeHandler; +pub struct CodeModeWaitHandler; + +#[derive(Debug, Deserialize)] +struct ExecWaitArgs { + session_id: i32, + #[serde(default = "default_wait_yield_time_ms")] + yield_time_ms: u64, + #[serde(default)] + max_tokens: Option, + #[serde(default)] + terminate: bool, +} + +fn default_wait_yield_time_ms() -> u64 { + DEFAULT_WAIT_YIELD_TIME_MS +} #[async_trait] impl ToolHandler for CodeModeHandler { @@ -29,25 +48,57 @@ impl ToolHandler for CodeModeHandler { session, turn, tracker, + tool_name, payload, .. } = invocation; - if !session.features().enabled(Feature::CodeMode) { - return Err(FunctionCallError::RespondToModel(format!( - "{PUBLIC_TOOL_NAME} is disabled by feature flag" - ))); - } - - let code = match payload { - ToolPayload::Custom { input } => input, - _ => { - return Err(FunctionCallError::RespondToModel(format!( - "{PUBLIC_TOOL_NAME} expects raw JavaScript source text" - ))); + match payload { + ToolPayload::Custom { input } if tool_name == PUBLIC_TOOL_NAME => { + code_mode::execute(session, turn, tracker, input).await } - }; - - code_mode::execute(session, turn, tracker, code).await + _ => Err(FunctionCallError::RespondToModel(format!( + "{PUBLIC_TOOL_NAME} expects raw JavaScript source text" + ))), + } + } +} + +#[async_trait] +impl ToolHandler for CodeModeWaitHandler { + type Output = FunctionToolOutput; + + fn kind(&self) -> ToolKind { + ToolKind::Function + } + + async fn handle(&self, invocation: ToolInvocation) -> Result { + let ToolInvocation { + session, + turn, + tracker, + tool_name, + payload, + .. + } = invocation; + + match payload { + ToolPayload::Function { arguments } if tool_name == WAIT_TOOL_NAME => { + let args: ExecWaitArgs = parse_arguments(&arguments)?; + code_mode::wait( + session, + turn, + tracker, + args.session_id, + args.yield_time_ms, + args.max_tokens, + args.terminate, + ) + .await + } + _ => Err(FunctionCallError::RespondToModel(format!( + "{WAIT_TOOL_NAME} expects JSON arguments" + ))), + } } } diff --git a/codex-rs/core/src/tools/handlers/mod.rs b/codex-rs/core/src/tools/handlers/mod.rs index 2177800463..068031b5a1 100644 --- a/codex-rs/core/src/tools/handlers/mod.rs +++ b/codex-rs/core/src/tools/handlers/mod.rs @@ -35,6 +35,7 @@ use crate::sandboxing::normalize_additional_permissions; pub use apply_patch::ApplyPatchHandler; pub use artifacts::ArtifactsHandler; pub use code_mode::CodeModeHandler; +pub use code_mode::CodeModeWaitHandler; use codex_protocol::models::PermissionProfile; use codex_protocol::protocol::AskForApproval; pub use dynamic::DynamicToolHandler; diff --git a/codex-rs/core/src/tools/spec.rs b/codex-rs/core/src/tools/spec.rs index afa4c18617..7d95afaebb 100644 --- a/codex-rs/core/src/tools/spec.rs +++ b/codex-rs/core/src/tools/spec.rs @@ -8,7 +8,9 @@ use crate::features::Features; use crate::mcp_connection_manager::ToolInfo; use crate::models_manager::collaboration_mode_presets::CollaborationModesConfig; use crate::original_image_detail::can_request_original_image_detail; +use crate::tools::code_mode::DEFAULT_WAIT_YIELD_TIME_MS; use crate::tools::code_mode::PUBLIC_TOOL_NAME; +use crate::tools::code_mode::WAIT_TOOL_NAME; use crate::tools::code_mode_description::augment_tool_spec_for_code_mode; use crate::tools::discoverable::DiscoverablePluginInfo; use crate::tools::discoverable::DiscoverableTool; @@ -589,6 +591,55 @@ fn create_write_stdin_tool() -> ToolSpec { }) } +fn create_exec_wait_tool() -> ToolSpec { + let properties = BTreeMap::from([ + ( + "session_id".to_string(), + JsonSchema::Number { + description: Some("Identifier of the running exec session.".to_string()), + }, + ), + ( + "yield_time_ms".to_string(), + JsonSchema::Number { + description: Some( + "How long to wait (in milliseconds) for more output before yielding again." + .to_string(), + ), + }, + ), + ( + "max_tokens".to_string(), + JsonSchema::Number { + description: Some( + "Maximum number of output tokens to return for this wait call.".to_string(), + ), + }, + ), + ( + "terminate".to_string(), + JsonSchema::Boolean { + description: Some("Whether to terminate the running exec session.".to_string()), + }, + ), + ]); + + ToolSpec::Function(ResponsesApiTool { + name: WAIT_TOOL_NAME.to_string(), + description: format!( + "Waits on a yielded `{PUBLIC_TOOL_NAME}` session and returns new output or completion." + ), + strict: false, + parameters: JsonSchema::Object { + properties, + required: Some(vec!["session_id".to_string()]), + additional_properties: Some(false.into()), + }, + output_schema: None, + defer_loading: None, + }) +} + fn create_shell_tool(request_permission_enabled: bool) -> ToolSpec { let mut properties = BTreeMap::from([ ( @@ -1832,7 +1883,7 @@ source: /[\s\S]+/ enabled_tool_names.join(", ") }; let description = format!( - "Runs JavaScript in a Node-backed `node:vm` context. This is a freeform tool: send raw JavaScript source text (no JSON/quotes/markdown fences). Direct tool calls remain available while `{PUBLIC_TOOL_NAME}` is enabled. Inside JavaScript, import nested tools from `tools.js`, for example `import {{ exec_command }} from \"tools.js\"` or `import {{ ALL_TOOLS }} from \"tools.js\"` to inspect the available `{{ module, name, description }}` entries. Namespaced tools are also available from `tools/.js`; MCP tools use `tools/mcp/.js`, for example `import {{ append_notebook_logs_chart }} from \"tools/mcp/ologs.js\"`. Nested tool calls resolve to their code-mode result values. Import `{{ output_text, output_image, set_max_output_tokens_per_exec_call, store, load }}` from `\"@openai/code_mode\"` (or `\"openai/code_mode\"`); `output_text(value)` surfaces text back to the model and stringifies non-string objects when possible, `output_image(imageUrl)` appends an `input_image` content item for `http(s)` or `data:` URLs, `store(key, value)` persists JSON-serializable values across `{PUBLIC_TOOL_NAME}` calls in the current session, `load(key)` returns a cloned stored value or `undefined`, and `set_max_output_tokens_per_exec_call(value)` sets the token budget used to truncate the final Rust-side result of the current `{PUBLIC_TOOL_NAME}` execution. The default is `10000`. This guards the overall `{PUBLIC_TOOL_NAME}` output, not individual nested tool invocations. The returned content starts with a separate `Script completed` or `Script failed` text item that includes wall time. When truncation happens, the final text may include `Total output lines:` and the usual `…N tokens truncated…` marker. Function tools require JSON object arguments. Freeform tools require raw strings. `add_content(value)` remains available for compatibility with a content item, content-item array, or string. Structured nested-tool results should be converted to text first, for example with `JSON.stringify(...)`. Only content passed to `output_text(...)`, `output_image(...)`, or `add_content(value)` is surfaced back to the model. Enabled nested tools: {enabled_list}." + "Runs JavaScript in a Node-backed `node:vm` context. This is a freeform tool: send raw JavaScript source text (no JSON/quotes/markdown fences). Direct tool calls remain available while `{PUBLIC_TOOL_NAME}` is enabled. Inside JavaScript, import nested tools from `tools.js`, for example `import {{ exec_command }} from \"tools.js\"` or `import {{ ALL_TOOLS }} from \"tools.js\"` to inspect the available `{{ module, name, description }}` entries. Namespaced tools are also available from `tools/.js`; MCP tools use `tools/mcp/.js`, for example `import {{ append_notebook_logs_chart }} from \"tools/mcp/ologs.js\"`. Nested tool calls resolve to their code-mode result values. Import `{{ output_text, output_image, set_max_output_tokens_per_exec_call, set_yield_time, store, load }}` from `\"@openai/code_mode\"` (or `\"openai/code_mode\"`); `output_text(value)` surfaces text back to the model and stringifies non-string objects when possible, `output_image(imageUrl)` appends an `input_image` content item for `http(s)` or `data:` URLs, `store(key, value)` persists JSON-serializable values across `{PUBLIC_TOOL_NAME}` calls in the current session, `load(key)` returns a cloned stored value or `undefined`, `set_max_output_tokens_per_exec_call(value)` sets the token budget used to truncate direct `{PUBLIC_TOOL_NAME}` returns, and `{WAIT_TOOL_NAME}` uses its own `max_tokens` argument with a default of `10000`. `set_yield_time(value)` asks `{PUBLIC_TOOL_NAME}` to return early if the script is still running after that many milliseconds so `{WAIT_TOOL_NAME}` can resume it later. The default wait timeout for `{WAIT_TOOL_NAME}` is {DEFAULT_WAIT_YIELD_TIME_MS}. The returned content starts with a separate `Script completed`, `Script failed`, or `Script running with session ID …` text item that includes wall time. When truncation happens, the final text may include `Total output lines:` and the usual `…N tokens truncated…` marker. Function tools require JSON object arguments. Freeform tools require raw strings. `add_content(value)` remains available for compatibility with a content item, content-item array, or string. Structured nested-tool results should be converted to text first, for example with `JSON.stringify(...)`. Only content passed to `output_text(...)`, `output_image(...)`, or `add_content(value)` is surfaced back to the model. Enabled nested tools: {enabled_list}." ); ToolSpec::Freeform(FreeformTool { @@ -1847,7 +1898,9 @@ source: /[\s\S]+/ } fn is_code_mode_nested_tool(spec: &ToolSpec) -> bool { - spec.name() != PUBLIC_TOOL_NAME && matches!(spec, ToolSpec::Function(_) | ToolSpec::Freeform(_)) + spec.name() != PUBLIC_TOOL_NAME + && spec.name() != WAIT_TOOL_NAME + && matches!(spec, ToolSpec::Function(_) | ToolSpec::Freeform(_)) } fn create_list_mcp_resources_tool() -> ToolSpec { @@ -2243,6 +2296,7 @@ pub(crate) fn build_specs_with_discoverable_tools( use crate::tools::handlers::ApplyPatchHandler; use crate::tools::handlers::ArtifactsHandler; use crate::tools::handlers::CodeModeHandler; + use crate::tools::handlers::CodeModeWaitHandler; use crate::tools::handlers::DynamicToolHandler; use crate::tools::handlers::GrepFilesHandler; use crate::tools::handlers::JsReplHandler; @@ -2281,6 +2335,7 @@ pub(crate) fn build_specs_with_discoverable_tools( }); let tool_suggest_handler = Arc::new(ToolSuggestHandler); let code_mode_handler = Arc::new(CodeModeHandler); + let code_mode_wait_handler = Arc::new(CodeModeWaitHandler); let js_repl_handler = Arc::new(JsReplHandler); let js_repl_reset_handler = Arc::new(JsReplResetHandler); let artifacts_handler = Arc::new(ArtifactsHandler); @@ -2311,6 +2366,13 @@ pub(crate) fn build_specs_with_discoverable_tools( config.code_mode_enabled, ); builder.register_handler(PUBLIC_TOOL_NAME, code_mode_handler); + push_tool_spec( + &mut builder, + create_exec_wait_tool(), + false, + config.code_mode_enabled, + ); + builder.register_handler(WAIT_TOOL_NAME, code_mode_wait_handler); } match &config.shell_type { diff --git a/codex-rs/core/tests/suite/code_mode.rs b/codex-rs/core/tests/suite/code_mode.rs index 07cadc3431..23fcd9c089 100644 --- a/codex-rs/core/tests/suite/code_mode.rs +++ b/codex-rs/core/tests/suite/code_mode.rs @@ -21,6 +21,7 @@ use pretty_assertions::assert_eq; use serde_json::Value; use std::collections::HashMap; use std::fs; +use std::path::Path; use std::time::Duration; use wiremock::MockServer; @@ -32,6 +33,16 @@ fn custom_tool_output_items(req: &ResponsesRequest, call_id: &str) -> Vec .clone() } +fn function_tool_output_items(req: &ResponsesRequest, call_id: &str) -> Vec { + match req.function_call_output(call_id).get("output") { + Some(Value::Array(items)) => items.clone(), + Some(Value::String(text)) => { + vec![serde_json::json!({ "type": "input_text", "text": text })] + } + _ => panic!("function tool output should be serialized as text or content items"), + } +} + fn text_item(items: &[Value], index: usize) -> &str { items[index] .get("text") @@ -39,6 +50,23 @@ fn text_item(items: &[Value], index: usize) -> &str { .expect("content item should be input_text") } +fn extract_running_session_id(text: &str) -> i32 { + text.strip_prefix("Script running with session ID ") + .and_then(|rest| rest.split('\n').next()) + .expect("running header should contain a session ID") + .parse() + .expect("session ID should parse as i32") +} + +fn wait_for_file_source(path: &Path) -> Result { + let quoted_path = shlex::try_join([path.to_string_lossy().as_ref()])?; + let command = format!("if [ -f {quoted_path} ]; then printf ready; fi"); + Ok(format!( + r#"while ((await exec_command({{ cmd: {command:?} }})).output !== "ready") {{ +}}"# + )) +} + fn custom_tool_output_body_and_success( req: &ResponsesRequest, call_id: &str, @@ -289,6 +317,799 @@ Error:\ boom\n Ok(()) } +#[cfg_attr(windows, ignore = "no exec_command on Windows")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn code_mode_can_yield_and_resume_with_exec_wait() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let mut builder = test_codex().with_config(move |config| { + let _ = config.features.enable(Feature::CodeMode); + }); + let test = builder.build(&server).await?; + let phase_2_gate = test.workspace_path("code-mode-phase-2.ready"); + let phase_3_gate = test.workspace_path("code-mode-phase-3.ready"); + let phase_2_wait = wait_for_file_source(&phase_2_gate)?; + let phase_3_wait = wait_for_file_source(&phase_3_gate)?; + + let code = format!( + r#" +import {{ output_text, set_yield_time }} from "@openai/code_mode"; +import {{ exec_command }} from "tools.js"; + +output_text("phase 1"); +set_yield_time(10); +{phase_2_wait} +output_text("phase 2"); +{phase_3_wait} +output_text("phase 3"); +"# + ); + + responses::mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-1"), + ev_custom_tool_call("call-1", "exec", &code), + ev_completed("resp-1"), + ]), + ) + .await; + let first_completion = responses::mount_sse_once( + &server, + sse(vec![ + ev_assistant_message("msg-1", "waiting"), + ev_completed("resp-2"), + ]), + ) + .await; + + test.submit_turn("start the long exec").await?; + + let first_request = first_completion.single_request(); + let first_items = custom_tool_output_items(&first_request, "call-1"); + assert_eq!(first_items.len(), 2); + assert_regex_match( + concat!( + r"(?s)\A", + r"Script running with session ID \d+\nWall time \d+\.\d seconds\nOutput:\n\z" + ), + text_item(&first_items, 0), + ); + assert_eq!(text_item(&first_items, 1), "phase 1"); + let session_id = extract_running_session_id(text_item(&first_items, 0)); + + responses::mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-3"), + responses::ev_function_call( + "call-2", + "exec_wait", + &serde_json::to_string(&serde_json::json!({ + "session_id": session_id, + "yield_time_ms": 1_000, + }))?, + ), + ev_completed("resp-3"), + ]), + ) + .await; + let second_completion = responses::mount_sse_once( + &server, + sse(vec![ + ev_assistant_message("msg-2", "still waiting"), + ev_completed("resp-4"), + ]), + ) + .await; + + fs::write(&phase_2_gate, "ready")?; + test.submit_turn("wait again").await?; + + let second_request = second_completion.single_request(); + let second_items = function_tool_output_items(&second_request, "call-2"); + assert_eq!(second_items.len(), 2); + assert_regex_match( + concat!( + r"(?s)\A", + r"Script running with session ID \d+\nWall time \d+\.\d seconds\nOutput:\n\z" + ), + text_item(&second_items, 0), + ); + assert_eq!( + extract_running_session_id(text_item(&second_items, 0)), + session_id + ); + assert_eq!(text_item(&second_items, 1), "phase 2"); + + responses::mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-5"), + responses::ev_function_call( + "call-3", + "exec_wait", + &serde_json::to_string(&serde_json::json!({ + "session_id": session_id, + "yield_time_ms": 1_000, + }))?, + ), + ev_completed("resp-5"), + ]), + ) + .await; + let third_completion = responses::mount_sse_once( + &server, + sse(vec![ + ev_assistant_message("msg-3", "done"), + ev_completed("resp-6"), + ]), + ) + .await; + + fs::write(&phase_3_gate, "ready")?; + test.submit_turn("wait for completion").await?; + + let third_request = third_completion.single_request(); + let third_items = function_tool_output_items(&third_request, "call-3"); + assert_eq!(third_items.len(), 2); + assert_regex_match( + concat!( + r"(?s)\A", + r"Script completed\nWall time \d+\.\d seconds\nOutput:\n\z" + ), + text_item(&third_items, 0), + ); + assert_eq!(text_item(&third_items, 1), "phase 3"); + + Ok(()) +} + +#[cfg_attr(windows, ignore = "no exec_command on Windows")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn code_mode_can_run_multiple_yielded_sessions() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let mut builder = test_codex().with_config(move |config| { + let _ = config.features.enable(Feature::CodeMode); + }); + let test = builder.build(&server).await?; + let session_a_gate = test.workspace_path("code-mode-session-a.ready"); + let session_b_gate = test.workspace_path("code-mode-session-b.ready"); + let session_a_wait = wait_for_file_source(&session_a_gate)?; + let session_b_wait = wait_for_file_source(&session_b_gate)?; + + let session_a_code = format!( + r#" +import {{ output_text, set_yield_time }} from "@openai/code_mode"; +import {{ exec_command }} from "tools.js"; + +output_text("session a start"); +set_yield_time(10); +{session_a_wait} +output_text("session a done"); +"# + ); + let session_b_code = format!( + r#" +import {{ output_text, set_yield_time }} from "@openai/code_mode"; +import {{ exec_command }} from "tools.js"; + +output_text("session b start"); +set_yield_time(10); +{session_b_wait} +output_text("session b done"); +"# + ); + + responses::mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-1"), + ev_custom_tool_call("call-1", "exec", &session_a_code), + ev_completed("resp-1"), + ]), + ) + .await; + let first_completion = responses::mount_sse_once( + &server, + sse(vec![ + ev_assistant_message("msg-1", "session a waiting"), + ev_completed("resp-2"), + ]), + ) + .await; + + test.submit_turn("start session a").await?; + + let first_request = first_completion.single_request(); + let first_items = custom_tool_output_items(&first_request, "call-1"); + assert_eq!(first_items.len(), 2); + let session_a_id = extract_running_session_id(text_item(&first_items, 0)); + assert_eq!(text_item(&first_items, 1), "session a start"); + + responses::mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-3"), + ev_custom_tool_call("call-2", "exec", &session_b_code), + ev_completed("resp-3"), + ]), + ) + .await; + let second_completion = responses::mount_sse_once( + &server, + sse(vec![ + ev_assistant_message("msg-2", "session b waiting"), + ev_completed("resp-4"), + ]), + ) + .await; + + test.submit_turn("start session b").await?; + + let second_request = second_completion.single_request(); + let second_items = custom_tool_output_items(&second_request, "call-2"); + assert_eq!(second_items.len(), 2); + let session_b_id = extract_running_session_id(text_item(&second_items, 0)); + assert_eq!(text_item(&second_items, 1), "session b start"); + assert_ne!(session_a_id, session_b_id); + + fs::write(&session_a_gate, "ready")?; + responses::mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-5"), + responses::ev_function_call( + "call-3", + "exec_wait", + &serde_json::to_string(&serde_json::json!({ + "session_id": session_a_id, + "yield_time_ms": 1_000, + }))?, + ), + ev_completed("resp-5"), + ]), + ) + .await; + let third_completion = responses::mount_sse_once( + &server, + sse(vec![ + ev_assistant_message("msg-3", "session a done"), + ev_completed("resp-6"), + ]), + ) + .await; + + test.submit_turn("wait session a").await?; + + let third_request = third_completion.single_request(); + let third_items = function_tool_output_items(&third_request, "call-3"); + assert_eq!(third_items.len(), 2); + assert_regex_match( + concat!( + r"(?s)\A", + r"Script completed\nWall time \d+\.\d seconds\nOutput:\n\z" + ), + text_item(&third_items, 0), + ); + assert_eq!(text_item(&third_items, 1), "session a done"); + + fs::write(&session_b_gate, "ready")?; + responses::mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-7"), + responses::ev_function_call( + "call-4", + "exec_wait", + &serde_json::to_string(&serde_json::json!({ + "session_id": session_b_id, + "yield_time_ms": 1_000, + }))?, + ), + ev_completed("resp-7"), + ]), + ) + .await; + let fourth_completion = responses::mount_sse_once( + &server, + sse(vec![ + ev_assistant_message("msg-4", "session b done"), + ev_completed("resp-8"), + ]), + ) + .await; + + test.submit_turn("wait session b").await?; + + let fourth_request = fourth_completion.single_request(); + let fourth_items = function_tool_output_items(&fourth_request, "call-4"); + assert_eq!(fourth_items.len(), 2); + assert_regex_match( + concat!( + r"(?s)\A", + r"Script completed\nWall time \d+\.\d seconds\nOutput:\n\z" + ), + text_item(&fourth_items, 0), + ); + assert_eq!(text_item(&fourth_items, 1), "session b done"); + + Ok(()) +} + +#[cfg_attr(windows, ignore = "no exec_command on Windows")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn code_mode_exec_wait_can_terminate_and_continue() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let mut builder = test_codex().with_config(move |config| { + let _ = config.features.enable(Feature::CodeMode); + }); + let test = builder.build(&server).await?; + let termination_gate = test.workspace_path("code-mode-terminate.ready"); + let termination_wait = wait_for_file_source(&termination_gate)?; + + let code = format!( + r#" +import {{ output_text, set_yield_time }} from "@openai/code_mode"; +import {{ exec_command }} from "tools.js"; + +output_text("phase 1"); +set_yield_time(10); +{termination_wait} +output_text("phase 2"); +"# + ); + + responses::mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-1"), + ev_custom_tool_call("call-1", "exec", &code), + ev_completed("resp-1"), + ]), + ) + .await; + let first_completion = responses::mount_sse_once( + &server, + sse(vec![ + ev_assistant_message("msg-1", "waiting"), + ev_completed("resp-2"), + ]), + ) + .await; + + test.submit_turn("start the long exec").await?; + + let first_request = first_completion.single_request(); + let first_items = custom_tool_output_items(&first_request, "call-1"); + assert_eq!(first_items.len(), 2); + let session_id = extract_running_session_id(text_item(&first_items, 0)); + assert_eq!(text_item(&first_items, 1), "phase 1"); + + responses::mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-3"), + responses::ev_function_call( + "call-2", + "exec_wait", + &serde_json::to_string(&serde_json::json!({ + "session_id": session_id, + "terminate": true, + }))?, + ), + ev_completed("resp-3"), + ]), + ) + .await; + let second_completion = responses::mount_sse_once( + &server, + sse(vec![ + ev_assistant_message("msg-2", "terminated"), + ev_completed("resp-4"), + ]), + ) + .await; + + test.submit_turn("terminate it").await?; + + let second_request = second_completion.single_request(); + let second_items = function_tool_output_items(&second_request, "call-2"); + assert_eq!(second_items.len(), 1); + assert_regex_match( + concat!( + r"(?s)\A", + r"Script terminated\nWall time \d+\.\d seconds\nOutput:\n\z" + ), + text_item(&second_items, 0), + ); + + responses::mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-5"), + ev_custom_tool_call( + "call-3", + "exec", + r#" +import { output_text } from "@openai/code_mode"; + +output_text("after terminate"); +"#, + ), + ev_completed("resp-5"), + ]), + ) + .await; + let third_completion = responses::mount_sse_once( + &server, + sse(vec![ + ev_assistant_message("msg-3", "done"), + ev_completed("resp-6"), + ]), + ) + .await; + + test.submit_turn("run another exec").await?; + + let third_request = third_completion.single_request(); + let third_items = custom_tool_output_items(&third_request, "call-3"); + assert_eq!(third_items.len(), 2); + assert_regex_match( + concat!( + r"(?s)\A", + r"Script completed\nWall time \d+\.\d seconds\nOutput:\n\z" + ), + text_item(&third_items, 0), + ); + assert_eq!(text_item(&third_items, 1), "after terminate"); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn code_mode_exec_wait_returns_error_for_unknown_session() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let mut builder = test_codex().with_config(move |config| { + let _ = config.features.enable(Feature::CodeMode); + }); + let test = builder.build(&server).await?; + + responses::mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-1"), + responses::ev_function_call( + "call-1", + "exec_wait", + &serde_json::to_string(&serde_json::json!({ + "session_id": 999_999, + "yield_time_ms": 1_000, + }))?, + ), + ev_completed("resp-1"), + ]), + ) + .await; + let completion = responses::mount_sse_once( + &server, + sse(vec![ + ev_assistant_message("msg-1", "done"), + ev_completed("resp-2"), + ]), + ) + .await; + + test.submit_turn("wait on an unknown exec session").await?; + + let request = completion.single_request(); + let (_, success) = request + .function_call_output_content_and_success("call-1") + .expect("function tool output should be present"); + assert_ne!(success, Some(true)); + + let items = function_tool_output_items(&request, "call-1"); + assert_eq!(items.len(), 2); + assert_regex_match( + concat!( + r"(?s)\A", + r"Script failed\nWall time \d+\.\d seconds\nOutput:\n\z" + ), + text_item(&items, 0), + ); + assert_eq!( + text_item(&items, 1), + "Script error:\nexec session 999999 not found" + ); + + Ok(()) +} + +#[cfg_attr(windows, ignore = "no exec_command on Windows")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn code_mode_exec_wait_terminate_returns_completed_session_if_it_finished_in_background() +-> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let mut builder = test_codex().with_config(move |config| { + let _ = config.features.enable(Feature::CodeMode); + }); + let test = builder.build(&server).await?; + let session_a_gate = test.workspace_path("code-mode-session-a-finished.ready"); + let session_b_gate = test.workspace_path("code-mode-session-b-blocked.ready"); + let session_a_wait = wait_for_file_source(&session_a_gate)?; + let session_b_wait = wait_for_file_source(&session_b_gate)?; + + let session_a_code = format!( + r#" +import {{ output_text, set_yield_time }} from "@openai/code_mode"; +import {{ exec_command }} from "tools.js"; + +output_text("session a start"); +set_yield_time(10); +{session_a_wait} +output_text("session a done"); +"# + ); + let session_b_code = format!( + r#" +import {{ output_text, set_yield_time }} from "@openai/code_mode"; +import {{ exec_command }} from "tools.js"; + +output_text("session b start"); +set_yield_time(10); +{session_b_wait} +output_text("session b done"); +"# + ); + + responses::mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-1"), + ev_custom_tool_call("call-1", "exec", &session_a_code), + ev_completed("resp-1"), + ]), + ) + .await; + let first_completion = responses::mount_sse_once( + &server, + sse(vec![ + ev_assistant_message("msg-1", "session a waiting"), + ev_completed("resp-2"), + ]), + ) + .await; + + test.submit_turn("start session a").await?; + + let first_request = first_completion.single_request(); + let first_items = custom_tool_output_items(&first_request, "call-1"); + assert_eq!(first_items.len(), 2); + let session_a_id = extract_running_session_id(text_item(&first_items, 0)); + assert_eq!(text_item(&first_items, 1), "session a start"); + + responses::mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-3"), + ev_custom_tool_call("call-2", "exec", &session_b_code), + ev_completed("resp-3"), + ]), + ) + .await; + let second_completion = responses::mount_sse_once( + &server, + sse(vec![ + ev_assistant_message("msg-2", "session b waiting"), + ev_completed("resp-4"), + ]), + ) + .await; + + test.submit_turn("start session b").await?; + + let second_request = second_completion.single_request(); + let second_items = custom_tool_output_items(&second_request, "call-2"); + assert_eq!(second_items.len(), 2); + let session_b_id = extract_running_session_id(text_item(&second_items, 0)); + assert_eq!(text_item(&second_items, 1), "session b start"); + + fs::write(&session_a_gate, "ready")?; + responses::mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-5"), + responses::ev_function_call( + "call-3", + "exec_wait", + &serde_json::to_string(&serde_json::json!({ + "session_id": session_b_id, + "yield_time_ms": 1_000, + }))?, + ), + ev_completed("resp-5"), + ]), + ) + .await; + let third_completion = responses::mount_sse_once( + &server, + sse(vec![ + ev_assistant_message("msg-3", "session b still waiting"), + ev_completed("resp-6"), + ]), + ) + .await; + + test.submit_turn("wait session b").await?; + + let third_request = third_completion.single_request(); + let third_items = function_tool_output_items(&third_request, "call-3"); + assert_eq!(third_items.len(), 1); + assert_regex_match( + concat!( + r"(?s)\A", + r"Script running with session ID \d+\nWall time \d+\.\d seconds\nOutput:\n\z" + ), + text_item(&third_items, 0), + ); + assert_eq!( + extract_running_session_id(text_item(&third_items, 0)), + session_b_id + ); + + responses::mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-7"), + responses::ev_function_call( + "call-4", + "exec_wait", + &serde_json::to_string(&serde_json::json!({ + "session_id": session_a_id, + "terminate": true, + }))?, + ), + ev_completed("resp-7"), + ]), + ) + .await; + let fourth_completion = responses::mount_sse_once( + &server, + sse(vec![ + ev_assistant_message("msg-4", "session a already done"), + ev_completed("resp-8"), + ]), + ) + .await; + + test.submit_turn("terminate session a").await?; + + let fourth_request = fourth_completion.single_request(); + let fourth_items = function_tool_output_items(&fourth_request, "call-4"); + assert_eq!(fourth_items.len(), 1); + assert_regex_match( + concat!( + r"(?s)\A", + r"Script terminated\nWall time \d+\.\d seconds\nOutput:\n\z" + ), + text_item(&fourth_items, 0), + ); + + Ok(()) +} + +#[cfg_attr(windows, ignore = "no exec_command on Windows")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn code_mode_exec_wait_uses_its_own_max_tokens_budget() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let mut builder = test_codex().with_config(move |config| { + let _ = config.features.enable(Feature::CodeMode); + }); + let test = builder.build(&server).await?; + let completion_gate = test.workspace_path("code-mode-max-tokens.ready"); + let completion_wait = wait_for_file_source(&completion_gate)?; + + let code = format!( + r#" +import {{ output_text, set_max_output_tokens_per_exec_call, set_yield_time }} from "@openai/code_mode"; +import {{ exec_command }} from "tools.js"; + +output_text("phase 1"); +set_max_output_tokens_per_exec_call(100); +set_yield_time(10); +{completion_wait} +output_text("token one token two token three token four token five token six token seven"); +"# + ); + + responses::mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-1"), + ev_custom_tool_call("call-1", "exec", &code), + ev_completed("resp-1"), + ]), + ) + .await; + let first_completion = responses::mount_sse_once( + &server, + sse(vec![ + ev_assistant_message("msg-1", "waiting"), + ev_completed("resp-2"), + ]), + ) + .await; + + test.submit_turn("start the long exec").await?; + + let first_request = first_completion.single_request(); + let first_items = custom_tool_output_items(&first_request, "call-1"); + assert_eq!(first_items.len(), 2); + assert_eq!(text_item(&first_items, 1), "phase 1"); + let session_id = extract_running_session_id(text_item(&first_items, 0)); + + fs::write(&completion_gate, "ready")?; + responses::mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-3"), + responses::ev_function_call( + "call-2", + "exec_wait", + &serde_json::to_string(&serde_json::json!({ + "session_id": session_id, + "yield_time_ms": 1_000, + "max_tokens": 6, + }))?, + ), + ev_completed("resp-3"), + ]), + ) + .await; + let second_completion = responses::mount_sse_once( + &server, + sse(vec![ + ev_assistant_message("msg-2", "done"), + ev_completed("resp-4"), + ]), + ) + .await; + + test.submit_turn("wait for completion").await?; + + let second_request = second_completion.single_request(); + let second_items = function_tool_output_items(&second_request, "call-2"); + assert_eq!(second_items.len(), 2); + assert_regex_match( + concat!( + r"(?s)\A", + r"Script completed\nWall time \d+\.\d seconds\nOutput:\n\z" + ), + text_item(&second_items, 0), + ); + let expected_pattern = r#"(?sx) +\A +Total\ output\ lines:\ 1\n +\n +.*…\d+\ tokens\ truncated….* +\z +"#; + assert_regex_match(expected_pattern, text_item(&second_items, 1)); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn code_mode_can_output_serialized_text_via_openai_code_mode_module() -> Result<()> { skip_if_no_network!(Ok(())); From b5f927b97327ab62047dba9eaf0e998bfaa7dab3 Mon Sep 17 00:00:00 2001 From: xl-openai Date: Wed, 11 Mar 2026 23:18:58 -0700 Subject: [PATCH 5/6] feat: refactor on openai-curated plugins. (#14427) - Curated repo sync now uses GitHub HTTP, not local git. - Curated plugin cache/versioning now uses commit SHA instead of local. - Startup sync now always repairs or refreshes curated plugin cache from tmp (auto update to the lastest) --- .../app-server/tests/suite/v2/plugin_list.rs | 10 +- codex-rs/core/src/plugins/curated_repo.rs | 574 +++++++++++------- codex-rs/core/src/plugins/manager.rs | 295 ++++++++- codex-rs/core/src/plugins/marketplace.rs | 9 + codex-rs/core/src/plugins/mod.rs | 1 + codex-rs/core/src/plugins/store.rs | 190 +++++- 6 files changed, 820 insertions(+), 259 deletions(-) diff --git a/codex-rs/app-server/tests/suite/v2/plugin_list.rs b/codex-rs/app-server/tests/suite/v2/plugin_list.rs index eefed46958..0a5cc8936e 100644 --- a/codex-rs/app-server/tests/suite/v2/plugin_list.rs +++ b/codex-rs/app-server/tests/suite/v2/plugin_list.rs @@ -26,6 +26,7 @@ use wiremock::matchers::method; use wiremock::matchers::path; const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10); +const TEST_CURATED_PLUGIN_SHA: &str = "0123456789abcdef0123456789abcdef01234567"; #[tokio::test] async fn plugin_list_returns_invalid_request_for_invalid_marketplace_file() -> Result<()> { @@ -613,7 +614,9 @@ async fn plugin_list_force_remote_sync_reconciles_curated_plugin_state() -> Resu assert!( codex_home .path() - .join("plugins/cache/openai-curated/gmail/local") + .join(format!( + "plugins/cache/openai-curated/gmail/{TEST_CURATED_PLUGIN_SHA}" + )) .is_dir() ); assert!( @@ -706,5 +709,10 @@ fn write_openai_curated_marketplace( format!(r#"{{"name":"{plugin_name}"}}"#), )?; } + std::fs::create_dir_all(codex_home.join(".tmp"))?; + std::fs::write( + codex_home.join(".tmp/plugins.sha"), + format!("{TEST_CURATED_PLUGIN_SHA}\n"), + )?; Ok(()) } diff --git a/codex-rs/core/src/plugins/curated_repo.rs b/codex-rs/core/src/plugins/curated_repo.rs index 3ec492db30..41f8347992 100644 --- a/codex-rs/core/src/plugins/curated_repo.rs +++ b/codex-rs/core/src/plugins/curated_repo.rs @@ -1,30 +1,67 @@ +use crate::default_client::build_reqwest_client; +use reqwest::Client; +use serde::Deserialize; use std::fs; +use std::io::Cursor; +#[cfg(unix)] +use std::os::unix::fs::PermissionsExt; +use std::path::Component; use std::path::Path; use std::path::PathBuf; -use std::process::Command; -use std::process::Output; -use std::process::Stdio; -use std::thread; use std::time::Duration; -use std::time::Instant; +use zip::ZipArchive; -const OPENAI_PLUGINS_REPO_URL: &str = "https://github.com/openai/plugins.git"; +const GITHUB_API_BASE_URL: &str = "https://api.github.com"; +const GITHUB_API_ACCEPT_HEADER: &str = "application/vnd.github+json"; +const GITHUB_API_VERSION_HEADER: &str = "2022-11-28"; +const OPENAI_PLUGINS_OWNER: &str = "openai"; +const OPENAI_PLUGINS_REPO: &str = "plugins"; const CURATED_PLUGINS_RELATIVE_DIR: &str = ".tmp/plugins"; const CURATED_PLUGINS_SHA_FILE: &str = ".tmp/plugins.sha"; -const CURATED_PLUGINS_GIT_TIMEOUT: Duration = Duration::from_secs(30); +const CURATED_PLUGINS_HTTP_TIMEOUT: Duration = Duration::from_secs(30); + +#[derive(Debug, Deserialize)] +struct GitHubRepositorySummary { + default_branch: String, +} + +#[derive(Debug, Deserialize)] +struct GitHubGitRefSummary { + object: GitHubGitRefObject, +} + +#[derive(Debug, Deserialize)] +struct GitHubGitRefObject { + sha: String, +} pub(crate) fn curated_plugins_repo_path(codex_home: &Path) -> PathBuf { codex_home.join(CURATED_PLUGINS_RELATIVE_DIR) } -pub(crate) fn sync_openai_plugins_repo(codex_home: &Path) -> Result<(), String> { +pub(crate) fn read_curated_plugins_sha(codex_home: &Path) -> Option { + read_sha_file(codex_home.join(CURATED_PLUGINS_SHA_FILE).as_path()) +} + +pub(crate) fn sync_openai_plugins_repo(codex_home: &Path) -> Result { + sync_openai_plugins_repo_with_api_base_url(codex_home, GITHUB_API_BASE_URL) +} + +fn sync_openai_plugins_repo_with_api_base_url( + codex_home: &Path, + api_base_url: &str, +) -> Result { let repo_path = curated_plugins_repo_path(codex_home); let sha_path = codex_home.join(CURATED_PLUGINS_SHA_FILE); - let remote_sha = git_ls_remote_head_sha()?; - let local_sha = read_local_sha(&repo_path, &sha_path); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|err| format!("failed to create curated plugins sync runtime: {err}"))?; + let remote_sha = runtime.block_on(fetch_curated_repo_remote_sha(api_base_url))?; + let local_sha = read_sha_file(&sha_path); - if local_sha.as_deref() == Some(remote_sha.as_str()) && repo_path.join(".git").is_dir() { - return Ok(()); + if local_sha.as_deref() == Some(remote_sha.as_str()) && repo_path.is_dir() { + return Ok(remote_sha); } let Some(parent) = repo_path.parent() else { @@ -50,23 +87,18 @@ pub(crate) fn sync_openai_plugins_repo(codex_home: &Path) -> Result<(), String> ) })?; let cloned_repo_path = clone_dir.path().join("repo"); - let clone_output = run_git_command_with_timeout( - Command::new("git") - .env("GIT_OPTIONAL_LOCKS", "0") - .arg("clone") - .arg("--depth") - .arg("1") - .arg(OPENAI_PLUGINS_REPO_URL) - .arg(&cloned_repo_path), - "git clone curated plugins repo", - CURATED_PLUGINS_GIT_TIMEOUT, - )?; - ensure_git_success(&clone_output, "git clone curated plugins repo")?; + let zipball_bytes = runtime.block_on(fetch_curated_repo_zipball(api_base_url, &remote_sha))?; + extract_zipball_to_dir(&zipball_bytes, &cloned_repo_path)?; - let cloned_sha = git_head_sha(&cloned_repo_path)?; - if cloned_sha != remote_sha { + if !cloned_repo_path + .join(".agents/plugins/marketplace.json") + .is_file() + { return Err(format!( - "curated plugins clone HEAD mismatch: expected {remote_sha}, got {cloned_sha}" + "curated plugins archive missing marketplace manifest at {}", + cloned_repo_path + .join(".agents/plugins/marketplace.json") + .display() )); } @@ -123,156 +155,215 @@ pub(crate) fn sync_openai_plugins_repo(codex_home: &Path) -> Result<(), String> ) })?; } - fs::write(&sha_path, format!("{cloned_sha}\n")).map_err(|err| { + fs::write(&sha_path, format!("{remote_sha}\n")).map_err(|err| { format!( "failed to write curated plugins sha file {}: {err}", sha_path.display() ) })?; - Ok(()) + Ok(remote_sha) } -fn read_local_sha(repo_path: &Path, sha_path: &Path) -> Option { - if repo_path.join(".git").is_dir() - && let Ok(sha) = git_head_sha(repo_path) - { - return Some(sha); +async fn fetch_curated_repo_remote_sha(api_base_url: &str) -> Result { + let api_base_url = api_base_url.trim_end_matches('/'); + let repo_url = format!("{api_base_url}/repos/{OPENAI_PLUGINS_OWNER}/{OPENAI_PLUGINS_REPO}"); + let client = build_reqwest_client(); + let repo_body = fetch_github_text(&client, &repo_url, "get curated plugins repository").await?; + let repo_summary: GitHubRepositorySummary = + serde_json::from_str(&repo_body).map_err(|err| { + format!("failed to parse curated plugins repository response from {repo_url}: {err}") + })?; + if repo_summary.default_branch.is_empty() { + return Err(format!( + "curated plugins repository response from {repo_url} did not include a default branch" + )); } + let git_ref_url = format!("{repo_url}/git/ref/heads/{}", repo_summary.default_branch); + let git_ref_body = + fetch_github_text(&client, &git_ref_url, "get curated plugins HEAD ref").await?; + let git_ref: GitHubGitRefSummary = serde_json::from_str(&git_ref_body).map_err(|err| { + format!("failed to parse curated plugins ref response from {git_ref_url}: {err}") + })?; + if git_ref.object.sha.is_empty() { + return Err(format!( + "curated plugins ref response from {git_ref_url} did not include a HEAD sha" + )); + } + + Ok(git_ref.object.sha) +} + +async fn fetch_curated_repo_zipball( + api_base_url: &str, + remote_sha: &str, +) -> Result, String> { + let api_base_url = api_base_url.trim_end_matches('/'); + let repo_url = format!("{api_base_url}/repos/{OPENAI_PLUGINS_OWNER}/{OPENAI_PLUGINS_REPO}"); + let zipball_url = format!("{repo_url}/zipball/{remote_sha}"); + let client = build_reqwest_client(); + fetch_github_bytes(&client, &zipball_url, "download curated plugins archive").await +} + +async fn fetch_github_text(client: &Client, url: &str, context: &str) -> Result { + let response = github_request(client, url) + .send() + .await + .map_err(|err| format!("failed to {context} from {url}: {err}"))?; + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + if !status.is_success() { + return Err(format!( + "{context} from {url} failed with status {status}: {body}" + )); + } + Ok(body) +} + +async fn fetch_github_bytes(client: &Client, url: &str, context: &str) -> Result, String> { + let response = github_request(client, url) + .send() + .await + .map_err(|err| format!("failed to {context} from {url}: {err}"))?; + let status = response.status(); + let body = response + .bytes() + .await + .map_err(|err| format!("failed to read {context} response from {url}: {err}"))?; + if !status.is_success() { + let body_text = String::from_utf8_lossy(&body); + return Err(format!( + "{context} from {url} failed with status {status}: {body_text}" + )); + } + Ok(body.to_vec()) +} + +fn github_request(client: &Client, url: &str) -> reqwest::RequestBuilder { + client + .get(url) + .timeout(CURATED_PLUGINS_HTTP_TIMEOUT) + .header("accept", GITHUB_API_ACCEPT_HEADER) + .header("x-github-api-version", GITHUB_API_VERSION_HEADER) +} + +fn read_sha_file(sha_path: &Path) -> Option { fs::read_to_string(sha_path) .ok() .map(|sha| sha.trim().to_string()) .filter(|sha| !sha.is_empty()) } -fn git_ls_remote_head_sha() -> Result { - let output = run_git_command_with_timeout( - Command::new("git") - .env("GIT_OPTIONAL_LOCKS", "0") - .arg("ls-remote") - .arg(OPENAI_PLUGINS_REPO_URL) - .arg("HEAD"), - "git ls-remote curated plugins repo", - CURATED_PLUGINS_GIT_TIMEOUT, - )?; - ensure_git_success(&output, "git ls-remote curated plugins repo")?; +fn extract_zipball_to_dir(bytes: &[u8], destination: &Path) -> Result<(), String> { + fs::create_dir_all(destination).map_err(|err| { + format!( + "failed to create curated plugins extraction directory {}: {err}", + destination.display() + ) + })?; - let stdout = String::from_utf8_lossy(&output.stdout); - let Some(first_line) = stdout.lines().next() else { - return Err("git ls-remote returned empty output for curated plugins repo".to_string()); - }; - let Some((sha, _)) = first_line.split_once('\t') else { - return Err(format!( - "unexpected git ls-remote output for curated plugins repo: {first_line}" - )); - }; - if sha.is_empty() { - return Err("git ls-remote returned empty sha for curated plugins repo".to_string()); - } - Ok(sha.to_string()) -} + let cursor = Cursor::new(bytes); + let mut archive = ZipArchive::new(cursor) + .map_err(|err| format!("failed to open curated plugins zip archive: {err}"))?; -fn git_head_sha(repo_path: &Path) -> Result { - let output = Command::new("git") - .env("GIT_OPTIONAL_LOCKS", "0") - .arg("-C") - .arg(repo_path) - .arg("rev-parse") - .arg("HEAD") - .output() - .map_err(|err| { + for index in 0..archive.len() { + let mut entry = archive + .by_index(index) + .map_err(|err| format!("failed to read curated plugins zip entry: {err}"))?; + let Some(relative_path) = entry.enclosed_name() else { + return Err(format!( + "curated plugins zip entry `{}` escapes extraction root", + entry.name() + )); + }; + + let mut components = relative_path.components(); + let Some(Component::Normal(_)) = components.next() else { + continue; + }; + + let output_relative = components.fold(PathBuf::new(), |mut path, component| { + if let Component::Normal(segment) = component { + path.push(segment); + } + path + }); + if output_relative.as_os_str().is_empty() { + continue; + } + + let output_path = destination.join(&output_relative); + if entry.is_dir() { + fs::create_dir_all(&output_path).map_err(|err| { + format!( + "failed to create curated plugins directory {}: {err}", + output_path.display() + ) + })?; + continue; + } + + if let Some(parent) = output_path.parent() { + fs::create_dir_all(parent).map_err(|err| { + format!( + "failed to create curated plugins directory {}: {err}", + parent.display() + ) + })?; + } + let mut output = fs::File::create(&output_path).map_err(|err| { format!( - "failed to run git rev-parse HEAD in {}: {err}", - repo_path.display() + "failed to create curated plugins file {}: {err}", + output_path.display() ) })?; - ensure_git_success(&output, "git rev-parse HEAD")?; - - let sha = String::from_utf8_lossy(&output.stdout).trim().to_string(); - if sha.is_empty() { - return Err(format!( - "git rev-parse HEAD returned empty output in {}", - repo_path.display() - )); + std::io::copy(&mut entry, &mut output).map_err(|err| { + format!( + "failed to write curated plugins file {}: {err}", + output_path.display() + ) + })?; + apply_zip_permissions(&entry, &output_path)?; } - Ok(sha) + + Ok(()) } -fn run_git_command_with_timeout( - command: &mut Command, - context: &str, - timeout: Duration, -) -> Result { - let mut child = command - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .map_err(|err| format!("failed to run {context}: {err}"))?; - - let start = Instant::now(); - loop { - match child.try_wait() { - Ok(Some(_)) => { - return child - .wait_with_output() - .map_err(|err| format!("failed to wait for {context}: {err}")); - } - Ok(None) => {} - Err(err) => return Err(format!("failed to poll {context}: {err}")), - } - - if start.elapsed() >= timeout { - match child.try_wait() { - Ok(Some(_)) => { - return child - .wait_with_output() - .map_err(|err| format!("failed to wait for {context}: {err}")); - } - Ok(None) => {} - Err(err) => return Err(format!("failed to poll {context}: {err}")), - } - - let _ = child.kill(); - let output = child - .wait_with_output() - .map_err(|err| format!("failed to wait for {context} after timeout: {err}"))?; - let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); - return if stderr.is_empty() { - Err(format!("{context} timed out after {}s", timeout.as_secs())) - } else { - Err(format!( - "{context} timed out after {}s: {stderr}", - timeout.as_secs() - )) - }; - } - - thread::sleep(Duration::from_millis(100)); - } -} - -fn ensure_git_success(output: &Output, context: &str) -> Result<(), String> { - if output.status.success() { +#[cfg(unix)] +fn apply_zip_permissions(entry: &zip::read::ZipFile<'_>, output_path: &Path) -> Result<(), String> { + let Some(mode) = entry.unix_mode() else { return Ok(()); - } - let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); - if stderr.is_empty() { - Err(format!("{context} failed with status {}", output.status)) - } else { - Err(format!( - "{context} failed with status {}: {stderr}", - output.status - )) - } + }; + fs::set_permissions(output_path, fs::Permissions::from_mode(mode)).map_err(|err| { + format!( + "failed to set permissions on curated plugins file {}: {err}", + output_path.display() + ) + }) +} + +#[cfg(not(unix))] +fn apply_zip_permissions( + _entry: &zip::read::ZipFile<'_>, + _output_path: &Path, +) -> Result<(), String> { + Ok(()) } #[cfg(test)] mod tests { use super::*; use pretty_assertions::assert_eq; + use std::io::Write; use tempfile::tempdir; + use wiremock::Mock; + use wiremock::MockServer; + use wiremock::ResponseTemplate; + use wiremock::matchers::method; + use wiremock::matchers::path; + use zip::ZipWriter; + use zip::write::SimpleFileOptions; #[test] fn curated_plugins_repo_path_uses_codex_home_tmp_dir() { @@ -284,70 +375,145 @@ mod tests { } #[test] - fn read_local_sha_prefers_repo_head_when_available() { + fn read_curated_plugins_sha_reads_trimmed_sha_file() { let tmp = tempdir().expect("tempdir"); - let repo_path = tmp.path().join("repo"); - let sha_path = tmp.path().join("plugins.sha"); + fs::create_dir_all(tmp.path().join(".tmp")).expect("create tmp"); + fs::write(tmp.path().join(".tmp/plugins.sha"), "abc123\n").expect("write sha"); - fs::create_dir_all(&repo_path).expect("create repo dir"); - fs::write(&sha_path, "abc123\n").expect("write sha"); - let init_output = Command::new("git") - .arg("init") - .arg(&repo_path) - .output() - .expect("git init should run"); - ensure_git_success(&init_output, "git init").expect("git init should succeed"); - let config_name_output = Command::new("git") - .arg("-C") - .arg(&repo_path) - .arg("config") - .arg("user.name") - .arg("Codex") - .output() - .expect("git config user.name should run"); - ensure_git_success(&config_name_output, "git config user.name") - .expect("git config user.name should succeed"); - let config_email_output = Command::new("git") - .arg("-C") - .arg(&repo_path) - .arg("config") - .arg("user.email") - .arg("codex@example.com") - .output() - .expect("git config user.email should run"); - ensure_git_success(&config_email_output, "git config user.email") - .expect("git config user.email should succeed"); - fs::write(repo_path.join("README.md"), "demo\n").expect("write file"); - let add_output = Command::new("git") - .arg("-C") - .arg(&repo_path) - .arg("add") - .arg(".") - .output() - .expect("git add should run"); - ensure_git_success(&add_output, "git add").expect("git add should succeed"); - let commit_output = Command::new("git") - .arg("-C") - .arg(&repo_path) - .arg("commit") - .arg("-m") - .arg("init") - .output() - .expect("git commit should run"); - ensure_git_success(&commit_output, "git commit").expect("git commit should succeed"); - - let sha = read_local_sha(&repo_path, &sha_path); - assert_eq!(sha, Some(git_head_sha(&repo_path).expect("repo head sha"))); + assert_eq!( + read_curated_plugins_sha(tmp.path()).as_deref(), + Some("abc123") + ); } - #[test] - fn read_local_sha_falls_back_to_sha_file() { + #[tokio::test] + async fn sync_openai_plugins_repo_downloads_zipball_and_records_sha() { let tmp = tempdir().expect("tempdir"); - let repo_path = tmp.path().join("repo"); - let sha_path = tmp.path().join("plugins.sha"); - fs::write(&sha_path, "abc123\n").expect("write sha"); + let server = MockServer::start().await; + let sha = "0123456789abcdef0123456789abcdef01234567"; - let sha = read_local_sha(&repo_path, &sha_path); - assert_eq!(sha.as_deref(), Some("abc123")); + Mock::given(method("GET")) + .and(path("/repos/openai/plugins")) + .respond_with( + ResponseTemplate::new(200).set_body_string(r#"{"default_branch":"main"}"#), + ) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/repos/openai/plugins/git/ref/heads/main")) + .respond_with( + ResponseTemplate::new(200) + .set_body_string(format!(r#"{{"object":{{"sha":"{sha}"}}}}"#)), + ) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path(format!("/repos/openai/plugins/zipball/{sha}"))) + .respond_with( + ResponseTemplate::new(200) + .insert_header("content-type", "application/zip") + .set_body_bytes(curated_repo_zipball_bytes(sha)), + ) + .mount(&server) + .await; + + let server_uri = server.uri(); + let tmp_path = tmp.path().to_path_buf(); + tokio::task::spawn_blocking(move || { + sync_openai_plugins_repo_with_api_base_url(tmp_path.as_path(), &server_uri) + }) + .await + .expect("sync task should join") + .expect("sync should succeed"); + + let repo_path = curated_plugins_repo_path(tmp.path()); + assert!(repo_path.join(".agents/plugins/marketplace.json").is_file()); + assert!( + repo_path + .join("plugins/gmail/.codex-plugin/plugin.json") + .is_file() + ); + assert_eq!(read_curated_plugins_sha(tmp.path()).as_deref(), Some(sha)); + } + + #[tokio::test] + async fn sync_openai_plugins_repo_skips_archive_download_when_sha_matches() { + let tmp = tempdir().expect("tempdir"); + let repo_path = curated_plugins_repo_path(tmp.path()); + fs::create_dir_all(repo_path.join(".agents/plugins")).expect("create repo"); + fs::write( + repo_path.join(".agents/plugins/marketplace.json"), + r#"{"name":"openai-curated","plugins":[]}"#, + ) + .expect("write marketplace"); + fs::create_dir_all(tmp.path().join(".tmp")).expect("create tmp"); + let sha = "fedcba9876543210fedcba9876543210fedcba98"; + fs::write(tmp.path().join(".tmp/plugins.sha"), format!("{sha}\n")).expect("write sha"); + + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/repos/openai/plugins")) + .respond_with( + ResponseTemplate::new(200).set_body_string(r#"{"default_branch":"main"}"#), + ) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/repos/openai/plugins/git/ref/heads/main")) + .respond_with( + ResponseTemplate::new(200) + .set_body_string(format!(r#"{{"object":{{"sha":"{sha}"}}}}"#)), + ) + .mount(&server) + .await; + + let server_uri = server.uri(); + let tmp_path = tmp.path().to_path_buf(); + tokio::task::spawn_blocking(move || { + sync_openai_plugins_repo_with_api_base_url(tmp_path.as_path(), &server_uri) + }) + .await + .expect("sync task should join") + .expect("sync should succeed"); + + assert_eq!(read_curated_plugins_sha(tmp.path()).as_deref(), Some(sha)); + assert!(repo_path.join(".agents/plugins/marketplace.json").is_file()); + } + + fn curated_repo_zipball_bytes(sha: &str) -> Vec { + let cursor = Cursor::new(Vec::new()); + let mut writer = ZipWriter::new(cursor); + let options = SimpleFileOptions::default(); + let root = format!("openai-plugins-{sha}"); + writer + .start_file(format!("{root}/.agents/plugins/marketplace.json"), options) + .expect("start marketplace entry"); + writer + .write_all( + br#"{ + "name": "openai-curated", + "plugins": [ + { + "name": "gmail", + "source": { + "source": "local", + "path": "./plugins/gmail" + } + } + ] +}"#, + ) + .expect("write marketplace"); + writer + .start_file( + format!("{root}/plugins/gmail/.codex-plugin/plugin.json"), + options, + ) + .expect("start plugin manifest entry"); + writer + .write_all(br#"{"name":"gmail"}"#) + .expect("write plugin manifest"); + + writer.finish().expect("finish zip writer").into_inner() } } diff --git a/codex-rs/core/src/plugins/manager.rs b/codex-rs/core/src/plugins/manager.rs index 91d4045ae4..a9d7cbdc79 100644 --- a/codex-rs/core/src/plugins/manager.rs +++ b/codex-rs/core/src/plugins/manager.rs @@ -11,6 +11,7 @@ use super::marketplace::load_marketplace_summary; use super::marketplace::resolve_marketplace_plugin; use super::plugin_manifest_name; use super::plugin_manifest_paths; +use super::read_curated_plugins_sha; use super::store::DEFAULT_PLUGIN_VERSION; use super::store::PluginId; use super::store::PluginIdError; @@ -45,6 +46,7 @@ use std::collections::HashSet; use std::fs; use std::path::Path; use std::path::PathBuf; +use std::sync::Arc; use std::sync::RwLock; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; @@ -56,7 +58,6 @@ use tracing::warn; const DEFAULT_SKILLS_DIR_NAME: &str = "skills"; const DEFAULT_MCP_CONFIG_FILE: &str = ".mcp.json"; const DEFAULT_APP_CONFIG_FILE: &str = ".app.json"; -const DISABLE_CURATED_PLUGIN_SYNC_ENV_VAR: &str = "CODEX_DISABLE_CURATED_PLUGIN_SYNC"; const OPENAI_CURATED_MARKETPLACE_NAME: &str = "openai-curated"; const REMOTE_PLUGIN_SYNC_TIMEOUT: Duration = Duration::from_secs(30); static CURATED_REPO_SYNC_STARTED: AtomicBool = AtomicBool::new(false); @@ -395,9 +396,25 @@ impl PluginsManager { ) -> Result { let resolved = resolve_marketplace_plugin(&request.marketplace_path, &request.plugin_name)?; let auth_policy = resolved.auth_policy; + let plugin_version = + if resolved.plugin_id.marketplace_name == OPENAI_CURATED_MARKETPLACE_NAME { + Some( + read_curated_plugins_sha(self.codex_home.as_path()).ok_or_else(|| { + PluginStoreError::Invalid( + "local curated marketplace sha is not available".to_string(), + ) + })?, + ) + } else { + None + }; let store = self.store.clone(); let result: StorePluginInstallResult = tokio::task::spawn_blocking(move || { - store.install(resolved.source_path, resolved.plugin_id) + if let Some(plugin_version) = plugin_version { + store.install_with_version(resolved.source_path, resolved.plugin_id, plugin_version) + } else { + store.install(resolved.source_path, resolved.plugin_id) + } }) .await .map_err(PluginInstallError::join)??; @@ -464,8 +481,19 @@ impl PluginsManager { }; let marketplace_name = curated_marketplace.name.clone(); - let mut local_plugins = - Vec::<(String, PluginId, AbsolutePathBuf, Option, bool)>::new(); + let curated_plugin_version = read_curated_plugins_sha(self.codex_home.as_path()) + .ok_or_else(|| { + PluginStoreError::Invalid( + "local curated marketplace sha is not available".to_string(), + ) + })?; + let mut local_plugins = Vec::<( + String, + PluginId, + AbsolutePathBuf, + Option, + Option, + )>::new(); let mut local_plugin_names = HashSet::new(); for plugin in curated_marketplace.plugins { let plugin_name = plugin.name; @@ -486,13 +514,13 @@ impl PluginsManager { let current_enabled = configured_plugins .get(&plugin_key) .map(|plugin| plugin.enabled); - let is_installed = self.store.is_installed(&plugin_id); + let installed_version = self.store.active_plugin_version(&plugin_id); local_plugins.push(( plugin_name, plugin_id, source_path, current_enabled, - is_installed, + installed_version, )); } @@ -528,11 +556,20 @@ impl PluginsManager { let remote_plugin_count = remote_enabled_by_name.len(); let local_plugin_count = local_plugins.len(); - for (plugin_name, plugin_id, source_path, current_enabled, is_installed) in local_plugins { + for (plugin_name, plugin_id, source_path, current_enabled, installed_version) in + local_plugins + { let plugin_key = plugin_id.as_key(); + let is_installed = installed_version.is_some(); if let Some(enabled) = remote_enabled_by_name.get(&plugin_name).copied() { if !is_installed { - installs.push((source_path, plugin_id.clone())); + installs.push(( + source_path, + plugin_id.clone(), + curated_plugin_version.clone(), + )); + } + if !is_installed { result.installed_plugin_ids.push(plugin_key.clone()); } @@ -565,8 +602,8 @@ impl PluginsManager { let store = self.store.clone(); let store_result = tokio::task::spawn_blocking(move || { - for (source_path, plugin_id) in installs { - store.install(source_path, plugin_id)?; + for (source_path, plugin_id, plugin_version) in installs { + store.install_with_version(source_path, plugin_id, plugin_version)?; } for plugin_id in uninstalls { store.uninstall(&plugin_id)?; @@ -668,28 +705,67 @@ impl PluginsManager { .collect()) } - pub fn maybe_start_curated_repo_sync_for_config(&self, config: &Config) { + pub fn maybe_start_curated_repo_sync_for_config(self: &Arc, config: &Config) { if plugins_feature_enabled_from_stack(&config.config_layer_stack) { - self.start_curated_repo_sync(); + let mut configured_curated_plugin_ids = + configured_plugins_from_stack(&config.config_layer_stack) + .into_keys() + .filter_map(|plugin_key| match PluginId::parse(&plugin_key) { + Ok(plugin_id) + if plugin_id.marketplace_name == OPENAI_CURATED_MARKETPLACE_NAME => + { + Some(plugin_id) + } + Ok(_) => None, + Err(err) => { + warn!( + plugin_key, + error = %err, + "ignoring invalid configured plugin key during curated sync setup" + ); + None + } + }) + .collect::>(); + configured_curated_plugin_ids.sort_unstable_by_key(super::store::PluginId::as_key); + self.start_curated_repo_sync(configured_curated_plugin_ids); } } - pub fn start_curated_repo_sync(&self) { - if std::env::var_os(DISABLE_CURATED_PLUGIN_SYNC_ENV_VAR).is_some() { - return; - } + fn start_curated_repo_sync(self: &Arc, configured_curated_plugin_ids: Vec) { if CURATED_REPO_SYNC_STARTED.swap(true, Ordering::SeqCst) { return; } + let manager = Arc::clone(self); let codex_home = self.codex_home.clone(); if let Err(err) = std::thread::Builder::new() .name("plugins-curated-repo-sync".to_string()) - .spawn(move || { - if let Err(err) = sync_openai_plugins_repo(codex_home.as_path()) { - CURATED_REPO_SYNC_STARTED.store(false, Ordering::SeqCst); - warn!("failed to sync curated plugins repo: {err}"); - } - }) + .spawn( + move || match sync_openai_plugins_repo(codex_home.as_path()) { + Ok(curated_plugin_version) => { + match refresh_curated_plugin_cache( + codex_home.as_path(), + &curated_plugin_version, + &configured_curated_plugin_ids, + ) { + Ok(cache_refreshed) => { + if cache_refreshed { + manager.clear_cache(); + } + } + Err(err) => { + manager.clear_cache(); + CURATED_REPO_SYNC_STARTED.store(false, Ordering::SeqCst); + warn!("failed to refresh curated plugin cache after sync: {err}"); + } + } + } + Err(err) => { + CURATED_REPO_SYNC_STARTED.store(false, Ordering::SeqCst); + warn!("failed to sync curated plugins repo: {err}"); + } + }, + ) { CURATED_REPO_SYNC_STARTED.store(false, Ordering::SeqCst); warn!("failed to start curated plugins repo sync task: {err}"); @@ -906,6 +982,65 @@ pub(crate) fn plugin_namespace_for_skill_path(path: &Path) -> Option { None } +fn refresh_curated_plugin_cache( + codex_home: &Path, + plugin_version: &str, + configured_curated_plugin_ids: &[PluginId], +) -> Result { + let store = PluginStore::new(codex_home.to_path_buf()); + let curated_marketplace_path = AbsolutePathBuf::try_from( + curated_plugins_repo_path(codex_home).join(".agents/plugins/marketplace.json"), + ) + .map_err(|_| "local curated marketplace is not available".to_string())?; + let curated_marketplace = load_marketplace_summary(&curated_marketplace_path) + .map_err(|err| format!("failed to load curated marketplace for cache refresh: {err}"))?; + + let mut plugin_sources = HashMap::::new(); + for plugin in curated_marketplace.plugins { + let plugin_name = plugin.name; + if plugin_sources.contains_key(&plugin_name) { + warn!( + plugin = plugin_name, + marketplace = OPENAI_CURATED_MARKETPLACE_NAME, + "ignoring duplicate curated plugin entry during cache refresh" + ); + continue; + } + let source_path = match plugin.source { + MarketplacePluginSourceSummary::Local { path } => path, + }; + plugin_sources.insert(plugin_name, source_path); + } + + let mut cache_refreshed = false; + for plugin_id in configured_curated_plugin_ids { + if store.active_plugin_version(plugin_id).as_deref() == Some(plugin_version) { + continue; + } + + let Some(source_path) = plugin_sources.get(&plugin_id.plugin_name).cloned() else { + warn!( + plugin = plugin_id.plugin_name, + marketplace = OPENAI_CURATED_MARKETPLACE_NAME, + "configured curated plugin no longer exists in curated marketplace during cache refresh" + ); + continue; + }; + + store + .install_with_version(source_path, plugin_id.clone(), plugin_version.to_string()) + .map_err(|err| { + format!( + "failed to refresh curated plugin cache for {}: {err}", + plugin_id.as_key() + ) + })?; + cache_refreshed = true; + } + + Ok(cache_refreshed) +} + fn configured_plugins_from_stack( config_layer_stack: &ConfigLayerStack, ) -> HashMap { @@ -926,9 +1061,11 @@ fn configured_plugins_from_stack( } fn load_plugin(config_name: String, plugin: &PluginConfig, store: &PluginStore) -> LoadedPlugin { - let plugin_version = DEFAULT_PLUGIN_VERSION.to_string(); - let plugin_root = PluginId::parse(&config_name) - .map(|plugin_id| store.plugin_root(&plugin_id, &plugin_version)); + let plugin_root = PluginId::parse(&config_name).map(|plugin_id| { + store + .active_plugin_root(&plugin_id) + .unwrap_or_else(|| store.plugin_root(&plugin_id, DEFAULT_PLUGIN_VERSION)) + }); let root = match &plugin_root { Ok(plugin_root) => plugin_root.clone(), Err(_) => store.root().clone(), @@ -1227,6 +1364,8 @@ mod tests { use wiremock::matchers::method; use wiremock::matchers::path; + const TEST_CURATED_PLUGIN_SHA: &str = "0123456789abcdef0123456789abcdef01234567"; + fn write_file(path: &Path, contents: &str) { fs::create_dir_all(path.parent().expect("file should have a parent")).unwrap(); fs::write(path, contents).unwrap(); @@ -1246,7 +1385,6 @@ mod tests { } fn write_openai_curated_marketplace(root: &Path, plugin_names: &[&str]) { - fs::create_dir_all(root.join(".git")).unwrap(); fs::create_dir_all(root.join(".agents/plugins")).unwrap(); let plugins = plugin_names .iter() @@ -1280,6 +1418,10 @@ mod tests { } } + fn write_curated_plugin_sha(codex_home: &Path, sha: &str) { + write_file(&codex_home.join(".tmp/plugins.sha"), &format!("{sha}\n")); + } + fn plugin_config_toml(enabled: bool, plugins_feature_enabled: bool) -> String { let mut root = toml::map::Map::new(); @@ -2134,7 +2276,6 @@ enabled = false let curated_root = curated_plugins_repo_path(tmp.path()); let plugin_root = curated_root.join("plugins/linear"); - fs::create_dir_all(curated_root.join(".git")).unwrap(); fs::create_dir_all(curated_root.join(".agents/plugins")).unwrap(); fs::create_dir_all(plugin_root.join(".codex-plugin")).unwrap(); fs::write( @@ -2404,6 +2545,7 @@ enabled = true let tmp = tempfile::tempdir().unwrap(); let curated_root = curated_plugins_repo_path(tmp.path()); write_openai_curated_marketplace(&curated_root, &["linear", "gmail", "calendar"]); + write_curated_plugin_sha(tmp.path(), TEST_CURATED_PLUGIN_SHA); write_plugin( &tmp.path().join("plugins/cache/openai-curated"), "linear/local", @@ -2469,7 +2611,9 @@ enabled = true ); assert!( tmp.path() - .join("plugins/cache/openai-curated/gmail/local") + .join(format!( + "plugins/cache/openai-curated/gmail/{TEST_CURATED_PLUGIN_SHA}" + )) .is_dir() ); assert!( @@ -2511,6 +2655,7 @@ enabled = true let tmp = tempfile::tempdir().unwrap(); let curated_root = curated_plugins_repo_path(tmp.path()); write_openai_curated_marketplace(&curated_root, &["linear"]); + write_curated_plugin_sha(tmp.path(), TEST_CURATED_PLUGIN_SHA); write_file( &tmp.path().join(CONFIG_TOML_FILE), r#"[features] @@ -2566,6 +2711,7 @@ enabled = false let tmp = tempfile::tempdir().unwrap(); let curated_root = curated_plugins_repo_path(tmp.path()); write_openai_curated_marketplace(&curated_root, &["linear", "gmail"]); + write_curated_plugin_sha(tmp.path(), TEST_CURATED_PLUGIN_SHA); fs::remove_dir_all(curated_root.join("plugins/gmail")).unwrap(); write_plugin( &tmp.path().join("plugins/cache/openai-curated"), @@ -2630,7 +2776,7 @@ enabled = false async fn sync_plugins_from_remote_uses_first_duplicate_local_plugin_entry() { let tmp = tempfile::tempdir().unwrap(); let curated_root = curated_plugins_repo_path(tmp.path()); - fs::create_dir_all(curated_root.join(".git")).unwrap(); + write_curated_plugin_sha(tmp.path(), TEST_CURATED_PLUGIN_SHA); fs::create_dir_all(curated_root.join(".agents/plugins")).unwrap(); fs::write( curated_root.join(".agents/plugins/marketplace.json"), @@ -2702,15 +2848,98 @@ plugins = true } ); assert_eq!( - fs::read_to_string( - tmp.path() - .join("plugins/cache/openai-curated/gmail/local/marker.txt") - ) + fs::read_to_string(tmp.path().join(format!( + "plugins/cache/openai-curated/gmail/{TEST_CURATED_PLUGIN_SHA}/marker.txt" + ))) .unwrap(), "first" ); } + #[test] + fn refresh_curated_plugin_cache_replaces_existing_local_version_with_sha() { + let tmp = tempfile::tempdir().unwrap(); + let curated_root = curated_plugins_repo_path(tmp.path()); + write_openai_curated_marketplace(&curated_root, &["slack"]); + write_curated_plugin_sha(tmp.path(), TEST_CURATED_PLUGIN_SHA); + let plugin_id = PluginId::new( + "slack".to_string(), + OPENAI_CURATED_MARKETPLACE_NAME.to_string(), + ) + .unwrap(); + write_plugin( + &tmp.path().join("plugins/cache/openai-curated"), + "slack/local", + "slack", + ); + + assert!( + refresh_curated_plugin_cache(tmp.path(), TEST_CURATED_PLUGIN_SHA, &[plugin_id]) + .expect("cache refresh should succeed") + ); + + assert!( + !tmp.path() + .join("plugins/cache/openai-curated/slack/local") + .exists() + ); + assert!( + tmp.path() + .join(format!( + "plugins/cache/openai-curated/slack/{TEST_CURATED_PLUGIN_SHA}" + )) + .is_dir() + ); + } + + #[test] + fn refresh_curated_plugin_cache_reinstalls_missing_configured_plugin_with_current_sha() { + let tmp = tempfile::tempdir().unwrap(); + let curated_root = curated_plugins_repo_path(tmp.path()); + write_openai_curated_marketplace(&curated_root, &["slack"]); + write_curated_plugin_sha(tmp.path(), TEST_CURATED_PLUGIN_SHA); + let plugin_id = PluginId::new( + "slack".to_string(), + OPENAI_CURATED_MARKETPLACE_NAME.to_string(), + ) + .unwrap(); + + assert!( + refresh_curated_plugin_cache(tmp.path(), TEST_CURATED_PLUGIN_SHA, &[plugin_id]) + .expect("cache refresh should recreate missing configured plugin") + ); + + assert!( + tmp.path() + .join(format!( + "plugins/cache/openai-curated/slack/{TEST_CURATED_PLUGIN_SHA}" + )) + .is_dir() + ); + } + + #[test] + fn refresh_curated_plugin_cache_returns_false_when_configured_plugins_are_current() { + let tmp = tempfile::tempdir().unwrap(); + let curated_root = curated_plugins_repo_path(tmp.path()); + write_openai_curated_marketplace(&curated_root, &["slack"]); + let plugin_id = PluginId::new( + "slack".to_string(), + OPENAI_CURATED_MARKETPLACE_NAME.to_string(), + ) + .unwrap(); + write_plugin( + &tmp.path().join("plugins/cache/openai-curated"), + &format!("slack/{TEST_CURATED_PLUGIN_SHA}"), + "slack", + ); + + assert!( + !refresh_curated_plugin_cache(tmp.path(), TEST_CURATED_PLUGIN_SHA, &[plugin_id]) + .expect("cache refresh should be a no-op when configured plugins are current") + ); + } + #[test] fn load_plugins_ignores_project_config_files() { let codex_home = TempDir::new().unwrap(); diff --git a/codex-rs/core/src/plugins/marketplace.rs b/codex-rs/core/src/plugins/marketplace.rs index 754d70f89b..cac6d1b027 100644 --- a/codex-rs/core/src/plugins/marketplace.rs +++ b/codex-rs/core/src/plugins/marketplace.rs @@ -245,6 +245,15 @@ fn discover_marketplace_paths_from_roots( } for root in additional_roots { + // Curated marketplaces can now come from an HTTP-downloaded directory that is not a git + // checkout, so check the root directly before falling back to repo-root discovery. + if let Ok(path) = root.join(MARKETPLACE_RELATIVE_PATH) + && path.as_path().is_file() + && !paths.contains(&path) + { + paths.push(path); + continue; + } if let Some(repo_root) = get_git_repo_root(root.as_path()) && let Ok(repo_root) = AbsolutePathBuf::try_from(repo_root) && let Ok(path) = repo_root.join(MARKETPLACE_RELATIVE_PATH) diff --git a/codex-rs/core/src/plugins/mod.rs b/codex-rs/core/src/plugins/mod.rs index 2b92037d4e..1540459aa7 100644 --- a/codex-rs/core/src/plugins/mod.rs +++ b/codex-rs/core/src/plugins/mod.rs @@ -7,6 +7,7 @@ mod render; mod store; pub(crate) use curated_repo::curated_plugins_repo_path; +pub(crate) use curated_repo::read_curated_plugins_sha; pub(crate) use curated_repo::sync_openai_plugins_repo; pub(crate) use injection::build_plugin_injections; pub use manager::AppConnectorId; diff --git a/codex-rs/core/src/plugins/store.rs b/codex-rs/core/src/plugins/store.rs index 0611df9a07..806c43f4d8 100644 --- a/codex-rs/core/src/plugins/store.rs +++ b/codex-rs/core/src/plugins/store.rs @@ -81,27 +81,65 @@ impl PluginStore { &self.root } - pub fn plugin_root(&self, plugin_id: &PluginId, plugin_version: &str) -> AbsolutePathBuf { + pub fn plugin_base_root(&self, plugin_id: &PluginId) -> AbsolutePathBuf { AbsolutePathBuf::try_from( self.root .as_path() .join(&plugin_id.marketplace_name) - .join(&plugin_id.plugin_name) + .join(&plugin_id.plugin_name), + ) + .unwrap_or_else(|err| panic!("plugin cache path should resolve to an absolute path: {err}")) + } + + pub fn plugin_root(&self, plugin_id: &PluginId, plugin_version: &str) -> AbsolutePathBuf { + AbsolutePathBuf::try_from( + self.plugin_base_root(plugin_id) + .as_path() .join(plugin_version), ) .unwrap_or_else(|err| panic!("plugin cache path should resolve to an absolute path: {err}")) } + pub fn active_plugin_version(&self, plugin_id: &PluginId) -> Option { + let mut discovered_versions = fs::read_dir(self.plugin_base_root(plugin_id).as_path()) + .ok()? + .filter_map(Result::ok) + .filter_map(|entry| { + entry.file_type().ok().filter(std::fs::FileType::is_dir)?; + entry.file_name().into_string().ok() + }) + .filter(|version| validate_plugin_segment(version, "plugin version").is_ok()) + .collect::>(); + discovered_versions.sort_unstable(); + if discovered_versions.len() == 1 { + discovered_versions.pop() + } else { + None + } + } + + pub fn active_plugin_root(&self, plugin_id: &PluginId) -> Option { + self.active_plugin_version(plugin_id) + .map(|plugin_version| self.plugin_root(plugin_id, &plugin_version)) + } + pub fn is_installed(&self, plugin_id: &PluginId) -> bool { - self.plugin_root(plugin_id, DEFAULT_PLUGIN_VERSION) - .as_path() - .is_dir() + self.active_plugin_version(plugin_id).is_some() } pub fn install( &self, source_path: AbsolutePathBuf, plugin_id: PluginId, + ) -> Result { + self.install_with_version(source_path, plugin_id, DEFAULT_PLUGIN_VERSION.to_string()) + } + + pub fn install_with_version( + &self, + source_path: AbsolutePathBuf, + plugin_id: PluginId, + plugin_version: String, ) -> Result { if !source_path.as_path().is_dir() { return Err(PluginStoreError::Invalid(format!( @@ -117,17 +155,14 @@ impl PluginStore { plugin_id.plugin_name ))); } - let plugin_version = DEFAULT_PLUGIN_VERSION.to_string(); + validate_plugin_segment(&plugin_version, "plugin version") + .map_err(PluginStoreError::Invalid)?; let installed_path = self.plugin_root(&plugin_id, &plugin_version); - - if let Some(parent) = installed_path.parent() { - fs::create_dir_all(parent.as_path()).map_err(|err| { - PluginStoreError::io("failed to create plugin cache directory", err) - })?; - } - - remove_existing_target(installed_path.as_path())?; - copy_dir_recursive(source_path.as_path(), installed_path.as_path())?; + replace_plugin_root_atomically( + source_path.as_path(), + self.plugin_base_root(&plugin_id).as_path(), + &plugin_version, + )?; Ok(PluginInstallResult { plugin_id, @@ -137,12 +172,7 @@ impl PluginStore { } pub fn uninstall(&self, plugin_id: &PluginId) -> Result<(), PluginStoreError> { - let plugin_path = self - .root - .as_path() - .join(&plugin_id.marketplace_name) - .join(&plugin_id.plugin_name); - remove_existing_target(&plugin_path) + remove_existing_target(self.plugin_base_root(plugin_id).as_path()) } } @@ -218,6 +248,73 @@ fn remove_existing_target(path: &Path) -> Result<(), PluginStoreError> { } } +fn replace_plugin_root_atomically( + source: &Path, + target_root: &Path, + plugin_version: &str, +) -> Result<(), PluginStoreError> { + let Some(parent) = target_root.parent() else { + return Err(PluginStoreError::Invalid(format!( + "plugin cache path has no parent: {}", + target_root.display() + ))); + }; + + fs::create_dir_all(parent) + .map_err(|err| PluginStoreError::io("failed to create plugin cache directory", err))?; + + let Some(plugin_dir_name) = target_root.file_name() else { + return Err(PluginStoreError::Invalid(format!( + "plugin cache path has no directory name: {}", + target_root.display() + ))); + }; + let staged_dir = tempfile::Builder::new() + .prefix("plugin-install-") + .tempdir_in(parent) + .map_err(|err| { + PluginStoreError::io("failed to create temporary plugin cache directory", err) + })?; + let staged_root = staged_dir.path().join(plugin_dir_name); + let staged_version_root = staged_root.join(plugin_version); + copy_dir_recursive(source, &staged_version_root)?; + + if target_root.exists() { + let backup_dir = tempfile::Builder::new() + .prefix("plugin-backup-") + .tempdir_in(parent) + .map_err(|err| { + PluginStoreError::io("failed to create plugin cache backup directory", err) + })?; + let backup_root = backup_dir.path().join(plugin_dir_name); + fs::rename(target_root, &backup_root) + .map_err(|err| PluginStoreError::io("failed to back up plugin cache entry", err))?; + + if let Err(err) = fs::rename(&staged_root, target_root) { + let rollback_result = fs::rename(&backup_root, target_root); + return match rollback_result { + Ok(()) => Err(PluginStoreError::io( + "failed to activate updated plugin cache entry", + err, + )), + Err(rollback_err) => { + let backup_path = backup_dir.keep().join(plugin_dir_name); + Err(PluginStoreError::Invalid(format!( + "failed to activate updated plugin cache entry at {}: {err}; failed to restore previous cache entry (left at {}): {rollback_err}", + target_root.display(), + backup_path.display() + ))) + } + }; + } + } else { + fs::rename(&staged_root, target_root) + .map_err(|err| PluginStoreError::io("failed to activate plugin cache entry", err))?; + } + + Ok(()) +} + fn copy_dir_recursive(source: &Path, target: &Path) -> Result<(), PluginStoreError> { fs::create_dir_all(target) .map_err(|err| PluginStoreError::io("failed to create plugin target directory", err))?; @@ -327,6 +424,57 @@ mod tests { ); } + #[test] + fn install_with_version_uses_requested_cache_version() { + let tmp = tempdir().unwrap(); + write_plugin(tmp.path(), "sample-plugin", "sample-plugin"); + let plugin_id = + PluginId::new("sample-plugin".to_string(), "openai-curated".to_string()).unwrap(); + let plugin_version = "0123456789abcdef".to_string(); + + let result = PluginStore::new(tmp.path().to_path_buf()) + .install_with_version( + AbsolutePathBuf::try_from(tmp.path().join("sample-plugin")).unwrap(), + plugin_id.clone(), + plugin_version.clone(), + ) + .unwrap(); + + let installed_path = tmp.path().join(format!( + "plugins/cache/openai-curated/sample-plugin/{plugin_version}" + )); + assert_eq!( + result, + PluginInstallResult { + plugin_id, + plugin_version, + installed_path: AbsolutePathBuf::try_from(installed_path.clone()).unwrap(), + } + ); + assert!(installed_path.join(".codex-plugin/plugin.json").is_file()); + } + + #[test] + fn active_plugin_version_reads_version_directory_name() { + let tmp = tempdir().unwrap(); + write_plugin( + &tmp.path().join("plugins/cache/debug"), + "sample-plugin/local", + "sample-plugin", + ); + let store = PluginStore::new(tmp.path().to_path_buf()); + let plugin_id = PluginId::new("sample-plugin".to_string(), "debug".to_string()).unwrap(); + + assert_eq!( + store.active_plugin_version(&plugin_id), + Some("local".to_string()) + ); + assert_eq!( + store.active_plugin_root(&plugin_id).unwrap().as_path(), + tmp.path().join("plugins/cache/debug/sample-plugin/local") + ); + } + #[test] fn plugin_root_rejects_path_separators_in_key_segments() { let err = PluginId::parse("../../etc@debug").unwrap_err(); From 04892b4ceb3ba6516eddd21c3f7ed2920a977442 Mon Sep 17 00:00:00 2001 From: viyatb-oai Date: Wed, 11 Mar 2026 23:31:18 -0700 Subject: [PATCH 6/6] refactor: make bubblewrap the default Linux sandbox (#13996) ## Summary - make bubblewrap the default Linux sandbox and keep `use_legacy_landlock` as the only override - remove `use_linux_sandbox_bwrap` from feature, config, schema, and docs surfaces - update Linux sandbox selection, CLI/config plumbing, and related tests/docs to match the new default - fold in the follow-up CI fixes for request-permissions responses and Linux read-only sandbox error text --- .../app-server/src/codex_message_processor.rs | 4 +- .../app-server/tests/suite/v2/command_exec.rs | 46 ++++++++++-------- codex-rs/cli/src/debug_sandbox.rs | 5 +- codex-rs/cli/src/lib.rs | 2 +- codex-rs/cli/src/main.rs | 2 +- codex-rs/core/config.schema.json | 4 +- codex-rs/core/src/codex.rs | 16 ++----- codex-rs/core/src/connectors.rs | 3 +- codex-rs/core/src/exec.rs | 8 ++-- codex-rs/core/src/features.rs | 42 +++++----------- codex-rs/core/src/landlock.rs | 34 ++++++------- codex-rs/core/src/mcp/mod.rs | 2 +- codex-rs/core/src/mcp_connection_manager.rs | 2 +- codex-rs/core/src/sandbox_tags.rs | 16 +++---- codex-rs/core/src/sandboxing/mod.rs | 12 ++--- codex-rs/core/src/tools/js_repl/mod.rs | 4 +- codex-rs/core/src/tools/orchestrator.rs | 7 ++- codex-rs/core/src/tools/registry.rs | 8 +--- .../tools/runtimes/shell/unix_escalation.rs | 8 ++-- .../runtimes/shell/unix_escalation_tests.rs | 6 +-- codex-rs/core/src/tools/sandboxing.rs | 4 +- codex-rs/core/src/turn_metadata.rs | 48 +++++++++---------- codex-rs/core/tests/suite/approvals.rs | 19 ++++---- codex-rs/linux-sandbox/README.md | 23 +++++---- codex-rs/linux-sandbox/src/linux_run_main.rs | 34 ++++++------- .../linux-sandbox/src/linux_run_main_tests.rs | 10 ++-- .../linux-sandbox/tests/suite/landlock.rs | 26 +++++----- .../tests/suite/managed_proxy.rs | 1 - codex-rs/utils/cli/src/config_override.rs | 10 ++-- 29 files changed, 184 insertions(+), 222 deletions(-) diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index 6a22d899b3..5b684a0eb5 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -1723,7 +1723,7 @@ impl CodexMessageProcessor { let outgoing = self.outgoing.clone(); let request_for_task = request.clone(); let started_network_proxy_for_task = started_network_proxy; - let use_linux_sandbox_bwrap = self.config.features.enabled(Feature::UseLinuxSandboxBwrap); + let use_legacy_landlock = self.config.features.use_legacy_landlock(); let size = match size.map(crate::command_exec::terminal_size_from_protocol) { Some(Ok(size)) => Some(size), Some(Err(error)) => { @@ -1740,7 +1740,7 @@ impl CodexMessageProcessor { effective_network_sandbox_policy, sandbox_cwd.as_path(), &codex_linux_sandbox_exe, - use_linux_sandbox_bwrap, + use_legacy_landlock, ) { Ok(exec_request) => { if let Err(error) = self diff --git a/codex-rs/app-server/tests/suite/v2/command_exec.rs b/codex-rs/app-server/tests/suite/v2/command_exec.rs index c0dc140c4e..ecd897eb84 100644 --- a/codex-rs/app-server/tests/suite/v2/command_exec.rs +++ b/codex-rs/app-server/tests/suite/v2/command_exec.rs @@ -710,6 +710,12 @@ async fn command_exec_process_ids_are_connection_scoped_and_disconnect_terminate let server = create_mock_responses_server_sequence_unchecked(Vec::new()).await; let codex_home = TempDir::new()?; create_config_toml(codex_home.path(), &server.uri(), "never")?; + let marker = format!( + "codex-command-exec-marker-{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH)? + .as_nanos() + ); let (mut process, bind_addr) = spawn_websocket_server(codex_home.path()).await?; @@ -726,7 +732,12 @@ async fn command_exec_process_ids_are_connection_scoped_and_disconnect_terminate "command/exec", 101, Some(serde_json::json!({ - "command": ["sh", "-lc", "printf 'ready\\n%s\\n' $$; sleep 30"], + "command": [ + "python3", + "-c", + "import time; print('ready', flush=True); time.sleep(30)", + marker, + ], "processId": "shared-process", "streamStdoutStderr": true, })), @@ -737,12 +748,8 @@ async fn command_exec_process_ids_are_connection_scoped_and_disconnect_terminate assert_eq!(delta.process_id, "shared-process"); assert_eq!(delta.stream, CommandExecOutputStream::Stdout); let delta_text = String::from_utf8(STANDARD.decode(&delta.delta_base64)?)?; - let pid = delta_text - .lines() - .last() - .context("delta should include shell pid")? - .parse::() - .context("parse shell pid")?; + assert!(delta_text.contains("ready")); + wait_for_process_marker(&marker, true).await?; send_request( &mut ws2, @@ -766,12 +773,12 @@ async fn command_exec_process_ids_are_connection_scoped_and_disconnect_terminate terminate_error.error.message, "no active command/exec for process id \"shared-process\"" ); - assert!(process_is_alive(pid)?); + wait_for_process_marker(&marker, true).await?; assert_no_message(&mut ws2, Duration::from_millis(250)).await?; ws1.close(None).await?; - wait_for_process_exit(pid).await?; + wait_for_process_marker(&marker, false).await?; process .kill() @@ -855,24 +862,25 @@ async fn read_initialize_response( } } -async fn wait_for_process_exit(pid: u32) -> Result<()> { +async fn wait_for_process_marker(marker: &str, should_exist: bool) -> Result<()> { let deadline = Instant::now() + Duration::from_secs(5); loop { - if !process_is_alive(pid)? { + if process_with_marker_exists(marker)? == should_exist { return Ok(()); } if Instant::now() >= deadline { - anyhow::bail!("process {pid} was still alive after websocket disconnect"); + let expectation = if should_exist { "appear" } else { "exit" }; + anyhow::bail!("process marker {marker:?} did not {expectation} before timeout"); } sleep(Duration::from_millis(50)).await; } } -fn process_is_alive(pid: u32) -> Result { - let status = std::process::Command::new("kill") - .arg("-0") - .arg(pid.to_string()) - .status() - .context("spawn kill -0")?; - Ok(status.success()) +fn process_with_marker_exists(marker: &str) -> Result { + let output = std::process::Command::new("ps") + .args(["-axo", "command"]) + .output() + .context("spawn ps -axo command")?; + let stdout = String::from_utf8(output.stdout).context("decode ps output")?; + Ok(stdout.lines().any(|line| line.contains(marker))) } diff --git a/codex-rs/cli/src/debug_sandbox.rs b/codex-rs/cli/src/debug_sandbox.rs index 3fd4350826..999a92db6c 100644 --- a/codex-rs/cli/src/debug_sandbox.rs +++ b/codex-rs/cli/src/debug_sandbox.rs @@ -250,19 +250,18 @@ async fn run_command_under_sandbox( .await? } SandboxType::Landlock => { - use codex_core::features::Feature; #[expect(clippy::expect_used)] let codex_linux_sandbox_exe = config .codex_linux_sandbox_exe .expect("codex-linux-sandbox executable not found"); - let use_bwrap_sandbox = config.features.enabled(Feature::UseLinuxSandboxBwrap); + let use_legacy_landlock = config.features.use_legacy_landlock(); spawn_command_under_linux_sandbox( codex_linux_sandbox_exe, command, cwd, config.permissions.sandbox_policy.get(), sandbox_policy_cwd.as_path(), - use_bwrap_sandbox, + use_legacy_landlock, stdio_policy, network.as_ref(), env, diff --git a/codex-rs/cli/src/lib.rs b/codex-rs/cli/src/lib.rs index f71d459839..b6174efa3a 100644 --- a/codex-rs/cli/src/lib.rs +++ b/codex-rs/cli/src/lib.rs @@ -32,7 +32,7 @@ pub struct LandlockCommand { #[clap(skip)] pub config_overrides: CliConfigOverrides, - /// Full command args to run under landlock. + /// Full command args to run under the Linux sandbox. #[arg(trailing_var_arg = true)] pub command: Vec, } diff --git a/codex-rs/cli/src/main.rs b/codex-rs/cli/src/main.rs index d977979b89..e44c2c624d 100644 --- a/codex-rs/cli/src/main.rs +++ b/codex-rs/cli/src/main.rs @@ -239,7 +239,7 @@ enum SandboxCommand { #[clap(visible_alias = "seatbelt")] Macos(SeatbeltCommand), - /// Run a command under Landlock+seccomp (Linux only). + /// Run a command under the Linux sandbox (bubblewrap by default). #[clap(visible_alias = "landlock")] Linux(LandlockCommand), diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.json index 3b7395f489..413bccb108 100644 --- a/codex-rs/core/config.schema.json +++ b/codex-rs/core/config.schema.json @@ -480,7 +480,7 @@ "unified_exec": { "type": "boolean" }, - "use_linux_sandbox_bwrap": { + "use_legacy_landlock": { "type": "boolean" }, "voice_transcription": { @@ -1985,7 +1985,7 @@ "unified_exec": { "type": "boolean" }, - "use_linux_sandbox_bwrap": { + "use_legacy_landlock": { "type": "boolean" }, "voice_transcription": { diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index b5695a0bff..fcc00fdf4e 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -1290,9 +1290,7 @@ impl Session { cwd.clone(), session_configuration.sandbox_policy.get(), session_configuration.windows_sandbox_level, - per_turn_config - .features - .enabled(Feature::UseLinuxSandboxBwrap), + per_turn_config.features.use_legacy_landlock(), )); let (current_date, timezone) = local_time_context(); TurnContext { @@ -1802,7 +1800,7 @@ impl Session { sandbox_policy: session_configuration.sandbox_policy.get().clone(), codex_linux_sandbox_exe: config.codex_linux_sandbox_exe.clone(), sandbox_cwd: session_configuration.cwd.clone(), - use_linux_sandbox_bwrap: config.features.enabled(Feature::UseLinuxSandboxBwrap), + use_legacy_landlock: config.features.use_legacy_landlock(), }; let mut required_mcp_servers: Vec = mcp_servers .iter() @@ -2275,9 +2273,7 @@ impl Session { sandbox_policy: per_turn_config.permissions.sandbox_policy.get().clone(), codex_linux_sandbox_exe: per_turn_config.codex_linux_sandbox_exe.clone(), sandbox_cwd: per_turn_config.cwd.clone(), - use_linux_sandbox_bwrap: per_turn_config - .features - .enabled(Feature::UseLinuxSandboxBwrap), + use_legacy_landlock: per_turn_config.features.use_legacy_landlock(), }; if let Err(e) = self .services @@ -3938,7 +3934,7 @@ impl Session { sandbox_policy: turn_context.sandbox_policy.get().clone(), codex_linux_sandbox_exe: turn_context.codex_linux_sandbox_exe.clone(), sandbox_cwd: turn_context.cwd.clone(), - use_linux_sandbox_bwrap: turn_context.features.enabled(Feature::UseLinuxSandboxBwrap), + use_legacy_landlock: turn_context.features.use_legacy_landlock(), }; { let mut guard = self.services.mcp_startup_cancellation_token.lock().await; @@ -5215,9 +5211,7 @@ async fn spawn_review_thread( parent_turn_context.cwd.clone(), parent_turn_context.sandbox_policy.get(), parent_turn_context.windows_sandbox_level, - parent_turn_context - .features - .enabled(Feature::UseLinuxSandboxBwrap), + parent_turn_context.features.use_legacy_landlock(), )); let review_turn_context = TurnContext { diff --git a/codex-rs/core/src/connectors.rs b/codex-rs/core/src/connectors.rs index 47d61fe984..92cc910c47 100644 --- a/codex-rs/core/src/connectors.rs +++ b/codex-rs/core/src/connectors.rs @@ -31,7 +31,6 @@ use crate::config::types::AppsConfigToml; use crate::default_client::create_client; use crate::default_client::is_first_party_chat_originator; use crate::default_client::originator; -use crate::features::Feature; use crate::mcp::CODEX_APPS_MCP_SERVER_NAME; use crate::mcp::McpManager; use crate::mcp::ToolPluginProvenance; @@ -203,7 +202,7 @@ pub async fn list_accessible_connectors_from_mcp_tools_with_options_and_status( sandbox_policy: SandboxPolicy::new_read_only_policy(), codex_linux_sandbox_exe: config.codex_linux_sandbox_exe.clone(), sandbox_cwd: env::current_dir().unwrap_or_else(|_| PathBuf::from("/")), - use_linux_sandbox_bwrap: config.features.enabled(Feature::UseLinuxSandboxBwrap), + use_legacy_landlock: config.features.use_legacy_landlock(), }; let (mcp_connection_manager, cancel_token) = McpConnectionManager::new( diff --git a/codex-rs/core/src/exec.rs b/codex-rs/core/src/exec.rs index 19d56d71de..1b9456fb87 100644 --- a/codex-rs/core/src/exec.rs +++ b/codex-rs/core/src/exec.rs @@ -185,7 +185,7 @@ pub async fn process_exec_tool_call( network_sandbox_policy: NetworkSandboxPolicy, sandbox_cwd: &Path, codex_linux_sandbox_exe: &Option, - use_linux_sandbox_bwrap: bool, + use_legacy_landlock: bool, stdout_stream: Option, ) -> Result { let exec_req = build_exec_request( @@ -195,7 +195,7 @@ pub async fn process_exec_tool_call( network_sandbox_policy, sandbox_cwd, codex_linux_sandbox_exe, - use_linux_sandbox_bwrap, + use_legacy_landlock, )?; // Route through the sandboxing module for a single, unified execution path. @@ -211,7 +211,7 @@ pub fn build_exec_request( network_sandbox_policy: NetworkSandboxPolicy, sandbox_cwd: &Path, codex_linux_sandbox_exe: &Option, - use_linux_sandbox_bwrap: bool, + use_legacy_landlock: bool, ) -> Result { let windows_sandbox_level = params.windows_sandbox_level; let enforce_managed_network = params.network.is_some(); @@ -269,7 +269,7 @@ pub fn build_exec_request( #[cfg(target_os = "macos")] macos_seatbelt_profile_extensions: None, codex_linux_sandbox_exe: codex_linux_sandbox_exe.as_ref(), - use_linux_sandbox_bwrap, + use_legacy_landlock, windows_sandbox_level, }) .map_err(CodexErr::from)?; diff --git a/codex-rs/core/src/features.rs b/codex-rs/core/src/features.rs index 76da67c323..27f27f55e7 100644 --- a/codex-rs/core/src/features.rs +++ b/codex-rs/core/src/features.rs @@ -108,8 +108,9 @@ pub enum Feature { WebSearchCached, /// Legacy search-tool feature flag kept for backward compatibility. SearchTool, - /// Use the bubblewrap-based Linux sandbox pipeline. - UseLinuxSandboxBwrap, + /// Use the legacy Landlock Linux sandbox fallback instead of the default + /// bubblewrap pipeline. + UseLegacyLandlock, /// Allow the model to request approval and propose exec rules. RequestRule, /// Enable Windows sandbox (restricted token) on Windows. @@ -284,6 +285,10 @@ impl Features { self.enabled(Feature::Apps) && auth.is_some_and(CodexAuth::is_chatgpt_auth) } + pub fn use_legacy_landlock(&self) -> bool { + self.enabled(Feature::UseLegacyLandlock) + } + pub fn enable(&mut self, f: Feature) -> &mut Self { self.enabled.insert(f); self @@ -636,16 +641,9 @@ pub const FEATURES: &[FeatureSpec] = &[ default_enabled: false, }, FeatureSpec { - id: Feature::UseLinuxSandboxBwrap, - key: "use_linux_sandbox_bwrap", - #[cfg(target_os = "linux")] - stage: Stage::Experimental { - name: "Bubblewrap sandbox", - menu_description: "Try the new linux sandbox based on bubblewrap.", - announcement: "NEW: Linux bubblewrap sandbox offers stronger filesystem and network controls than Landlock alone, including keeping .git and .codex read-only inside writable workspaces. Enable it in /experimental and restart Codex to try it.", - }, - #[cfg(not(target_os = "linux"))] - stage: Stage::UnderDevelopment, + id: Feature::UseLegacyLandlock, + key: "use_legacy_landlock", + stage: Stage::Stable, default_enabled: false, }, FeatureSpec { @@ -932,24 +930,10 @@ mod tests { } } - #[cfg(target_os = "linux")] #[test] - fn use_linux_sandbox_bwrap_is_experimental_on_linux() { - assert!(matches!( - Feature::UseLinuxSandboxBwrap.stage(), - Stage::Experimental { .. } - )); - assert_eq!(Feature::UseLinuxSandboxBwrap.default_enabled(), false); - } - - #[cfg(not(target_os = "linux"))] - #[test] - fn use_linux_sandbox_bwrap_is_under_development_off_linux() { - assert_eq!( - Feature::UseLinuxSandboxBwrap.stage(), - Stage::UnderDevelopment - ); - assert_eq!(Feature::UseLinuxSandboxBwrap.default_enabled(), false); + fn use_legacy_landlock_is_stable_and_disabled_by_default() { + assert_eq!(Feature::UseLegacyLandlock.stage(), Stage::Stable); + assert_eq!(Feature::UseLegacyLandlock.default_enabled(), false); } #[test] diff --git a/codex-rs/core/src/landlock.rs b/codex-rs/core/src/landlock.rs index e90daf37f2..1072854883 100644 --- a/codex-rs/core/src/landlock.rs +++ b/codex-rs/core/src/landlock.rs @@ -11,7 +11,7 @@ use std::path::PathBuf; use tokio::process::Child; /// Spawn a shell tool command under the Linux sandbox helper -/// (codex-linux-sandbox), which currently uses bubblewrap for filesystem +/// (codex-linux-sandbox), which defaults to bubblewrap for filesystem /// isolation plus seccomp for network restrictions. /// /// Unlike macOS Seatbelt where we directly embed the policy text, the Linux @@ -25,7 +25,7 @@ pub async fn spawn_command_under_linux_sandbox

( command_cwd: PathBuf, sandbox_policy: &SandboxPolicy, sandbox_policy_cwd: &Path, - use_bwrap_sandbox: bool, + use_legacy_landlock: bool, stdio_policy: StdioPolicy, network: Option<&NetworkProxy>, env: HashMap, @@ -42,7 +42,7 @@ where &file_system_sandbox_policy, network_sandbox_policy, sandbox_policy_cwd, - use_bwrap_sandbox, + use_legacy_landlock, allow_network_for_proxy(false), ); let arg0 = Some("codex-linux-sandbox"); @@ -69,7 +69,7 @@ pub(crate) fn allow_network_for_proxy(enforce_managed_network: bool) -> bool { /// Converts the sandbox policies into the CLI invocation for /// `codex-linux-sandbox`. /// -/// The helper performs the actual sandboxing (bubblewrap + seccomp) after +/// The helper performs the actual sandboxing (bubblewrap by default + seccomp) after /// parsing these arguments. Policy JSON flags are emitted before helper feature /// flags so the argv order matches the helper's CLI shape. See /// `docs/linux_sandbox.md` for the Linux semantics. @@ -80,7 +80,7 @@ pub(crate) fn create_linux_sandbox_command_args_for_policies( file_system_sandbox_policy: &FileSystemSandboxPolicy, network_sandbox_policy: NetworkSandboxPolicy, sandbox_policy_cwd: &Path, - use_bwrap_sandbox: bool, + use_legacy_landlock: bool, allow_network_for_proxy: bool, ) -> Vec { let sandbox_policy_json = serde_json::to_string(sandbox_policy) @@ -104,8 +104,8 @@ pub(crate) fn create_linux_sandbox_command_args_for_policies( "--network-sandbox-policy".to_string(), network_policy_json, ]; - if use_bwrap_sandbox { - linux_cmd.push("--use-bwrap-sandbox".to_string()); + if use_legacy_landlock { + linux_cmd.push("--use-legacy-landlock".to_string()); } if allow_network_for_proxy { linux_cmd.push("--allow-network-for-proxy".to_string()); @@ -121,7 +121,7 @@ pub(crate) fn create_linux_sandbox_command_args_for_policies( pub(crate) fn create_linux_sandbox_command_args( command: Vec, sandbox_policy_cwd: &Path, - use_bwrap_sandbox: bool, + use_legacy_landlock: bool, allow_network_for_proxy: bool, ) -> Vec { let sandbox_policy_cwd = sandbox_policy_cwd @@ -130,8 +130,8 @@ pub(crate) fn create_linux_sandbox_command_args( .to_string(); let mut linux_cmd: Vec = vec!["--sandbox-policy-cwd".to_string(), sandbox_policy_cwd]; - if use_bwrap_sandbox { - linux_cmd.push("--use-bwrap-sandbox".to_string()); + if use_legacy_landlock { + linux_cmd.push("--use-legacy-landlock".to_string()); } if allow_network_for_proxy { linux_cmd.push("--allow-network-for-proxy".to_string()); @@ -153,20 +153,20 @@ mod tests { use pretty_assertions::assert_eq; #[test] - fn bwrap_flags_are_feature_gated() { + fn legacy_landlock_flag_is_included_when_requested() { let command = vec!["/bin/true".to_string()]; let cwd = Path::new("/tmp"); - let with_bwrap = create_linux_sandbox_command_args(command.clone(), cwd, true, false); + let default_bwrap = create_linux_sandbox_command_args(command.clone(), cwd, false, false); assert_eq!( - with_bwrap.contains(&"--use-bwrap-sandbox".to_string()), - true + default_bwrap.contains(&"--use-legacy-landlock".to_string()), + false ); - let without_bwrap = create_linux_sandbox_command_args(command, cwd, false, false); + let legacy_landlock = create_linux_sandbox_command_args(command, cwd, true, false); assert_eq!( - without_bwrap.contains(&"--use-bwrap-sandbox".to_string()), - false + legacy_landlock.contains(&"--use-legacy-landlock".to_string()), + true ); } diff --git a/codex-rs/core/src/mcp/mod.rs b/codex-rs/core/src/mcp/mod.rs index 39eca4b29b..ed93106fe9 100644 --- a/codex-rs/core/src/mcp/mod.rs +++ b/codex-rs/core/src/mcp/mod.rs @@ -304,7 +304,7 @@ pub async fn collect_mcp_snapshot(config: &Config) -> McpListToolsResponseEvent sandbox_policy: SandboxPolicy::new_read_only_policy(), codex_linux_sandbox_exe: config.codex_linux_sandbox_exe.clone(), sandbox_cwd: env::current_dir().unwrap_or_else(|_| PathBuf::from("/")), - use_linux_sandbox_bwrap: config.features.enabled(Feature::UseLinuxSandboxBwrap), + use_legacy_landlock: config.features.use_legacy_landlock(), }; let (mcp_connection_manager, cancel_token) = McpConnectionManager::new( diff --git a/codex-rs/core/src/mcp_connection_manager.rs b/codex-rs/core/src/mcp_connection_manager.rs index de8ad51430..5eb5ebc793 100644 --- a/codex-rs/core/src/mcp_connection_manager.rs +++ b/codex-rs/core/src/mcp_connection_manager.rs @@ -591,7 +591,7 @@ pub struct SandboxState { pub codex_linux_sandbox_exe: Option, pub sandbox_cwd: PathBuf, #[serde(default)] - pub use_linux_sandbox_bwrap: bool, + pub use_legacy_landlock: bool, } /// A thin wrapper around a set of running [`RmcpClient`] instances. diff --git a/codex-rs/core/src/sandbox_tags.rs b/codex-rs/core/src/sandbox_tags.rs index 56c9e0f184..4910d85f18 100644 --- a/codex-rs/core/src/sandbox_tags.rs +++ b/codex-rs/core/src/sandbox_tags.rs @@ -6,7 +6,7 @@ use codex_protocol::config_types::WindowsSandboxLevel; pub(crate) fn sandbox_tag( policy: &SandboxPolicy, windows_sandbox_level: WindowsSandboxLevel, - use_linux_sandbox_bwrap: bool, + use_legacy_landlock: bool, ) -> &'static str { if matches!(policy, SandboxPolicy::DangerFullAccess) { return "none"; @@ -18,7 +18,7 @@ pub(crate) fn sandbox_tag( { return "windows_elevated"; } - if cfg!(target_os = "linux") && use_linux_sandbox_bwrap { + if cfg!(target_os = "linux") && !use_legacy_landlock { return "linux_bubblewrap"; } @@ -38,33 +38,33 @@ mod tests { use pretty_assertions::assert_eq; #[test] - fn danger_full_access_is_untagged_even_when_bubblewrap_is_enabled() { + fn danger_full_access_is_untagged_even_when_bubblewrap_is_default() { let actual = sandbox_tag( &SandboxPolicy::DangerFullAccess, WindowsSandboxLevel::Disabled, - true, + false, ); assert_eq!(actual, "none"); } #[test] - fn external_sandbox_keeps_external_tag_when_bubblewrap_is_enabled() { + fn external_sandbox_keeps_external_tag_when_bubblewrap_is_default() { let actual = sandbox_tag( &SandboxPolicy::ExternalSandbox { network_access: NetworkAccess::Enabled, }, WindowsSandboxLevel::Disabled, - true, + false, ); assert_eq!(actual, "external"); } #[test] - fn bubblewrap_feature_sets_distinct_linux_tag() { + fn bubblewrap_default_sets_distinct_linux_tag() { let actual = sandbox_tag( &SandboxPolicy::new_read_only_policy(), WindowsSandboxLevel::Disabled, - true, + false, ); let expected = if cfg!(target_os = "linux") { "linux_bubblewrap" diff --git a/codex-rs/core/src/sandboxing/mod.rs b/codex-rs/core/src/sandboxing/mod.rs index 377ecb3db8..a3a5617af6 100644 --- a/codex-rs/core/src/sandboxing/mod.rs +++ b/codex-rs/core/src/sandboxing/mod.rs @@ -94,7 +94,7 @@ pub(crate) struct SandboxTransformRequest<'a> { #[cfg(target_os = "macos")] pub macos_seatbelt_profile_extensions: Option<&'a MacOsSeatbeltProfileExtensions>, pub codex_linux_sandbox_exe: Option<&'a PathBuf>, - pub use_linux_sandbox_bwrap: bool, + pub use_legacy_landlock: bool, pub windows_sandbox_level: WindowsSandboxLevel, } @@ -571,7 +571,7 @@ impl SandboxManager { #[cfg(target_os = "macos")] macos_seatbelt_profile_extensions, codex_linux_sandbox_exe, - use_linux_sandbox_bwrap, + use_legacy_landlock, windows_sandbox_level, } = request; #[cfg(not(target_os = "macos"))] @@ -653,7 +653,7 @@ impl SandboxManager { &effective_file_system_policy, effective_network_policy, sandbox_policy_cwd, - use_linux_sandbox_bwrap, + use_legacy_landlock, allow_proxy_network, ); let mut full_command = Vec::with_capacity(1 + args.len()); @@ -886,7 +886,7 @@ mod tests { #[cfg(target_os = "macos")] macos_seatbelt_profile_extensions: None, codex_linux_sandbox_exe: None, - use_linux_sandbox_bwrap: false, + use_legacy_landlock: false, windows_sandbox_level: WindowsSandboxLevel::Disabled, }) .expect("transform"); @@ -1219,7 +1219,7 @@ mod tests { #[cfg(target_os = "macos")] macos_seatbelt_profile_extensions: None, codex_linux_sandbox_exe: None, - use_linux_sandbox_bwrap: false, + use_legacy_landlock: false, windows_sandbox_level: WindowsSandboxLevel::Disabled, }) .expect("transform"); @@ -1291,7 +1291,7 @@ mod tests { #[cfg(target_os = "macos")] macos_seatbelt_profile_extensions: None, codex_linux_sandbox_exe: None, - use_linux_sandbox_bwrap: false, + use_legacy_landlock: false, windows_sandbox_level: WindowsSandboxLevel::Disabled, }) .expect("transform"); diff --git a/codex-rs/core/src/tools/js_repl/mod.rs b/codex-rs/core/src/tools/js_repl/mod.rs index 1fb3d528fe..a6ce016ffe 100644 --- a/codex-rs/core/src/tools/js_repl/mod.rs +++ b/codex-rs/core/src/tools/js_repl/mod.rs @@ -884,9 +884,7 @@ impl JsReplManager { #[cfg(target_os = "macos")] macos_seatbelt_profile_extensions: None, codex_linux_sandbox_exe: turn.codex_linux_sandbox_exe.as_ref(), - use_linux_sandbox_bwrap: turn - .features - .enabled(crate::features::Feature::UseLinuxSandboxBwrap), + use_legacy_landlock: turn.features.use_legacy_landlock(), windows_sandbox_level: turn.windows_sandbox_level, }) .map_err(|err| format!("failed to configure sandbox for js_repl: {err}"))?; diff --git a/codex-rs/core/src/tools/orchestrator.rs b/codex-rs/core/src/tools/orchestrator.rs index b26ea0bee8..d773bd913c 100644 --- a/codex-rs/core/src/tools/orchestrator.rs +++ b/codex-rs/core/src/tools/orchestrator.rs @@ -9,7 +9,6 @@ caching). use crate::error::CodexErr; use crate::error::SandboxErr; use crate::exec::ExecToolCallOutput; -use crate::features::Feature; use crate::guardian::GUARDIAN_REJECTION_MESSAGE; use crate::guardian::routes_approval_to_guardian; use crate::network_policy_decision::network_approval_context_from_payload; @@ -186,7 +185,7 @@ impl ToolOrchestrator { // Platform-specific flag gating is handled by SandboxManager::select_initial // via crate::safety::get_platform_sandbox(..). - let use_linux_sandbox_bwrap = turn_ctx.features.enabled(Feature::UseLinuxSandboxBwrap); + let use_legacy_landlock = turn_ctx.features.use_legacy_landlock(); let initial_attempt = SandboxAttempt { sandbox: initial_sandbox, policy: &turn_ctx.sandbox_policy, @@ -196,7 +195,7 @@ impl ToolOrchestrator { manager: &self.sandbox, sandbox_cwd: &turn_ctx.cwd, codex_linux_sandbox_exe: turn_ctx.codex_linux_sandbox_exe.as_ref(), - use_linux_sandbox_bwrap, + use_legacy_landlock, windows_sandbox_level: turn_ctx.windows_sandbox_level, }; @@ -318,7 +317,7 @@ impl ToolOrchestrator { manager: &self.sandbox, sandbox_cwd: &turn_ctx.cwd, codex_linux_sandbox_exe: None, - use_linux_sandbox_bwrap, + use_legacy_landlock, windows_sandbox_level: turn_ctx.windows_sandbox_level, }; diff --git a/codex-rs/core/src/tools/registry.rs b/codex-rs/core/src/tools/registry.rs index 47c2e12b63..088e3c8364 100644 --- a/codex-rs/core/src/tools/registry.rs +++ b/codex-rs/core/src/tools/registry.rs @@ -4,7 +4,6 @@ use std::time::Duration; use std::time::Instant; use crate::client_common::tools::ToolSpec; -use crate::features::Feature; use crate::function_tool::FunctionCallError; use crate::memories::usage::emit_metric_for_tool_read; use crate::protocol::SandboxPolicy; @@ -174,10 +173,7 @@ impl ToolRegistry { sandbox_tag( &invocation.turn.sandbox_policy, invocation.turn.windows_sandbox_level, - invocation - .turn - .features - .enabled(Feature::UseLinuxSandboxBwrap), + invocation.turn.features.use_legacy_landlock(), ), ), ( @@ -505,7 +501,7 @@ async fn dispatch_after_tool_use_hook( sandbox: sandbox_tag( &turn.sandbox_policy, turn.windows_sandbox_level, - turn.features.enabled(Feature::UseLinuxSandboxBwrap), + turn.features.use_legacy_landlock(), ) .to_string(), sandbox_policy: sandbox_policy_tag(&turn.sandbox_policy).to_string(), diff --git a/codex-rs/core/src/tools/runtimes/shell/unix_escalation.rs b/codex-rs/core/src/tools/runtimes/shell/unix_escalation.rs index f0be71210e..36260d59a0 100644 --- a/codex-rs/core/src/tools/runtimes/shell/unix_escalation.rs +++ b/codex-rs/core/src/tools/runtimes/shell/unix_escalation.rs @@ -146,7 +146,7 @@ pub(super) async fn try_run_zsh_fork( .macos_seatbelt_profile_extensions .clone(), codex_linux_sandbox_exe: ctx.turn.codex_linux_sandbox_exe.clone(), - use_linux_sandbox_bwrap: ctx.turn.features.enabled(Feature::UseLinuxSandboxBwrap), + use_legacy_landlock: ctx.turn.features.use_legacy_landlock(), }; let main_execve_wrapper_exe = ctx .session @@ -258,7 +258,7 @@ pub(crate) async fn prepare_unified_exec_zsh_fork( .macos_seatbelt_profile_extensions .clone(), codex_linux_sandbox_exe: ctx.turn.codex_linux_sandbox_exe.clone(), - use_linux_sandbox_bwrap: ctx.turn.features.enabled(Feature::UseLinuxSandboxBwrap), + use_legacy_landlock: ctx.turn.features.use_legacy_landlock(), }; let main_execve_wrapper_exe = ctx .session @@ -855,7 +855,7 @@ struct CoreShellCommandExecutor { #[cfg_attr(not(target_os = "macos"), allow(dead_code))] macos_seatbelt_profile_extensions: Option, codex_linux_sandbox_exe: Option, - use_linux_sandbox_bwrap: bool, + use_legacy_landlock: bool, } struct PrepareSandboxedExecParams<'a> { @@ -1052,7 +1052,7 @@ impl CoreShellCommandExecutor { #[cfg(target_os = "macos")] macos_seatbelt_profile_extensions, codex_linux_sandbox_exe: self.codex_linux_sandbox_exe.as_ref(), - use_linux_sandbox_bwrap: self.use_linux_sandbox_bwrap, + use_legacy_landlock: self.use_legacy_landlock, windows_sandbox_level: self.windows_sandbox_level, })?; if let Some(network) = exec_request.network.as_ref() { diff --git a/codex-rs/core/src/tools/runtimes/shell/unix_escalation_tests.rs b/codex-rs/core/src/tools/runtimes/shell/unix_escalation_tests.rs index 4f9f31a788..a1a34cff41 100644 --- a/codex-rs/core/src/tools/runtimes/shell/unix_escalation_tests.rs +++ b/codex-rs/core/src/tools/runtimes/shell/unix_escalation_tests.rs @@ -611,7 +611,7 @@ async fn prepare_escalated_exec_turn_default_preserves_macos_seatbelt_extensions ..Default::default() }), codex_linux_sandbox_exe: None, - use_linux_sandbox_bwrap: false, + use_legacy_landlock: false, }; let prepared = executor @@ -660,7 +660,7 @@ async fn prepare_escalated_exec_permissions_preserve_macos_seatbelt_extensions() sandbox_policy_cwd: cwd.to_path_buf(), macos_seatbelt_profile_extensions: None, codex_linux_sandbox_exe: None, - use_linux_sandbox_bwrap: false, + use_legacy_landlock: false, }; let permissions = Permissions { @@ -737,7 +737,7 @@ async fn prepare_escalated_exec_permission_profile_unions_turn_and_requested_mac ..Default::default() }), codex_linux_sandbox_exe: None, - use_linux_sandbox_bwrap: false, + use_legacy_landlock: false, }; let prepared = executor diff --git a/codex-rs/core/src/tools/sandboxing.rs b/codex-rs/core/src/tools/sandboxing.rs index 9225444715..c133dea7d8 100644 --- a/codex-rs/core/src/tools/sandboxing.rs +++ b/codex-rs/core/src/tools/sandboxing.rs @@ -330,7 +330,7 @@ pub(crate) struct SandboxAttempt<'a> { pub(crate) manager: &'a SandboxManager, pub(crate) sandbox_cwd: &'a Path, pub codex_linux_sandbox_exe: Option<&'a std::path::PathBuf>, - pub use_linux_sandbox_bwrap: bool, + pub use_legacy_landlock: bool, pub windows_sandbox_level: codex_protocol::config_types::WindowsSandboxLevel, } @@ -353,7 +353,7 @@ impl<'a> SandboxAttempt<'a> { #[cfg(target_os = "macos")] macos_seatbelt_profile_extensions: None, codex_linux_sandbox_exe: self.codex_linux_sandbox_exe, - use_linux_sandbox_bwrap: self.use_linux_sandbox_bwrap, + use_legacy_landlock: self.use_legacy_landlock, windows_sandbox_level: self.windows_sandbox_level, }) } diff --git a/codex-rs/core/src/turn_metadata.rs b/codex-rs/core/src/turn_metadata.rs index 0814897d91..0ddaad9186 100644 --- a/codex-rs/core/src/turn_metadata.rs +++ b/codex-rs/core/src/turn_metadata.rs @@ -132,16 +132,11 @@ impl TurnMetadataState { cwd: PathBuf, sandbox_policy: &SandboxPolicy, windows_sandbox_level: WindowsSandboxLevel, - use_linux_sandbox_bwrap: bool, + use_legacy_landlock: bool, ) -> Self { let repo_root = get_git_repo_root(&cwd).map(|root| root.to_string_lossy().into_owned()); let sandbox = Some( - sandbox_tag( - sandbox_policy, - windows_sandbox_level, - use_linux_sandbox_bwrap, - ) - .to_string(), + sandbox_tag(sandbox_policy, windows_sandbox_level, use_legacy_landlock).to_string(), ); let base_metadata = build_turn_metadata_bag(Some(turn_id), sandbox, None, None); let base_header = base_metadata @@ -300,19 +295,19 @@ mod tests { } #[test] - fn turn_metadata_state_respects_linux_bubblewrap_toggle() { + fn turn_metadata_state_respects_legacy_landlock_flag() { let temp_dir = TempDir::new().expect("temp dir"); let cwd = temp_dir.path().to_path_buf(); let sandbox_policy = SandboxPolicy::new_read_only_policy(); - let without_bubblewrap = TurnMetadataState::new( + let default_bubblewrap = TurnMetadataState::new( "turn-a".to_string(), cwd.clone(), &sandbox_policy, WindowsSandboxLevel::Disabled, false, ); - let with_bubblewrap = TurnMetadataState::new( + let legacy_landlock = TurnMetadataState::new( "turn-b".to_string(), cwd, &sandbox_policy, @@ -320,30 +315,33 @@ mod tests { true, ); - let without_bubblewrap_header = without_bubblewrap + let default_bubblewrap_header = default_bubblewrap .current_header_value() - .expect("without_bubblewrap_header"); - let with_bubblewrap_header = with_bubblewrap + .expect("default_bubblewrap_header"); + let legacy_landlock_header = legacy_landlock .current_header_value() - .expect("with_bubblewrap_header"); + .expect("legacy_landlock_header"); - let without_bubblewrap_json: Value = - serde_json::from_str(&without_bubblewrap_header).expect("without_bubblewrap_json"); - let with_bubblewrap_json: Value = - serde_json::from_str(&with_bubblewrap_header).expect("with_bubblewrap_json"); + let default_bubblewrap_json: Value = + serde_json::from_str(&default_bubblewrap_header).expect("default_bubblewrap_json"); + let legacy_landlock_json: Value = + serde_json::from_str(&legacy_landlock_header).expect("legacy_landlock_json"); - let without_bubblewrap_sandbox = without_bubblewrap_json + let default_bubblewrap_sandbox = default_bubblewrap_json .get("sandbox") .and_then(Value::as_str); - let with_bubblewrap_sandbox = with_bubblewrap_json.get("sandbox").and_then(Value::as_str); + let legacy_landlock_sandbox = legacy_landlock_json.get("sandbox").and_then(Value::as_str); - let expected_with_bubblewrap = - sandbox_tag(&sandbox_policy, WindowsSandboxLevel::Disabled, true); - assert_eq!(with_bubblewrap_sandbox, Some(expected_with_bubblewrap)); + let expected_default_bubblewrap = + sandbox_tag(&sandbox_policy, WindowsSandboxLevel::Disabled, false); + assert_eq!( + default_bubblewrap_sandbox, + Some(expected_default_bubblewrap) + ); if cfg!(target_os = "linux") { - assert_eq!(with_bubblewrap_sandbox, Some("linux_bubblewrap")); - assert_ne!(with_bubblewrap_sandbox, without_bubblewrap_sandbox); + assert_eq!(default_bubblewrap_sandbox, Some("linux_bubblewrap")); + assert_ne!(default_bubblewrap_sandbox, legacy_landlock_sandbox); } } } diff --git a/codex-rs/core/tests/suite/approvals.rs b/codex-rs/core/tests/suite/approvals.rs index abeb792afc..66c30f6143 100644 --- a/codex-rs/core/tests/suite/approvals.rs +++ b/codex-rs/core/tests/suite/approvals.rs @@ -1321,7 +1321,7 @@ fn scenarios() -> Vec { expectation: Expectation::FileNotCreated { target: TargetPath::Workspace("ro_never.txt"), message_contains: if cfg!(target_os = "linux") { - &["Permission denied"] + &["Permission denied|Read-only file system"] } else { &[ "Permission denied|Operation not permitted|operation not permitted|\ @@ -1468,7 +1468,7 @@ fn scenarios() -> Vec { expectation: Expectation::FileNotCreated { target: TargetPath::OutsideWorkspace("ww_never.txt"), message_contains: if cfg!(target_os = "linux") { - &["Permission denied"] + &["Permission denied|Read-only file system"] } else { &[ "Permission denied|Operation not permitted|operation not permitted|\ @@ -2290,20 +2290,17 @@ allow_local_binding = true test.config.permissions.network.is_some(), "expected managed network proxy config to be present" ); - let runtime_proxy = test - .session_configured + test.session_configured .network_proxy .as_ref() .expect("expected runtime managed network proxy addresses"); - let proxy_addr = runtime_proxy.http_addr.as_str(); let call_id_first = "allow-network-first"; - // Use the same urllib-based pattern as the other network integration tests, - // but point it at the runtime proxy directly so the blocked host reliably - // produces a network approval request without relying on curl. - let fetch_command = format!( - "python3 -c \"import urllib.request; proxy = urllib.request.ProxyHandler({{'http': 'http://{proxy_addr}'}}); opener = urllib.request.build_opener(proxy); print('OK:' + opener.open('http://codex-network-test.invalid', timeout=30).read().decode(errors='replace'))\"" - ); + // Use urllib without overriding proxy settings so managed-network sessions + // continue to exercise the env-based proxy routing path under bubblewrap. + let fetch_command = + "python3 -c \"import urllib.request; opener = urllib.request.build_opener(urllib.request.ProxyHandler()); print('OK:' + opener.open('http://codex-network-test.invalid', timeout=30).read().decode(errors='replace'))\"" + .to_string(); let first_event = shell_event( call_id_first, &fetch_command, diff --git a/codex-rs/linux-sandbox/README.md b/codex-rs/linux-sandbox/README.md index 32d8d99f01..5b02be8bdc 100644 --- a/codex-rs/linux-sandbox/README.md +++ b/codex-rs/linux-sandbox/README.md @@ -12,29 +12,28 @@ into this binary. **Current Behavior** - Legacy Landlock + mount protections remain available as the legacy pipeline. -- The bubblewrap pipeline is standardized on the vendored path. -- During rollout, the bubblewrap pipeline is gated by the temporary feature - flag `use_linux_sandbox_bwrap` (CLI `-c` alias for - `features.use_linux_sandbox_bwrap`; legacy remains default when off). -- When enabled, the bubblewrap pipeline applies `PR_SET_NO_NEW_PRIVS` and a +- The default Linux sandbox pipeline is bubblewrap on the vendored path. +- Set `features.use_legacy_landlock = true` (or CLI `-c use_legacy_landlock=true`) + to force the legacy Landlock fallback. +- When the default bubblewrap pipeline is active, it applies `PR_SET_NO_NEW_PRIVS` and a seccomp network filter in-process. -- When enabled, the filesystem is read-only by default via `--ro-bind / /`. -- When enabled, writable roots are layered with `--bind `. -- When enabled, protected subpaths under writable roots (for example `.git`, +- When the default bubblewrap pipeline is active, the filesystem is read-only by default via `--ro-bind / /`. +- When the default bubblewrap pipeline is active, writable roots are layered with `--bind `. +- When the default bubblewrap pipeline is active, protected subpaths under writable roots (for example `.git`, resolved `gitdir:`, and `.codex`) are re-applied as read-only via `--ro-bind`. -- When enabled, symlink-in-path and non-existent protected paths inside +- When the default bubblewrap pipeline is active, symlink-in-path and non-existent protected paths inside writable roots are blocked by mounting `/dev/null` on the symlink or first missing component. -- When enabled, the helper explicitly isolates the user namespace via +- When the default bubblewrap pipeline is active, the helper explicitly isolates the user namespace via `--unshare-user` and the PID namespace via `--unshare-pid`. -- When enabled and network is restricted without proxy routing, the helper also +- When the default bubblewrap pipeline is active and network is restricted without proxy routing, the helper also isolates the network namespace via `--unshare-net`. - In managed proxy mode, the helper uses `--unshare-net` plus an internal TCP->UDS->TCP routing bridge so tool traffic reaches only configured proxy endpoints. - In managed proxy mode, after the bridge is live, seccomp blocks new AF_UNIX/socketpair creation for the user command. -- When enabled, it mounts a fresh `/proc` via `--proc /proc` by default, but +- When the default bubblewrap pipeline is active, it mounts a fresh `/proc` via `--proc /proc` by default, but you can skip this in restrictive container environments with `--no-proc`. **Notes** diff --git a/codex-rs/linux-sandbox/src/linux_run_main.rs b/codex-rs/linux-sandbox/src/linux_run_main.rs index e6304d2d60..9a5a4c738d 100644 --- a/codex-rs/linux-sandbox/src/linux_run_main.rs +++ b/codex-rs/linux-sandbox/src/linux_run_main.rs @@ -22,7 +22,8 @@ use codex_protocol::protocol::SandboxPolicy; /// CLI surface for the Linux sandbox helper. /// /// The type name remains `LandlockCommand` for compatibility with existing -/// wiring, but the filesystem sandbox now uses bubblewrap. +/// wiring, but bubblewrap is now the default filesystem sandbox and Landlock +/// is the legacy fallback. pub struct LandlockCommand { /// It is possible that the cwd used in the context of the sandbox policy /// is different from the cwd of the process to spawn. @@ -42,11 +43,11 @@ pub struct LandlockCommand { #[arg(long = "network-sandbox-policy", hide = true)] pub network_sandbox_policy: Option, - /// Opt-in: use the bubblewrap-based Linux sandbox pipeline. + /// Opt-in: use the legacy Landlock Linux sandbox fallback. /// - /// When not set, we fall back to the legacy Landlock + mount pipeline. - #[arg(long = "use-bwrap-sandbox", hide = true, default_value_t = false)] - pub use_bwrap_sandbox: bool, + /// When not set, the helper uses the default bubblewrap pipeline. + #[arg(long = "use-legacy-landlock", hide = true, default_value_t = false)] + pub use_legacy_landlock: bool, /// Internal: apply seccomp and `no_new_privs` in the already-sandboxed /// process, then exec the user command. @@ -92,7 +93,7 @@ pub fn run_main() -> ! { sandbox_policy, file_system_sandbox_policy, network_sandbox_policy, - use_bwrap_sandbox, + use_legacy_landlock, apply_seccomp_then_exec, allow_network_for_proxy, proxy_route_spec, @@ -103,7 +104,7 @@ pub fn run_main() -> ! { if command.is_empty() { panic!("No command specified to execute."); } - ensure_inner_stage_mode_is_valid(apply_seccomp_then_exec, use_bwrap_sandbox); + ensure_inner_stage_mode_is_valid(apply_seccomp_then_exec, use_legacy_landlock); let EffectiveSandboxPolicies { sandbox_policy, file_system_sandbox_policy, @@ -154,7 +155,7 @@ pub fn run_main() -> ! { exec_or_panic(command); } - if use_bwrap_sandbox { + if !use_legacy_landlock { // Outer stage: bubblewrap first, then re-enter this binary in the // sandboxed environment to apply seccomp. This path never falls back // to legacy Landlock on failure. @@ -171,7 +172,6 @@ pub fn run_main() -> ! { sandbox_policy: &sandbox_policy, file_system_sandbox_policy: &file_system_sandbox_policy, network_sandbox_policy, - use_bwrap_sandbox, allow_network_for_proxy, proxy_route_spec, command, @@ -256,9 +256,9 @@ fn resolve_sandbox_policies( } } -fn ensure_inner_stage_mode_is_valid(apply_seccomp_then_exec: bool, use_bwrap_sandbox: bool) { - if apply_seccomp_then_exec && !use_bwrap_sandbox { - panic!("--apply-seccomp-then-exec requires --use-bwrap-sandbox"); +fn ensure_inner_stage_mode_is_valid(apply_seccomp_then_exec: bool, use_legacy_landlock: bool) { + if apply_seccomp_then_exec && use_legacy_landlock { + panic!("--apply-seccomp-then-exec is incompatible with --use-legacy-landlock"); } } @@ -280,7 +280,8 @@ fn run_bwrap_with_proc_fallback( network_mode, ) { - eprintln!("codex-linux-sandbox: bwrap could not mount /proc; retrying with --no-proc"); + // Keep the retry silent so sandbox-internal diagnostics do not leak into the + // child process stderr stream. mount_proc = false; } @@ -470,7 +471,6 @@ struct InnerSeccompCommandArgs<'a> { sandbox_policy: &'a SandboxPolicy, file_system_sandbox_policy: &'a FileSystemSandboxPolicy, network_sandbox_policy: NetworkSandboxPolicy, - use_bwrap_sandbox: bool, allow_network_for_proxy: bool, proxy_route_spec: Option, command: Vec, @@ -483,7 +483,6 @@ fn build_inner_seccomp_command(args: InnerSeccompCommandArgs<'_>) -> Vec sandbox_policy, file_system_sandbox_policy, network_sandbox_policy, - use_bwrap_sandbox, allow_network_for_proxy, proxy_route_spec, command, @@ -516,10 +515,7 @@ fn build_inner_seccomp_command(args: InnerSeccompCommandArgs<'_>) -> Vec "--network-sandbox-policy".to_string(), network_policy_json, ]; - if use_bwrap_sandbox { - inner.push("--use-bwrap-sandbox".to_string()); - inner.push("--apply-seccomp-then-exec".to_string()); - } + inner.push("--apply-seccomp-then-exec".to_string()); if allow_network_for_proxy { inner.push("--allow-network-for-proxy".to_string()); let proxy_route_spec = proxy_route_spec diff --git a/codex-rs/linux-sandbox/src/linux_run_main_tests.rs b/codex-rs/linux-sandbox/src/linux_run_main_tests.rs index a1adf65b1d..dec1c12144 100644 --- a/codex-rs/linux-sandbox/src/linux_run_main_tests.rs +++ b/codex-rs/linux-sandbox/src/linux_run_main_tests.rs @@ -127,7 +127,6 @@ fn managed_proxy_inner_command_includes_route_spec() { sandbox_policy: &sandbox_policy, file_system_sandbox_policy: &FileSystemSandboxPolicy::from(&sandbox_policy), network_sandbox_policy: NetworkSandboxPolicy::Restricted, - use_bwrap_sandbox: true, allow_network_for_proxy: true, proxy_route_spec: Some("{\"routes\":[]}".to_string()), command: vec!["/bin/true".to_string()], @@ -145,7 +144,6 @@ fn inner_command_includes_split_policy_flags() { sandbox_policy: &sandbox_policy, file_system_sandbox_policy: &FileSystemSandboxPolicy::from(&sandbox_policy), network_sandbox_policy: NetworkSandboxPolicy::Restricted, - use_bwrap_sandbox: true, allow_network_for_proxy: false, proxy_route_spec: None, command: vec!["/bin/true".to_string()], @@ -163,7 +161,6 @@ fn non_managed_inner_command_omits_route_spec() { sandbox_policy: &sandbox_policy, file_system_sandbox_policy: &FileSystemSandboxPolicy::from(&sandbox_policy), network_sandbox_policy: NetworkSandboxPolicy::Restricted, - use_bwrap_sandbox: true, allow_network_for_proxy: false, proxy_route_spec: None, command: vec!["/bin/true".to_string()], @@ -181,7 +178,6 @@ fn managed_proxy_inner_command_requires_route_spec() { sandbox_policy: &sandbox_policy, file_system_sandbox_policy: &FileSystemSandboxPolicy::from(&sandbox_policy), network_sandbox_policy: NetworkSandboxPolicy::Restricted, - use_bwrap_sandbox: true, allow_network_for_proxy: true, proxy_route_spec: None, command: vec!["/bin/true".to_string()], @@ -244,8 +240,8 @@ fn resolve_sandbox_policies_rejects_partial_split_policies() { } #[test] -fn apply_seccomp_then_exec_without_bwrap_panics() { - let result = std::panic::catch_unwind(|| ensure_inner_stage_mode_is_valid(true, false)); +fn apply_seccomp_then_exec_with_legacy_landlock_panics() { + let result = std::panic::catch_unwind(|| ensure_inner_stage_mode_is_valid(true, true)); assert!(result.is_err()); } @@ -253,5 +249,5 @@ fn apply_seccomp_then_exec_without_bwrap_panics() { fn valid_inner_stage_modes_do_not_panic() { ensure_inner_stage_mode_is_valid(false, false); ensure_inner_stage_mode_is_valid(false, true); - ensure_inner_stage_mode_is_valid(true, true); + ensure_inner_stage_mode_is_valid(true, false); } diff --git a/codex-rs/linux-sandbox/tests/suite/landlock.rs b/codex-rs/linux-sandbox/tests/suite/landlock.rs index 8e2f5ef41e..774fb4f17b 100644 --- a/codex-rs/linux-sandbox/tests/suite/landlock.rs +++ b/codex-rs/linux-sandbox/tests/suite/landlock.rs @@ -72,7 +72,7 @@ async fn run_cmd_result_with_writable_roots( cmd: &[&str], writable_roots: &[PathBuf], timeout_ms: u64, - use_bwrap_sandbox: bool, + use_legacy_landlock: bool, network_access: bool, ) -> Result { let sandbox_policy = SandboxPolicy::WorkspaceWrite { @@ -96,7 +96,7 @@ async fn run_cmd_result_with_writable_roots( file_system_sandbox_policy, network_sandbox_policy, timeout_ms, - use_bwrap_sandbox, + use_legacy_landlock, ) .await } @@ -108,7 +108,7 @@ async fn run_cmd_result_with_policies( file_system_sandbox_policy: FileSystemSandboxPolicy, network_sandbox_policy: NetworkSandboxPolicy, timeout_ms: u64, - use_bwrap_sandbox: bool, + use_legacy_landlock: bool, ) -> Result { let cwd = std::env::current_dir().expect("cwd should exist"); let sandbox_cwd = cwd.clone(); @@ -133,7 +133,7 @@ async fn run_cmd_result_with_policies( network_sandbox_policy, sandbox_cwd.as_path(), &codex_linux_sandbox_exe, - use_bwrap_sandbox, + use_legacy_landlock, None, ) .await @@ -155,7 +155,7 @@ async fn should_skip_bwrap_tests() -> bool { &["bash", "-lc", "true"], &[], NETWORK_TIMEOUT_MS, - true, + false, true, ) .await @@ -216,7 +216,7 @@ async fn test_dev_null_write() { // We have seen timeouts when running this test in CI on GitHub, // so we are using a generous timeout until we can diagnose further. LONG_TIMEOUT_MS, - true, + false, true, ) .await @@ -240,7 +240,7 @@ async fn bwrap_populates_minimal_dev_nodes() { ], &[], LONG_TIMEOUT_MS, - true, + false, true, ) .await @@ -278,7 +278,7 @@ async fn bwrap_preserves_writable_dev_shm_bind_mount() { ], &[PathBuf::from("/dev/shm")], LONG_TIMEOUT_MS, - true, + false, true, ) .await @@ -442,7 +442,7 @@ async fn sandbox_blocks_git_and_codex_writes_inside_writable_root() { ], &[tmpdir.path().to_path_buf()], LONG_TIMEOUT_MS, - true, + false, true, ) .await, @@ -458,7 +458,7 @@ async fn sandbox_blocks_git_and_codex_writes_inside_writable_root() { ], &[tmpdir.path().to_path_buf()], LONG_TIMEOUT_MS, - true, + false, true, ) .await, @@ -495,7 +495,7 @@ async fn sandbox_blocks_codex_symlink_replacement_attack() { ], &[tmpdir.path().to_path_buf()], LONG_TIMEOUT_MS, - true, + false, true, ) .await, @@ -548,7 +548,7 @@ async fn sandbox_blocks_explicit_split_policy_carveouts_under_bwrap() { file_system_sandbox_policy, NetworkSandboxPolicy::Enabled, LONG_TIMEOUT_MS, - true, + false, ) .await, "explicit split-policy carveout should be denied under bubblewrap", @@ -599,7 +599,7 @@ async fn sandbox_blocks_root_read_carveouts_under_bwrap() { file_system_sandbox_policy, NetworkSandboxPolicy::Enabled, LONG_TIMEOUT_MS, - true, + false, ) .await, "root-read carveout should be denied under bubblewrap", diff --git a/codex-rs/linux-sandbox/tests/suite/managed_proxy.rs b/codex-rs/linux-sandbox/tests/suite/managed_proxy.rs index e27930cdb5..4d7aa2ac7c 100644 --- a/codex-rs/linux-sandbox/tests/suite/managed_proxy.rs +++ b/codex-rs/linux-sandbox/tests/suite/managed_proxy.rs @@ -133,7 +133,6 @@ async fn run_linux_sandbox_direct( cwd.to_string_lossy().to_string(), "--sandbox-policy".to_string(), policy_json, - "--use-bwrap-sandbox".to_string(), ]; if allow_network_for_proxy { args.push("--allow-network-for-proxy".to_string()); diff --git a/codex-rs/utils/cli/src/config_override.rs b/codex-rs/utils/cli/src/config_override.rs index 1592563a0b..41be3ca6b6 100644 --- a/codex-rs/utils/cli/src/config_override.rs +++ b/codex-rs/utils/cli/src/config_override.rs @@ -89,8 +89,8 @@ impl CliConfigOverrides { } fn canonicalize_override_key(key: &str) -> String { - if key == "use_linux_sandbox_bwrap" { - "features.use_linux_sandbox_bwrap".to_string() + if key == "use_legacy_landlock" { + "features.use_legacy_landlock".to_string() } else { key.to_string() } @@ -181,12 +181,12 @@ mod tests { } #[test] - fn canonicalizes_use_linux_sandbox_bwrap_alias() { + fn canonicalizes_use_legacy_landlock_alias() { let overrides = CliConfigOverrides { - raw_overrides: vec!["use_linux_sandbox_bwrap=true".to_string()], + raw_overrides: vec!["use_legacy_landlock=true".to_string()], }; let parsed = overrides.parse_overrides().expect("parse_overrides"); - assert_eq!(parsed[0].0.as_str(), "features.use_linux_sandbox_bwrap"); + assert_eq!(parsed[0].0.as_str(), "features.use_legacy_landlock"); assert_eq!(parsed[0].1.as_bool(), Some(true)); }