From 1026e9de1be292ebb01579dfcce5a34ab224917a Mon Sep 17 00:00:00 2001 From: jif Date: Tue, 9 Jun 2026 19:40:40 +0200 Subject: [PATCH 01/47] app-server: reject direct input to multi-agent v2 sub-agents (#27173) ## Why Multi-agent v2 sub-agents are owned and coordinated by their parent agent. Allowing an app-server client to start or steer turns on a spawned child bypasses the multi-agent messaging path and creates a second, conflicting source of work for that sub-agent. ## What changed - Reject direct `turn/start` and `turn/steer` requests targeting multi-agent v2 thread-spawn sub-agents. - Identify these targets using both the thread's resolved multi-agent version and its `SubAgentSource::ThreadSpawn` session source, leaving root threads, v1 agents, and other sub-agent types unchanged. - Return a consistent invalid-request error before validating or applying the submitted input. ## Testing - Added an app-server integration test that spawns a real multi-agent v2 child and verifies that direct `turn/start` and `turn/steer` requests are rejected. --- .../src/request_processors/turn_processor.rs | 42 +++++- .../app-server/tests/suite/v2/turn_start.rs | 130 ++++++++++++++++++ 2 files changed, 166 insertions(+), 6 deletions(-) diff --git a/codex-rs/app-server/src/request_processors/turn_processor.rs b/codex-rs/app-server/src/request_processors/turn_processor.rs index 7bdd9df0d8..3d44781946 100644 --- a/codex-rs/app-server/src/request_processors/turn_processor.rs +++ b/codex-rs/app-server/src/request_processors/turn_processor.rs @@ -1,6 +1,12 @@ use super::*; use codex_protocol::protocol::AdditionalContextEntry as CoreAdditionalContextEntry; use codex_protocol::protocol::AdditionalContextKind as CoreAdditionalContextKind; +use codex_protocol::protocol::MultiAgentVersion; +use codex_protocol::protocol::SessionSource; +use codex_protocol::protocol::SubAgentSource; + +const DIRECT_INPUT_TO_MULTI_AGENT_V2_SUBAGENT_ERROR: &str = + "direct app-server input is not allowed for multi-agent v2 sub-agents"; #[derive(Clone)] pub(crate) struct TurnRequestProcessor { @@ -236,6 +242,26 @@ impl TurnRequestProcessor { Ok((thread_id, thread)) } + + async fn ensure_direct_input_allowed( + &self, + request_id: &ConnectionRequestId, + thread: &CodexThread, + ) -> Result<(), JSONRPCErrorError> { + if thread.multi_agent_version() == Some(MultiAgentVersion::V2) + && matches!( + thread.config_snapshot().await.session_source, + SessionSource::SubAgent(SubAgentSource::ThreadSpawn { .. }) + ) + { + let error = invalid_request(DIRECT_INPUT_TO_MULTI_AGENT_V2_SUBAGENT_ERROR); + self.track_error_response(request_id, &error, /*error_type*/ None); + return Err(error); + } + + Ok(()) + } + fn normalize_collaboration_mode( &self, mut collaboration_mode: CollaborationMode, @@ -370,6 +396,14 @@ impl TurnRequestProcessor { app_server_client_name: Option, app_server_client_version: Option, ) -> Result { + let (thread_id, thread) = + self.load_thread(¶ms.thread_id) + .await + .inspect_err(|error| { + self.track_error_response(&request_id, error, /*error_type*/ None); + })?; + self.ensure_direct_input_allowed(&request_id, thread.as_ref()) + .await?; if let Err(error) = Self::validate_v2_input_limit(¶ms.input) { self.track_error_response( &request_id, @@ -378,12 +412,6 @@ impl TurnRequestProcessor { ); return Err(error); } - let (thread_id, thread) = - self.load_thread(¶ms.thread_id) - .await - .inspect_err(|error| { - self.track_error_response(&request_id, error, /*error_type*/ None); - })?; Self::set_app_server_client_info( thread.as_ref(), app_server_client_name, @@ -762,6 +790,8 @@ impl TurnRequestProcessor { .inspect_err(|error| { self.track_error_response(request_id, error, /*error_type*/ None); })?; + self.ensure_direct_input_allowed(request_id, thread.as_ref()) + .await?; if params.expected_turn_id.is_empty() { return Err(invalid_request("expectedTurnId must not be empty")); diff --git a/codex-rs/app-server/tests/suite/v2/turn_start.rs b/codex-rs/app-server/tests/suite/v2/turn_start.rs index aed3c58697..5080937997 100644 --- a/codex-rs/app-server/tests/suite/v2/turn_start.rs +++ b/codex-rs/app-server/tests/suite/v2/turn_start.rs @@ -41,6 +41,7 @@ use codex_app_server_protocol::PatchChangeKind; use codex_app_server_protocol::RequestId; use codex_app_server_protocol::ServerRequest; use codex_app_server_protocol::ServerRequestResolvedNotification; +use codex_app_server_protocol::SubAgentActivityKind; use codex_app_server_protocol::TextElement; use codex_app_server_protocol::ThreadItem; use codex_app_server_protocol::ThreadSource; @@ -53,6 +54,7 @@ use codex_app_server_protocol::TurnStartParams; use codex_app_server_protocol::TurnStartResponse; use codex_app_server_protocol::TurnStartedNotification; use codex_app_server_protocol::TurnStatus; +use codex_app_server_protocol::TurnSteerParams; use codex_app_server_protocol::UserInput as V2UserInput; use codex_app_server_protocol::WarningNotification; use codex_config::config_toml::ConfigToml; @@ -3363,6 +3365,134 @@ async fn turn_start_emits_spawn_agent_item_with_model_metadata_v2() -> Result<() Ok(()) } +#[tokio::test] +async fn direct_input_to_multi_agent_v2_subagent_is_rejected() -> Result<()> { + const CHILD_PROMPT: &str = "child: do work"; + const PARENT_PROMPT: &str = "spawn a child and continue"; + const SPAWN_CALL_ID: &str = "spawn-call-direct-input-rejection"; + const ERROR_MESSAGE: &str = + "direct app-server input is not allowed for multi-agent v2 sub-agents"; + + let server = responses::start_mock_server().await; + let spawn_args = serde_json::to_string(&json!({ + "message": CHILD_PROMPT, + "task_name": "worker", + }))?; + let _parent_turn = responses::mount_sse_once_match( + &server, + |req: &wiremock::Request| body_contains(req, PARENT_PROMPT), + responses::sse(vec![ + responses::ev_response_created("resp-parent-1"), + responses::ev_function_call(SPAWN_CALL_ID, "spawn_agent", &spawn_args), + responses::ev_completed("resp-parent-1"), + ]), + ) + .await; + let codex_home = TempDir::new()?; + create_config_toml( + codex_home.path(), + &server.uri(), + "never", + &BTreeMap::from([(Feature::MultiAgentV2, true)]), + )?; + write_models_cache(codex_home.path())?; + + let mut mcp = TestAppServer::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let thread_req = mcp + .send_thread_start_request(ThreadStartParams { + model: Some("gpt-5.3-codex".to_string()), + ..Default::default() + }) + .await?; + let thread_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(thread_req)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response::(thread_resp)?; + + let turn_req = mcp + .send_turn_start_request(TurnStartParams { + thread_id: thread.id, + input: vec![V2UserInput::Text { + text: PARENT_PROMPT.to_string(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + let turn_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(turn_req)), + ) + .await??; + let _: TurnStartResponse = to_response(turn_resp)?; + + let child_thread_id = timeout(DEFAULT_READ_TIMEOUT, async { + loop { + let completed_notif = mcp + .read_stream_until_notification_message("item/completed") + .await?; + let completed: ItemCompletedNotification = + serde_json::from_value(completed_notif.params.expect("item/completed params"))?; + if let ThreadItem::SubAgentActivity { + id, + kind: SubAgentActivityKind::Started, + agent_thread_id, + .. + } = completed.item + && id == SPAWN_CALL_ID + { + return Ok::(agent_thread_id); + } + } + }) + .await??; + + let direct_turn_req = mcp + .send_turn_start_request(TurnStartParams { + thread_id: child_thread_id.clone(), + input: vec![V2UserInput::Text { + text: "direct app-server turn".to_string(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + let direct_turn_error: JSONRPCError = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_error_message(RequestId::Integer(direct_turn_req)), + ) + .await??; + assert_eq!(direct_turn_error.error.code, INVALID_REQUEST_ERROR_CODE); + assert_eq!(direct_turn_error.error.message, ERROR_MESSAGE); + + let direct_steer_req = mcp + .send_turn_steer_request(TurnSteerParams { + thread_id: child_thread_id, + client_user_message_id: None, + input: vec![V2UserInput::Text { + text: "direct app-server steer".to_string(), + text_elements: Vec::new(), + }], + responsesapi_client_metadata: None, + additional_context: None, + expected_turn_id: "any-active-turn".to_string(), + }) + .await?; + let direct_steer_error: JSONRPCError = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_error_message(RequestId::Integer(direct_steer_req)), + ) + .await??; + assert_eq!(direct_steer_error.error.code, INVALID_REQUEST_ERROR_CODE); + assert_eq!(direct_steer_error.error.message, ERROR_MESSAGE); + + Ok(()) +} + #[tokio::test] async fn turn_start_emits_spawn_agent_item_with_effective_role_model_metadata_v2() -> Result<()> { skip_if_no_network!(Ok(())); From 89ac3ec27cae0000da14fbe4160a79ed465843fb Mon Sep 17 00:00:00 2001 From: jif Date: Tue, 9 Jun 2026 19:51:54 +0200 Subject: [PATCH 02/47] Load selected executor skills through extensions (#27184) ## Why CCA is moving toward a split runtime where the orchestrator may not have a filesystem, while executors can expose preinstalled plugins and skills. A thread therefore needs to select capabilities without asking app-server or core to interpret executor-owned paths through the orchestrator's filesystem. The longer-term model is broader than executor skills: - A plugin is a bundle of skills, MCP servers, connectors/apps, and hooks. - A plugin root can be local, executor-owned, or hosted by a backend. - Components inside one plugin can use different access and execution mechanisms. A skill may be read from a filesystem or through backend tools; an HTTP MCP server can run without an executor; a stdio MCP server or hook needs an execution environment. - Core should carry generic extension initialization data. The extension that owns a component should discover it, expose it to the model, and invoke it through the appropriate runtime. This PR establishes that architecture through one complete vertical: selecting a root on an executor, discovering the skills beneath it, exposing those skills to the model, and reading an explicitly invoked `SKILL.md` through the same executor. ## Contract `thread/start` gains an experimental `selectedCapabilityRoots` field: ```json { "selectedCapabilityRoots": [ { "id": "deploy-plugin@1", "location": { "type": "environment", "environmentId": "workspace", "path": "/opt/codex/plugins/deploy" } } ] } ``` The root is intentionally not classified as a "plugin" or "skill" in the API. It can point at a standalone skill, a directory containing several skills, or a plugin containing skills and other components. This PR only teaches the skills extension how to consume it; later extensions can resolve MCP, connector, and hook components from the same selection. The platform-supplied `id` is stable selection identity. The location says which runtime owns the root and gives that runtime an opaque path. App-server does not inspect or canonicalize the path. ## What changed ### Generic thread extension initialization App-server converts selected roots into `ExtensionDataInit`. Core carries that generic initialization value until the final thread ID is known, then creates thread-scoped `ExtensionData` before lifecycle contributors run. This keeps `Session` and core independent of the capability-selection contract. The initialization value is consumed during construction; it is not retained as another long-lived `Session` field. ### Executor-backed skills The skills extension now owns an `ExecutorSkillProvider` that: - resolves the selected environment through `EnvironmentManager` - discovers, canonicalizes, and reads skills through that environment's `ExecutorFileSystem` - contributes the bounded selected-skill catalog as stable developer context - reads an explicitly invoked skill body through the authority that listed it - warns when an environment or root is unavailable - never falls back to the orchestrator filesystem for an executor-owned root Skill catalog and instruction fragments have hard byte bounds, which also bound them below the 10K-token per-item context limit. If a selected executor skill has the same name as a legacy local skill, the executor selection owns that invocation and the local body is not injected a second time. Existing local and bundled skill loading remains in place. Omitting `selectedCapabilityRoots` therefore preserves current local-only behavior. ## Current semantics - Only environment-owned locations are represented in this first contract. - Roots are resolved by the destination extension, not by app-server or core. - An unavailable executor or invalid root produces a warning and no capabilities from that root; it does not trigger a local-filesystem fallback. - Selection applies to a newly started active thread. - MCP servers, connectors, and hooks beneath a selected plugin root are not activated yet. - Selection is not yet persisted or inherited across resume, fork, or subagent creation. Existing local capabilities continue to behave as they do today in those flows. ## Planned vertical follow-ups 1. **Hosted HTTP MCP:** add an extension-backed HTTP MCP source that works without an executor, then replace the special-purpose MCP plugins loader with that implementation. 2. **Executor MCP:** register and execute stdio MCP servers through the environment that owns the selected plugin root. 3. **Backend skills:** add a hosted skill source whose catalog and bodies are accessed through extension tools rather than a filesystem. 4. **Connectors and hooks:** activate those components through their owning extensions, using the same selected-root boundary and component-specific runtime. 5. **Durable selection:** define the desired-selection lifecycle, persist it, and make resume, fork, and subagent inheritance explicit rather than accidental. 6. **Local convergence:** incrementally route existing local plugin, skill, and MCP loading through the same extension model while preserving current local behavior. Each follow-up remains reviewable as an end-to-end capability. The platform selects roots, generic thread extension data carries the selection, and the owning extension resolves and operates its component. ## Verification Coverage added for: - app-server end-to-end discovery and explicit invocation of a skill inside an executor-selected plugin root - exclusive invocation when a selected executor skill collides with a local skill name - executor filesystem authority for discovery, canonicalization, and reads - thread extension initialization before lifecycle contributors run - stable executor catalog context, explicit invocation, context rebuilding, hidden skills, and preserved host/remote catalog behavior Targeted protocol, core-skills, skills-extension, core lifecycle, and app-server executor-skill tests were run during development. --- codex-rs/Cargo.lock | 4 + codex-rs/Cargo.toml | 1 + .../schema/json/ClientRequest.json | 52 +++ .../codex_app_server_protocol.schemas.json | 52 +++ .../codex_app_server_protocol.v2.schemas.json | 52 +++ .../schema/json/v2/ThreadStartParams.json | 52 +++ .../typescript/v2/CapabilityRootLocation.ts | 8 + .../typescript/v2/SelectedCapabilityRoot.ts | 17 + .../schema/typescript/v2/index.ts | 2 + .../src/protocol/v2/thread.rs | 6 + codex-rs/app-server/Cargo.toml | 1 + codex-rs/app-server/README.md | 14 +- codex-rs/app-server/src/extensions.rs | 6 + codex-rs/app-server/src/mcp_refresh.rs | 10 +- codex-rs/app-server/src/message_processor.rs | 9 + .../request_processors/thread_processor.rs | 10 + .../tests/suite/v2/executor_skills.rs | 148 ++++++++ codex-rs/app-server/tests/suite/v2/mod.rs | 1 + .../app-server/tests/suite/v2/skills_list.rs | 1 + codex-rs/core-skills/src/loader.rs | 24 +- codex-rs/core/src/codex_delegate.rs | 1 + codex-rs/core/src/session/mod.rs | 4 + codex-rs/core/src/session/session.rs | 8 +- codex-rs/core/src/session/tests.rs | 3 + .../core/src/session/tests/guardian_tests.rs | 1 + codex-rs/core/src/thread_manager.rs | 15 + codex-rs/core/src/thread_manager_tests.rs | 79 ++++ .../tests/suite/subagent_notifications.rs | 1 + codex-rs/ext/extension-api/src/lib.rs | 1 + codex-rs/ext/extension-api/src/state.rs | 33 +- codex-rs/ext/skills/Cargo.toml | 3 + codex-rs/ext/skills/src/catalog.rs | 35 +- codex-rs/ext/skills/src/extension.rs | 104 ++++-- codex-rs/ext/skills/src/lib.rs | 2 + codex-rs/ext/skills/src/provider.rs | 5 +- codex-rs/ext/skills/src/provider/executor.rs | 197 ++++++++++ codex-rs/ext/skills/src/provider/host.rs | 10 +- codex-rs/ext/skills/src/render.rs | 29 +- codex-rs/ext/skills/src/selection.rs | 4 +- codex-rs/ext/skills/src/sources.rs | 2 +- codex-rs/ext/skills/src/state.rs | 12 +- .../tests/executor_file_system_authority.rs | 337 ++++++++++++++++++ codex-rs/ext/skills/tests/skills_extension.rs | 199 +++++++---- codex-rs/memories/write/src/runtime.rs | 1 + codex-rs/protocol/src/capabilities.rs | 30 ++ codex-rs/protocol/src/lib.rs | 1 + 46 files changed, 1460 insertions(+), 127 deletions(-) create mode 100644 codex-rs/app-server-protocol/schema/typescript/v2/CapabilityRootLocation.ts create mode 100644 codex-rs/app-server-protocol/schema/typescript/v2/SelectedCapabilityRoot.ts create mode 100644 codex-rs/app-server/tests/suite/v2/executor_skills.rs create mode 100644 codex-rs/ext/skills/src/provider/executor.rs create mode 100644 codex-rs/ext/skills/tests/executor_file_system_authority.rs create mode 100644 codex-rs/protocol/src/capabilities.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 3d8086c27b..064d9ad944 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1994,6 +1994,7 @@ dependencies = [ "codex-rollout", "codex-sandboxing", "codex-shell-command", + "codex-skills-extension", "codex-state", "codex-thread-store", "codex-tools", @@ -3796,8 +3797,11 @@ dependencies = [ "async-trait", "codex-core", "codex-core-skills", + "codex-exec-server", "codex-extension-api", "codex-protocol", + "codex-utils-absolute-path", + "codex-utils-string", "pretty_assertions", "tokio", ] diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index c72b7c7844..955d8987f7 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -212,6 +212,7 @@ codex-sandboxing = { path = "sandboxing" } codex-secrets = { path = "secrets" } codex-shell-command = { path = "shell-command" } codex-shell-escalation = { path = "shell-escalation" } +codex-skills-extension = { path = "ext/skills" } codex-skills = { path = "skills" } codex-state = { path = "state" } codex-stdio-to-uds = { path = "stdio-to-uds" } diff --git a/codex-rs/app-server-protocol/schema/json/ClientRequest.json b/codex-rs/app-server-protocol/schema/json/ClientRequest.json index 6d9be4ec02..c499376547 100644 --- a/codex-rs/app-server-protocol/schema/json/ClientRequest.json +++ b/codex-rs/app-server-protocol/schema/json/ClientRequest.json @@ -180,6 +180,36 @@ ], "type": "object" }, + "CapabilityRootLocation": { + "description": "Location used to resolve a selected capability root.", + "oneOf": [ + { + "description": "A path owned by an execution environment.", + "properties": { + "environmentId": { + "type": "string" + }, + "path": { + "type": "string" + }, + "type": { + "enum": [ + "environment" + ], + "title": "EnvironmentCapabilityRootLocationType", + "type": "string" + } + }, + "required": [ + "environmentId", + "path", + "type" + ], + "title": "EnvironmentCapabilityRootLocation", + "type": "object" + } + ] + }, "ClientInfo": { "properties": { "name": { @@ -2947,6 +2977,28 @@ } ] }, + "SelectedCapabilityRoot": { + "description": "A user-selected root that can expose one or more runtime capabilities.", + "properties": { + "id": { + "description": "Stable identifier supplied by the capability selection platform.", + "type": "string" + }, + "location": { + "allOf": [ + { + "$ref": "#/definitions/CapabilityRootLocation" + } + ], + "description": "Where the selected root can be resolved." + } + }, + "required": [ + "id", + "location" + ], + "type": "object" + }, "SendAddCreditsNudgeEmailParams": { "properties": { "creditType": { diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index 7abc5af7d0..475154b38a 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -6765,6 +6765,36 @@ ], "type": "string" }, + "CapabilityRootLocation": { + "description": "Location used to resolve a selected capability root.", + "oneOf": [ + { + "description": "A path owned by an execution environment.", + "properties": { + "environmentId": { + "type": "string" + }, + "path": { + "type": "string" + }, + "type": { + "enum": [ + "environment" + ], + "title": "EnvironmentCapabilityRootLocationType", + "type": "string" + } + }, + "required": [ + "environmentId", + "path", + "type" + ], + "title": "EnvironmentCapabilityRootLocation", + "type": "object" + } + ] + }, "CodexErrorInfo": { "description": "This translation layer make sure that we expose codex error code in camel case.\n\nWhen an upstream HTTP status is available (for example, from the Responses API or a provider), it is forwarded in `httpStatusCode` on the relevant `codexErrorInfo` variant.", "oneOf": [ @@ -15061,6 +15091,28 @@ }, "type": "object" }, + "SelectedCapabilityRoot": { + "description": "A user-selected root that can expose one or more runtime capabilities.", + "properties": { + "id": { + "description": "Stable identifier supplied by the capability selection platform.", + "type": "string" + }, + "location": { + "allOf": [ + { + "$ref": "#/definitions/v2/CapabilityRootLocation" + } + ], + "description": "Where the selected root can be resolved." + } + }, + "required": [ + "id", + "location" + ], + "type": "object" + }, "SendAddCreditsNudgeEmailParams": { "$schema": "http://json-schema.org/draft-07/schema#", "properties": { diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index e5ac77c088..8ed40282a9 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -1087,6 +1087,36 @@ ], "type": "string" }, + "CapabilityRootLocation": { + "description": "Location used to resolve a selected capability root.", + "oneOf": [ + { + "description": "A path owned by an execution environment.", + "properties": { + "environmentId": { + "type": "string" + }, + "path": { + "type": "string" + }, + "type": { + "enum": [ + "environment" + ], + "title": "EnvironmentCapabilityRootLocationType", + "type": "string" + } + }, + "required": [ + "environmentId", + "path", + "type" + ], + "title": "EnvironmentCapabilityRootLocation", + "type": "object" + } + ] + }, "ClientInfo": { "properties": { "name": { @@ -11563,6 +11593,28 @@ }, "type": "object" }, + "SelectedCapabilityRoot": { + "description": "A user-selected root that can expose one or more runtime capabilities.", + "properties": { + "id": { + "description": "Stable identifier supplied by the capability selection platform.", + "type": "string" + }, + "location": { + "allOf": [ + { + "$ref": "#/definitions/CapabilityRootLocation" + } + ], + "description": "Where the selected root can be resolved." + } + }, + "required": [ + "id", + "location" + ], + "type": "object" + }, "SendAddCreditsNudgeEmailParams": { "$schema": "http://json-schema.org/draft-07/schema#", "properties": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartParams.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartParams.json index 99b25490ab..7f0d4ad9a4 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartParams.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartParams.json @@ -64,6 +64,36 @@ } ] }, + "CapabilityRootLocation": { + "description": "Location used to resolve a selected capability root.", + "oneOf": [ + { + "description": "A path owned by an execution environment.", + "properties": { + "environmentId": { + "type": "string" + }, + "path": { + "type": "string" + }, + "type": { + "enum": [ + "environment" + ], + "title": "EnvironmentCapabilityRootLocationType", + "type": "string" + } + }, + "required": [ + "environmentId", + "path", + "type" + ], + "title": "EnvironmentCapabilityRootLocation", + "type": "object" + } + ] + }, "DynamicToolSpec": { "properties": { "deferLoading": { @@ -106,6 +136,28 @@ ], "type": "string" }, + "SelectedCapabilityRoot": { + "description": "A user-selected root that can expose one or more runtime capabilities.", + "properties": { + "id": { + "description": "Stable identifier supplied by the capability selection platform.", + "type": "string" + }, + "location": { + "allOf": [ + { + "$ref": "#/definitions/CapabilityRootLocation" + } + ], + "description": "Where the selected root can be resolved." + } + }, + "required": [ + "id", + "location" + ], + "type": "object" + }, "ThreadSource": { "enum": [ "user", diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/CapabilityRootLocation.ts b/codex-rs/app-server-protocol/schema/typescript/v2/CapabilityRootLocation.ts new file mode 100644 index 0000000000..6266101eb8 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/CapabilityRootLocation.ts @@ -0,0 +1,8 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +/** + * Location used to resolve a selected capability root. + */ +export type CapabilityRootLocation = { "type": "environment", environmentId: string, path: string, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/SelectedCapabilityRoot.ts b/codex-rs/app-server-protocol/schema/typescript/v2/SelectedCapabilityRoot.ts new file mode 100644 index 0000000000..849d5c7aae --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/SelectedCapabilityRoot.ts @@ -0,0 +1,17 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { CapabilityRootLocation } from "./CapabilityRootLocation"; + +/** + * A user-selected root that can expose one or more runtime capabilities. + */ +export type SelectedCapabilityRoot = { +/** + * Stable identifier supplied by the capability selection platform. + */ +id: string, +/** + * Where the selected root can be resolved. + */ +location: CapabilityRootLocation, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/index.ts b/codex-rs/app-server-protocol/schema/typescript/v2/index.ts index eab3494e1f..71f4c67cd0 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/index.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/index.ts @@ -40,6 +40,7 @@ export type { ByteRange } from "./ByteRange"; export type { CancelLoginAccountParams } from "./CancelLoginAccountParams"; export type { CancelLoginAccountResponse } from "./CancelLoginAccountResponse"; export type { CancelLoginAccountStatus } from "./CancelLoginAccountStatus"; +export type { CapabilityRootLocation } from "./CapabilityRootLocation"; export type { ChatgptAuthTokensRefreshParams } from "./ChatgptAuthTokensRefreshParams"; export type { ChatgptAuthTokensRefreshReason } from "./ChatgptAuthTokensRefreshReason"; export type { ChatgptAuthTokensRefreshResponse } from "./ChatgptAuthTokensRefreshResponse"; @@ -334,6 +335,7 @@ export type { ReviewTarget } from "./ReviewTarget"; export type { SandboxMode } from "./SandboxMode"; export type { SandboxPolicy } from "./SandboxPolicy"; export type { SandboxWorkspaceWrite } from "./SandboxWorkspaceWrite"; +export type { SelectedCapabilityRoot } from "./SelectedCapabilityRoot"; export type { SendAddCreditsNudgeEmailParams } from "./SendAddCreditsNudgeEmailParams"; export type { SendAddCreditsNudgeEmailResponse } from "./SendAddCreditsNudgeEmailResponse"; export type { ServerRequestResolvedNotification } from "./ServerRequestResolvedNotification"; diff --git a/codex-rs/app-server-protocol/src/protocol/v2/thread.rs b/codex-rs/app-server-protocol/src/protocol/v2/thread.rs index 486e121cf7..0c7c362b52 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/thread.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/thread.rs @@ -11,6 +11,8 @@ use super::TurnEnvironmentParams; use super::TurnItemsView; use super::shared::v2_enum_from_core; use codex_experimental_api_macros::ExperimentalApi; +pub use codex_protocol::capabilities::CapabilityRootLocation; +pub use codex_protocol::capabilities::SelectedCapabilityRoot; use codex_protocol::config_types::CollaborationMode; use codex_protocol::config_types::Personality; use codex_protocol::config_types::ReasoningSummary; @@ -153,6 +155,10 @@ pub struct ThreadStartParams { #[experimental("thread/start.dynamicTools")] #[ts(optional = nullable)] pub dynamic_tools: Option>, + /// Capability roots selected for this thread by the hosting platform. + #[experimental("thread/start.selectedCapabilityRoots")] + #[ts(optional = nullable)] + pub selected_capability_roots: Option>, /// Test-only experimental field used to validate experimental gating and /// schema filtering behavior in a stable way. #[experimental("thread/start.mockExperimentalField")] diff --git a/codex-rs/app-server/Cargo.toml b/codex-rs/app-server/Cargo.toml index 4bdc430a1a..d6f71ab82f 100644 --- a/codex-rs/app-server/Cargo.toml +++ b/codex-rs/app-server/Cargo.toml @@ -49,6 +49,7 @@ codex-hooks = { workspace = true } codex-otel = { workspace = true } codex-plugin = { workspace = true } codex-shell-command = { workspace = true } +codex-skills-extension = { workspace = true } codex-utils-cli = { workspace = true } codex-utils-pty = { workspace = true } codex-backend-client = { workspace = true } diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index a06adf03e7..e09866e196 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -130,7 +130,7 @@ Example with notification opt-out: ## API Overview -- `thread/start` — create a new thread; emits `thread/started` (including the current `thread.status`) and auto-subscribes you to turn/item events for that thread. When the request includes a `cwd` and the resolved sandbox is `workspace-write` or full access, app-server also marks that project as trusted in the user `config.toml`. Pass `sessionStartSource: "clear"` when starting a replacement thread after clearing the current session so `SessionStart` hooks receive `source: "clear"` instead of the default `"startup"`. Experimental `runtimeWorkspaceRoots` replaces the thread-scoped runtime workspace roots used to materialize `:workspace_roots`; paths must be absolute. For permissions, prefer experimental `permissions` profile selection by id; the legacy `sandbox` shorthand is still accepted but cannot be combined with `permissions`. Experimental `environments` selects the sticky execution environments for turns on the thread; omit it to use the server default, pass `[]` to disable environments, or pass explicit environment ids with per-environment `cwd`. +- `thread/start` — create a new thread; emits `thread/started` (including the current `thread.status`) and auto-subscribes you to turn/item events for that thread. When the request includes a `cwd` and the resolved sandbox is `workspace-write` or full access, app-server also marks that project as trusted in the user `config.toml`. Pass `sessionStartSource: "clear"` when starting a replacement thread after clearing the current session so `SessionStart` hooks receive `source: "clear"` instead of the default `"startup"`. Experimental `runtimeWorkspaceRoots` replaces the thread-scoped runtime workspace roots used to materialize `:workspace_roots`; paths must be absolute. For permissions, prefer experimental `permissions` profile selection by id; the legacy `sandbox` shorthand is still accepted but cannot be combined with `permissions`. Experimental `environments` selects the sticky execution environments for turns on the thread; omit it to use the server default, pass `[]` to disable environments, or pass explicit environment ids with per-environment `cwd`. Experimental `selectedCapabilityRoots` selects environment-owned plugin or standalone-skill roots. Skills found below those roots are listed and read through the owning environment; other plugin capabilities are not activated yet. - `thread/resume` — reopen an existing thread by id so subsequent `turn/start` calls append to it. Accepts the same permission override rules as `thread/start`. - `thread/fork` — fork an existing thread into a new thread id by copying the stored history; if the source thread is currently mid-turn, the fork records the same interruption marker as `turn/interrupt` instead of inheriting an unmarked partial turn suffix. The returned `thread.forkedFromId` points at the source thread when known. Accepts `ephemeral: true` for an in-memory temporary fork, emits `thread/started` (including the current `thread.status`), and auto-subscribes you to turn/item events for the new thread. Experimental clients can pass `excludeTurns: true` when they plan to page fork history via `thread/turns/list` instead of receiving the full turn array immediately. Accepts the same permission override rules as `thread/start`. - `thread/start`, `thread/resume`, and `thread/fork` responses include the legacy `sandbox` compatibility projection. Experimental clients can read `runtimeWorkspaceRoots` for the thread-scoped runtime roots and `activePermissionProfile` for the named or implicit built-in profile identity/provenance when known. @@ -249,6 +249,18 @@ Start a fresh thread when you need a new Codex conversation. // "permissions": ":workspace" // Experimental runtime roots for :workspace_roots materialization: // "runtimeWorkspaceRoots": ["/Users/me/project", "/Users/me/openai"], + // Experimental capability roots selected by the hosting platform: + "selectedCapabilityRoots": [ + { + "id": "github@openai", + "location": { + "type": "environment", + "environmentId": "workspace", + // Opaque to app-server; interpreted in the selected environment. + "path": "/opt/cca/plugins/github" + } + } + ], // Do not send both "sandbox" and "permissions". "personality": "friendly", "serviceName": "my_app_server_client", // optional metrics tag (`service_name`) diff --git a/codex-rs/app-server/src/extensions.rs b/codex-rs/app-server/src/extensions.rs index 53f19997cd..b18331064f 100644 --- a/codex-rs/app-server/src/extensions.rs +++ b/codex-rs/app-server/src/extensions.rs @@ -32,6 +32,7 @@ pub(crate) fn thread_extensions( state_db: Option, thread_manager: Weak, goal_service: Arc, + executor_skill_provider: Arc, ) -> Arc> where S: AgentSpawner + 'static, @@ -51,6 +52,11 @@ where codex_memories_extension::install(&mut builder, codex_otel::global()); codex_web_search_extension::install(&mut builder, auth_manager.clone()); codex_image_generation_extension::install(&mut builder, auth_manager); + codex_skills_extension::install_with_providers( + &mut builder, + codex_skills_extension::SkillProviders::new() + .with_executor_provider(executor_skill_provider), + ); Arc::new(builder.build()) } diff --git a/codex-rs/app-server/src/mcp_refresh.rs b/codex-rs/app-server/src/mcp_refresh.rs index 8c72533ba5..821af862d5 100644 --- a/codex-rs/app-server/src/mcp_refresh.rs +++ b/codex-rs/app-server/src/mcp_refresh.rs @@ -181,12 +181,19 @@ mod tests { .await .expect("refresh tests require state db"); let thread_store = thread_store_from_config(&good_config, Some(state_db.clone())); + let environment_manager = Arc::new(EnvironmentManager::default_for_tests()); + let executor_skill_provider: Arc = Arc::new( + codex_skills_extension::ExecutorSkillProvider::new_with_restriction_product( + Arc::clone(&environment_manager), + SessionSource::Exec.restriction_product(), + ), + ); let thread_manager = Arc::new_cyclic(|thread_manager| { ThreadManager::new( &good_config, auth_manager.clone(), SessionSource::Exec, - Arc::new(EnvironmentManager::default_for_tests()), + Arc::clone(&environment_manager), thread_extensions( guardian_agent_spawner(thread_manager.clone()), Arc::new(NoopExtensionEventSink), @@ -194,6 +201,7 @@ mod tests { Some(state_db.clone()), thread_manager.clone(), Arc::new(codex_goal_extension::GoalService::new()), + Arc::clone(&executor_skill_provider), ), /*analytics_events_client*/ None, Arc::clone(&thread_store), diff --git a/codex-rs/app-server/src/message_processor.rs b/codex-rs/app-server/src/message_processor.rs index 84d3d88dda..78ba2c12cf 100644 --- a/codex-rs/app-server/src/message_processor.rs +++ b/codex-rs/app-server/src/message_processor.rs @@ -307,6 +307,14 @@ impl MessageProcessor { // resumed, or forked threads to a different persistence backend/root. let thread_store = codex_core::thread_store_from_config(config.as_ref(), state_db.clone()); let environment_manager_for_requests = Arc::clone(&environment_manager); + let environment_manager_for_extensions = Arc::clone(&environment_manager); + let restriction_product = session_source.restriction_product(); + let executor_skill_provider: Arc = Arc::new( + codex_skills_extension::ExecutorSkillProvider::new_with_restriction_product( + environment_manager_for_extensions, + restriction_product, + ), + ); let goal_service = Arc::new(GoalService::new()); let thread_manager = Arc::new_cyclic(|thread_manager| { ThreadManager::new( @@ -321,6 +329,7 @@ impl MessageProcessor { state_db.clone(), thread_manager.clone(), Arc::clone(&goal_service), + Arc::clone(&executor_skill_provider), ), Some(analytics_events_client.clone()), Arc::clone(&thread_store), diff --git a/codex-rs/app-server/src/request_processors/thread_processor.rs b/codex-rs/app-server/src/request_processors/thread_processor.rs index ec3bd57b07..be5cffb49a 100644 --- a/codex-rs/app-server/src/request_processors/thread_processor.rs +++ b/codex-rs/app-server/src/request_processors/thread_processor.rs @@ -1,5 +1,7 @@ use super::*; use crate::error_code::method_not_found; +use codex_app_server_protocol::SelectedCapabilityRoot; +use codex_extension_api::ExtensionDataInit; use codex_protocol::models::BUILT_IN_PERMISSION_PROFILE_DANGER_FULL_ACCESS; use codex_protocol::models::BUILT_IN_PERMISSION_PROFILE_WORKSPACE; @@ -833,6 +835,7 @@ impl ThreadRequestProcessor { base_instructions, developer_instructions, dynamic_tools, + selected_capability_roots, mock_experimental_field: _mock_experimental_field, experimental_raw_events, personality, @@ -888,6 +891,7 @@ impl ThreadRequestProcessor { config, typesafe_overrides, dynamic_tools, + selected_capability_roots.unwrap_or_default(), session_start_source, thread_source.map(Into::into), environment_selections, @@ -960,6 +964,7 @@ impl ThreadRequestProcessor { config_overrides: Option>, typesafe_overrides: ConfigOverrides, dynamic_tools: Option>, + selected_capability_roots: Vec, session_start_source: Option, thread_source: Option, environments: Option>, @@ -1061,6 +1066,10 @@ impl ThreadRequestProcessor { .collect() }; let core_dynamic_tool_count = core_dynamic_tools.len(); + let mut thread_extension_init = ExtensionDataInit::new(); + if !selected_capability_roots.is_empty() { + thread_extension_init.insert(selected_capability_roots); + } let create_thread_started_at = std::time::Instant::now(); let NewThread { thread_id, @@ -1083,6 +1092,7 @@ impl ThreadRequestProcessor { metrics_service_name: service_name, parent_trace: request_trace, environments, + thread_extension_init, }) .instrument(tracing::info_span!( "app_server.thread_start.create_thread", diff --git a/codex-rs/app-server/tests/suite/v2/executor_skills.rs b/codex-rs/app-server/tests/suite/v2/executor_skills.rs new file mode 100644 index 0000000000..ede5b20fb8 --- /dev/null +++ b/codex-rs/app-server/tests/suite/v2/executor_skills.rs @@ -0,0 +1,148 @@ +use std::time::Duration; + +use anyhow::Result; +use app_test_support::TestAppServer; +use app_test_support::to_response; +use codex_app_server_protocol::CapabilityRootLocation; +use codex_app_server_protocol::JSONRPCResponse; +use codex_app_server_protocol::RequestId; +use codex_app_server_protocol::SelectedCapabilityRoot; +use codex_app_server_protocol::ThreadStartParams; +use codex_app_server_protocol::ThreadStartResponse; +use codex_app_server_protocol::TurnStartParams; +use codex_app_server_protocol::UserInput; +use core_test_support::responses; +use tempfile::TempDir; +use tokio::time::timeout; + +const READ_TIMEOUT: Duration = Duration::from_secs(10); +const SKILL_NAME: &str = "demo-plugin:deploy"; +const SKILL_MARKER: &str = "EXECUTOR_SKILL_BODY_MARKER"; +const LOCAL_SKILL_MARKER: &str = "LOCAL_SKILL_BODY_MARKER"; + +#[tokio::test] +async fn selected_executor_root_exposes_plugin_skill() -> Result<()> { + let server = responses::start_mock_server().await; + let response_mock = responses::mount_sse_once( + &server, + responses::sse(vec![ + responses::ev_response_created("resp-selected"), + responses::ev_assistant_message("msg-selected", "Done"), + responses::ev_completed("resp-selected"), + ]), + ) + .await; + + let codex_home = TempDir::new()?; + std::fs::write( + codex_home.path().join("config.toml"), + format!( + r#" +model = "mock-model" +approval_policy = "never" +sandbox_mode = "read-only" +model_provider = "mock_provider" + +[skills] +include_instructions = true + +[model_providers.mock_provider] +name = "Mock provider for test" +base_url = "{}/v1" +wire_api = "responses" +request_max_retries = 0 +stream_max_retries = 0 +"#, + server.uri() + ), + )?; + let local_skill_dir = codex_home.path().join("skills/local-deploy"); + std::fs::create_dir_all(&local_skill_dir)?; + std::fs::write( + local_skill_dir.join("SKILL.md"), + format!( + "---\nname: {SKILL_NAME}\ndescription: Colliding local skill.\n---\n\n# Local deploy\n\n{LOCAL_SKILL_MARKER}\n" + ), + )?; + let plugin_dir = TempDir::new()?; + let manifest_dir = plugin_dir.path().join(".codex-plugin"); + let skill_dir = plugin_dir.path().join("skills/deploy"); + std::fs::create_dir_all(&manifest_dir)?; + std::fs::create_dir_all(&skill_dir)?; + std::fs::write( + manifest_dir.join("plugin.json"), + r#"{"name":"demo-plugin"}"#, + )?; + std::fs::write( + skill_dir.join("SKILL.md"), + format!( + "---\nname: deploy\ndescription: Deploy through the executor.\n---\n\n# Deploy\n\n{SKILL_MARKER}\n" + ), + )?; + + let mut app_server = TestAppServer::new(codex_home.path()).await?; + timeout(READ_TIMEOUT, app_server.initialize()).await??; + + let request_id = app_server + .send_thread_start_request(ThreadStartParams { + model: Some("mock-model".to_string()), + selected_capability_roots: Some(vec![SelectedCapabilityRoot { + id: "demo-plugin@1".to_string(), + location: CapabilityRootLocation::Environment { + environment_id: "local".to_string(), + path: plugin_dir.path().to_string_lossy().into_owned(), + }, + }]), + ..Default::default() + }) + .await?; + let response: JSONRPCResponse = timeout( + READ_TIMEOUT, + app_server.read_stream_until_response_message(RequestId::Integer(request_id)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response(response)?; + + let request_id = app_server + .send_turn_start_request(TurnStartParams { + thread_id: thread.id, + input: vec![UserInput::Text { + text: format!("Use ${SKILL_NAME}"), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + timeout( + READ_TIMEOUT, + app_server.read_stream_until_response_message(RequestId::Integer(request_id)), + ) + .await??; + timeout( + READ_TIMEOUT, + app_server.read_stream_until_notification_message("turn/completed"), + ) + .await??; + + let request = response_mock.single_request(); + assert!( + request + .message_input_texts("developer") + .iter() + .any(|text| text.contains(SKILL_NAME)) + ); + let skill_fragments = request + .message_input_texts("user") + .into_iter() + .filter(|text| text.starts_with("")) + .collect::>(); + assert_eq!(1, skill_fragments.len()); + let skill_fragment = skill_fragments + .first() + .expect("executor skill instructions should be model-visible"); + assert!(skill_fragment.contains(&format!("{SKILL_NAME}"))); + assert!(skill_fragment.contains(SKILL_MARKER)); + assert!(!skill_fragment.contains(LOCAL_SKILL_MARKER)); + + Ok(()) +} diff --git a/codex-rs/app-server/tests/suite/v2/mod.rs b/codex-rs/app-server/tests/suite/v2/mod.rs index 9aef40aade..77d4b37589 100644 --- a/codex-rs/app-server/tests/suite/v2/mod.rs +++ b/codex-rs/app-server/tests/suite/v2/mod.rs @@ -12,6 +12,7 @@ mod connection_handling_websocket; #[cfg(unix)] mod connection_handling_websocket_unix; mod dynamic_tools; +mod executor_skills; mod experimental_api; mod experimental_feature_list; mod external_agent_config; diff --git a/codex-rs/app-server/tests/suite/v2/skills_list.rs b/codex-rs/app-server/tests/suite/v2/skills_list.rs index cba2f4ee3f..147d40c47a 100644 --- a/codex-rs/app-server/tests/suite/v2/skills_list.rs +++ b/codex-rs/app-server/tests/suite/v2/skills_list.rs @@ -793,6 +793,7 @@ async fn skills_changed_notification_is_emitted_after_skill_change() -> Result<( thread_source: None, dynamic_tools: None, environments: None, + selected_capability_roots: None, mock_experimental_field: None, experimental_raw_events: false, }) diff --git a/codex-rs/core-skills/src/loader.rs b/codex-rs/core-skills/src/loader.rs index 7dac256010..e2c3896cba 100644 --- a/codex-rs/core-skills/src/loader.rs +++ b/codex-rs/core-skills/src/loader.rs @@ -169,8 +169,8 @@ where let mut file_systems_by_skill_path: HashMap> = HashMap::new(); for root in roots { - let root_path = canonicalize_for_skill_identity(&root.path); let fs = root.file_system; + let root_path = canonicalize_for_skill_identity(fs.as_ref(), &root.path).await; let skills_before_root = outcome.skills.len(); discover_skills_under_root( fs.as_ref(), @@ -471,8 +471,13 @@ fn dedupe_skill_roots_by_path(roots: &mut Vec) { roots.retain(|root| seen.insert(root.path.clone())); } -fn canonicalize_for_skill_identity(path: &AbsolutePathBuf) -> AbsolutePathBuf { - path.canonicalize().unwrap_or_else(|_| path.clone()) +async fn canonicalize_for_skill_identity( + fs: &dyn ExecutorFileSystem, + path: &AbsolutePathBuf, +) -> AbsolutePathBuf { + fs.canonicalize(path, /*sandbox*/ None) + .await + .unwrap_or_else(|_| path.clone()) } async fn discover_skills_under_root( @@ -483,8 +488,11 @@ async fn discover_skills_under_root( plugin_root: Option<&AbsolutePathBuf>, outcome: &mut SkillLoadOutcome, ) { - let root = canonicalize_for_skill_identity(root); - let plugin_root = plugin_root.map(canonicalize_for_skill_identity); + let root = root.clone(); + let plugin_root = match plugin_root { + Some(plugin_root) => Some(canonicalize_for_skill_identity(fs, plugin_root).await), + None => None, + }; match fs.get_metadata(&root, /*sandbox*/ None).await { Ok(metadata) if metadata.is_directory => {} @@ -557,7 +565,7 @@ async fn discover_skills_under_root( } match fs.read_directory(&path, /*sandbox*/ None).await { Ok(_) => { - let resolved_dir = canonicalize_for_skill_identity(&path); + let resolved_dir = canonicalize_for_skill_identity(fs, &path).await; enqueue_dir( &mut queue, &mut visited_dirs, @@ -582,7 +590,7 @@ async fn discover_skills_under_root( } if metadata.is_directory { - let resolved_dir = canonicalize_for_skill_identity(&path); + let resolved_dir = canonicalize_for_skill_identity(fs, &path).await; enqueue_dir( &mut queue, &mut visited_dirs, @@ -672,7 +680,7 @@ async fn parse_skill_file( )?; } - let resolved_path = canonicalize_for_skill_identity(path); + let resolved_path = canonicalize_for_skill_identity(fs, path).await; Ok(SkillMetadata { name, diff --git a/codex-rs/core/src/codex_delegate.rs b/codex-rs/core/src/codex_delegate.rs index 5f816bd075..5b68ecf917 100644 --- a/codex-rs/core/src/codex_delegate.rs +++ b/codex-rs/core/src/codex_delegate.rs @@ -103,6 +103,7 @@ pub(crate) async fn run_codex_thread_interactive( parent_rollout_thread_trace: codex_rollout_trace::ThreadTraceContext::disabled(), parent_trace: None, environment_selections: parent_ctx.environments.clone(), + thread_extension_init: codex_extension_api::ExtensionDataInit::default(), analytics_events_client: Some(parent_session.services.analytics_events_client.clone()), thread_store: Arc::clone(&parent_session.services.thread_store), attestation_provider: parent_session.services.attestation_provider.clone(), diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs index bb66e8684e..23dbe1f45e 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -51,6 +51,7 @@ use codex_config::types::OAuthCredentialsStoreMode; use codex_exec_server::Environment; use codex_exec_server::EnvironmentManager; use codex_exec_server::FileSystemSandboxContext; +use codex_extension_api::ExtensionDataInit; use codex_extension_api::PromptSlot; use codex_features::FEATURES; use codex_features::Feature; @@ -417,6 +418,7 @@ pub(crate) struct CodexSpawnArgs { pub(crate) user_shell_override: Option, pub(crate) parent_trace: Option, pub(crate) environment_selections: ResolvedTurnEnvironments, + pub(crate) thread_extension_init: ExtensionDataInit, pub(crate) analytics_events_client: Option, pub(crate) thread_store: Arc, pub(crate) attestation_provider: Option>, @@ -497,6 +499,7 @@ impl Codex { parent_rollout_thread_trace, parent_trace: _, environment_selections, + thread_extension_init, analytics_events_client, thread_store, attestation_provider, @@ -641,6 +644,7 @@ impl Codex { plugins_manager, mcp_manager.clone(), extensions, + thread_extension_init, agent_control, environment_manager, analytics_events_client, diff --git a/codex-rs/core/src/session/session.rs b/codex-rs/core/src/session/session.rs index c6d0d9767e..e51a83113b 100644 --- a/codex-rs/core/src/session/session.rs +++ b/codex-rs/core/src/session/session.rs @@ -4,6 +4,7 @@ use crate::agents_md::LoadedAgentsMd; use crate::config::ConstraintError; use crate::skills::SkillError; use crate::state::ActiveTurn; +use codex_extension_api::ExtensionDataInit; use codex_protocol::SessionId; use codex_protocol::config_types::SERVICE_TIER_DEFAULT_REQUEST_VALUE; use codex_protocol::config_types::ServiceTier; @@ -487,6 +488,7 @@ impl Session { plugins_manager: Arc, mcp_manager: Arc, extensions: Arc>, + thread_extension_init: ExtensionDataInit, agent_control: AgentControl, environment_manager: Arc, analytics_events_client: Option, @@ -961,8 +963,10 @@ impl Session { ); let session_extension_data = codex_extension_api::ExtensionData::new(session_id.to_string()); - let thread_extension_data = - codex_extension_api::ExtensionData::new(thread_id.to_string()); + let thread_extension_data = codex_extension_api::ExtensionData::new_with_init( + thread_id.to_string(), + thread_extension_init, + ); for contributor in extensions.thread_lifecycle_contributors() { contributor.on_thread_start(codex_extension_api::ThreadStartInput { config: config.as_ref(), diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs index ef3f6dbfba..f6166260a0 100644 --- a/codex-rs/core/src/session/tests.rs +++ b/codex-rs/core/src/session/tests.rs @@ -4690,6 +4690,7 @@ async fn session_new_fails_when_zsh_fork_enabled_without_packaged_zsh() { plugins_manager, mcp_manager, Arc::new(codex_extension_api::ExtensionRegistryBuilder::new().build()), + codex_extension_api::ExtensionDataInit::default(), AgentControl::default(), Arc::new(codex_exec_server::EnvironmentManager::default_for_tests()), /*analytics_events_client*/ None, @@ -5030,6 +5031,7 @@ async fn make_session_with_config_and_rx( plugins_manager, mcp_manager, Arc::new(codex_extension_api::ExtensionRegistryBuilder::new().build()), + codex_extension_api::ExtensionDataInit::default(), AgentControl::default(), Arc::new(codex_exec_server::EnvironmentManager::default_for_tests()), /*analytics_events_client*/ None, @@ -5131,6 +5133,7 @@ async fn make_session_with_history_source_and_agent_control_and_rx( plugins_manager, mcp_manager, Arc::new(codex_extension_api::ExtensionRegistryBuilder::new().build()), + codex_extension_api::ExtensionDataInit::default(), agent_control, Arc::new(codex_exec_server::EnvironmentManager::default_for_tests()), /*analytics_events_client*/ None, diff --git a/codex-rs/core/src/session/tests/guardian_tests.rs b/codex-rs/core/src/session/tests/guardian_tests.rs index d8b46c9999..c73add2920 100644 --- a/codex-rs/core/src/session/tests/guardian_tests.rs +++ b/codex-rs/core/src/session/tests/guardian_tests.rs @@ -731,6 +731,7 @@ async fn guardian_subagent_does_not_inherit_parent_exec_policy_rules() { environment_selections: ResolvedTurnEnvironments { turn_environments: Vec::new(), }, + thread_extension_init: codex_extension_api::ExtensionDataInit::default(), analytics_events_client: None, thread_store, attestation_provider: None, diff --git a/codex-rs/core/src/thread_manager.rs b/codex-rs/core/src/thread_manager.rs index 1cde8c9320..a0c1d63d71 100644 --- a/codex-rs/core/src/thread_manager.rs +++ b/codex-rs/core/src/thread_manager.rs @@ -21,6 +21,7 @@ use codex_app_server_protocol::ThreadHistoryBuilder; use codex_app_server_protocol::TurnStatus; use codex_core_plugins::PluginsManager; use codex_exec_server::EnvironmentManager; +use codex_extension_api::ExtensionDataInit; use codex_extension_api::ExtensionRegistry; use codex_extension_api::empty_extension_registry; use codex_features::Feature; @@ -182,6 +183,7 @@ pub struct StartThreadOptions { pub metrics_service_name: Option, pub parent_trace: Option, pub environments: Vec, + pub thread_extension_init: ExtensionDataInit, } pub(crate) struct ResumeThreadWithHistoryOptions { @@ -576,6 +578,7 @@ impl ThreadManager { metrics_service_name: None, parent_trace: None, environments, + thread_extension_init: ExtensionDataInit::default(), })) .await } @@ -614,6 +617,7 @@ impl ThreadManager { /*inherited_exec_policy*/ None, options.parent_trace, options.environments, + options.thread_extension_init, /*user_shell_override*/ None, )) .await @@ -702,6 +706,7 @@ impl ThreadManager { /*inherited_exec_policy*/ None, parent_trace, environments, + /*thread_extension_init*/ ExtensionDataInit::default(), /*user_shell_override*/ None, )) .await @@ -728,6 +733,7 @@ impl ThreadManager { /*metrics_service_name*/ None, /*parent_trace*/ None, environments, + /*thread_extension_init*/ ExtensionDataInit::default(), /*user_shell_override*/ Some(user_shell_override), )) .await @@ -763,6 +769,7 @@ impl ThreadManager { /*inherited_exec_policy*/ None, /*parent_trace*/ None, environments, + /*thread_extension_init*/ ExtensionDataInit::default(), /*user_shell_override*/ Some(user_shell_override), )) .await @@ -931,6 +938,7 @@ impl ThreadManager { /*metrics_service_name*/ None, parent_trace, environments, + /*thread_extension_init*/ ExtensionDataInit::default(), /*user_shell_override*/ None, )) .await @@ -1136,6 +1144,7 @@ impl ThreadManagerState { inherited_exec_policy, /*parent_trace*/ None, environments, + /*thread_extension_init*/ ExtensionDataInit::default(), /*user_shell_override*/ None, )) .await @@ -1172,6 +1181,7 @@ impl ThreadManagerState { inherited_exec_policy, /*parent_trace*/ None, environments, + /*thread_extension_init*/ ExtensionDataInit::default(), /*user_shell_override*/ None, )) .await @@ -1209,6 +1219,7 @@ impl ThreadManagerState { inherited_exec_policy, /*parent_trace*/ None, environments, + /*thread_extension_init*/ ExtensionDataInit::default(), /*user_shell_override*/ None, )) .await @@ -1229,6 +1240,7 @@ impl ThreadManagerState { metrics_service_name: Option, parent_trace: Option, environments: Vec, + thread_extension_init: ExtensionDataInit, user_shell_override: Option, ) -> CodexResult { Box::pin(self.spawn_thread_with_source( @@ -1246,6 +1258,7 @@ impl ThreadManagerState { /*inherited_exec_policy*/ None, parent_trace, environments, + thread_extension_init, user_shell_override, )) .await @@ -1268,6 +1281,7 @@ impl ThreadManagerState { inherited_exec_policy: Option>, parent_trace: Option, environments: Vec, + thread_extension_init: ExtensionDataInit, user_shell_override: Option, ) -> CodexResult { let is_resumed_thread = matches!(&initial_history, InitialHistory::Resumed(_)); @@ -1332,6 +1346,7 @@ impl ThreadManagerState { user_shell_override, parent_trace, environment_selections, + thread_extension_init, analytics_events_client: self.analytics_events_client.clone(), thread_store: Arc::clone(&self.thread_store), attestation_provider: self.attestation_provider.clone(), diff --git a/codex-rs/core/src/thread_manager_tests.rs b/codex-rs/core/src/thread_manager_tests.rs index 116473258e..bb0c0af314 100644 --- a/codex-rs/core/src/thread_manager_tests.rs +++ b/codex-rs/core/src/thread_manager_tests.rs @@ -331,6 +331,7 @@ async fn start_thread_rejects_explicit_local_environment_when_default_provider_i environment_id: "local".to_string(), cwd: config.cwd.clone(), }], + thread_extension_init: Default::default(), }) .await; let err = match result { @@ -368,6 +369,7 @@ async fn start_thread_keeps_internal_threads_hidden_from_normal_lookups() { metrics_service_name: None, parent_trace: None, environments: Vec::new(), + thread_extension_init: Default::default(), }) .await .expect("internal thread should start"); @@ -384,6 +386,81 @@ async fn start_thread_keeps_internal_threads_hidden_from_normal_lookups() { assert!(manager.list_thread_ids().await.is_empty()); } +#[tokio::test] +async fn start_thread_seeds_extension_data_before_lifecycle_contributors_run() { + struct InitialMarker(&'static str); + + struct InitialDataRecorder { + observed: Arc>>, + } + + #[async_trait::async_trait] + impl codex_extension_api::ThreadLifecycleContributor for InitialDataRecorder { + async fn on_thread_start(&self, input: codex_extension_api::ThreadStartInput<'_, Config>) { + let marker = input + .thread_store + .get::() + .expect("initial extension data should be available"); + *self + .observed + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(( + input.thread_store.level_id().to_string(), + marker.0.to_string(), + )); + } + } + + let temp_dir = tempdir().expect("tempdir"); + let mut config = test_config().await; + config.codex_home = temp_dir.path().join("codex-home").abs(); + config.cwd = config.codex_home.abs(); + std::fs::create_dir_all(&config.codex_home).expect("create codex home"); + + let observed = Arc::new(std::sync::Mutex::new(None)); + let mut extensions = codex_extension_api::ExtensionRegistryBuilder::new(); + extensions.thread_lifecycle_contributor(Arc::new(InitialDataRecorder { + observed: Arc::clone(&observed), + })); + let manager = ThreadManager::new( + &config, + AuthManager::from_auth_for_testing(CodexAuth::create_dummy_chatgpt_auth_for_testing()), + SessionSource::Exec, + Arc::new(codex_exec_server::EnvironmentManager::default_for_tests()), + Arc::new(extensions.build()), + /*analytics_events_client*/ None, + thread_store_from_config(&config, /*state_db*/ None), + /*state_db*/ None, + TEST_INSTALLATION_ID.to_string(), + /*attestation_provider*/ None, + ); + let mut thread_extension_init = codex_extension_api::ExtensionDataInit::new(); + thread_extension_init.insert(InitialMarker("seeded")); + + let thread = manager + .start_thread_with_options(StartThreadOptions { + config, + initial_history: InitialHistory::New, + session_source: None, + thread_source: None, + dynamic_tools: Vec::new(), + metrics_service_name: None, + parent_trace: None, + environments: Vec::new(), + thread_extension_init, + }) + .await + .expect("start thread"); + + assert_eq!( + observed + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(), + Some((thread.thread_id.to_string(), "seeded".to_string())) + ); +} + #[tokio::test] async fn resume_and_fork_do_not_restore_thread_environments_from_rollout() { let temp_dir = tempdir().expect("tempdir"); @@ -423,6 +500,7 @@ async fn resume_and_fork_do_not_restore_thread_environments_from_rollout() { metrics_service_name: None, parent_trace: None, environments: environments.clone(), + thread_extension_init: Default::default(), }) .await .expect("start source thread"); @@ -693,6 +771,7 @@ async fn resume_stopped_thread_from_rollout_preserves_thread_source() { metrics_service_name: None, parent_trace: None, environments: Vec::new(), + thread_extension_init: Default::default(), }) .await .expect("start source thread"); diff --git a/codex-rs/core/tests/suite/subagent_notifications.rs b/codex-rs/core/tests/suite/subagent_notifications.rs index 4d6cd17bb8..c3468e4653 100644 --- a/codex-rs/core/tests/suite/subagent_notifications.rs +++ b/codex-rs/core/tests/suite/subagent_notifications.rs @@ -757,6 +757,7 @@ async fn subagent_stop_replaces_stop_and_skips_internal_subagents() -> Result<() metrics_service_name: None, parent_trace: None, environments: Vec::new(), + thread_extension_init: Default::default(), }) .await?; diff --git a/codex-rs/ext/extension-api/src/lib.rs b/codex-rs/ext/extension-api/src/lib.rs index 7fa60c0fe7..7d8d95d139 100644 --- a/codex-rs/ext/extension-api/src/lib.rs +++ b/codex-rs/ext/extension-api/src/lib.rs @@ -59,3 +59,4 @@ pub use registry::ExtensionRegistry; pub use registry::ExtensionRegistryBuilder; pub use registry::empty_extension_registry; pub use state::ExtensionData; +pub use state::ExtensionDataInit; diff --git a/codex-rs/ext/extension-api/src/state.rs b/codex-rs/ext/extension-api/src/state.rs index aab37f5059..97152df5a3 100644 --- a/codex-rs/ext/extension-api/src/state.rs +++ b/codex-rs/ext/extension-api/src/state.rs @@ -7,6 +7,32 @@ use std::sync::PoisonError; type ErasedData = Arc; +/// Typed values supplied before an [`ExtensionData`] scope is created. +/// +/// Hosts consume this value once to seed a scope before lifecycle contributors +/// run. It does not install extensions or provide persistence. +#[derive(Debug, Default)] +pub struct ExtensionDataInit { + entries: HashMap, +} + +impl ExtensionDataInit { + /// Creates an empty extension data initializer. + pub fn new() -> Self { + Self::default() + } + + /// Stores `value` as the initial attachment of type `T`. + pub fn insert(&mut self, value: T) -> Option> + where + T: Any + Send + Sync, + { + self.entries + .insert(TypeId::of::(), Arc::new(value)) + .map(downcast_data) + } +} + /// Typed extension-owned data attached to one host object. #[derive(Debug)] pub struct ExtensionData { @@ -17,9 +43,14 @@ pub struct ExtensionData { impl ExtensionData { /// Creates an empty attachment map for one host-owned scope. pub fn new(level_id: impl Into) -> Self { + Self::new_with_init(level_id, ExtensionDataInit::default()) + } + + /// Creates an attachment map seeded with host-supplied initial data. + pub fn new_with_init(level_id: impl Into, init: ExtensionDataInit) -> Self { Self { level_id: level_id.into(), - entries: Mutex::new(HashMap::new()), + entries: Mutex::new(init.entries), } } diff --git a/codex-rs/ext/skills/Cargo.toml b/codex-rs/ext/skills/Cargo.toml index 29767d58fe..9ff42cfafc 100644 --- a/codex-rs/ext/skills/Cargo.toml +++ b/codex-rs/ext/skills/Cargo.toml @@ -17,8 +17,11 @@ workspace = true async-trait = { workspace = true } codex-core = { workspace = true } codex-core-skills = { workspace = true } +codex-exec-server = { workspace = true } codex-extension-api = { workspace = true } codex-protocol = { workspace = true } +codex-utils-absolute-path = { workspace = true } +codex-utils-string = { workspace = true } [dev-dependencies] pretty_assertions = { workspace = true } diff --git a/codex-rs/ext/skills/src/catalog.rs b/codex-rs/ext/skills/src/catalog.rs index a09a81ff79..c966cc09e9 100644 --- a/codex-rs/ext/skills/src/catalog.rs +++ b/codex-rs/ext/skills/src/catalog.rs @@ -1,4 +1,5 @@ use codex_core_skills::model::SkillDependencies; +use codex_exec_server::EnvironmentPathRef; /// Source authority that owns a skill package and must be used to read it. #[derive(Clone, Debug, PartialEq, Eq, Hash)] @@ -56,9 +57,37 @@ impl SkillAuthority { #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct SkillPackageId(pub String); -/// Opaque resource id inside a skill package. +/// Opaque resource id inside a skill package, optionally bound to the +/// environment path that owns its contents. #[derive(Clone, Debug, PartialEq, Eq, Hash)] -pub struct SkillResourceId(pub String); +pub struct SkillResourceId { + id: String, + environment_path: Option, +} + +impl SkillResourceId { + pub fn new(id: impl Into) -> Self { + Self { + id: id.into(), + environment_path: None, + } + } + + pub fn environment(id: impl Into, path: EnvironmentPathRef) -> Self { + Self { + id: id.into(), + environment_path: Some(path), + } + } + + pub fn as_str(&self) -> &str { + &self.id + } + + pub(crate) fn environment_path(&self) -> Option<&EnvironmentPathRef> { + self.environment_path.as_ref() + } +} /// Metadata shown in the always-visible skills catalog. #[derive(Clone, Debug, PartialEq, Eq)] @@ -125,7 +154,7 @@ impl SkillCatalogEntry { pub(crate) fn rendered_path(&self) -> &str { self.display_path .as_deref() - .unwrap_or(self.main_prompt.0.as_str()) + .unwrap_or_else(|| self.main_prompt.as_str()) } } diff --git a/codex-rs/ext/skills/src/extension.rs b/codex-rs/ext/skills/src/extension.rs index 3961d7d85d..45d7c95cb7 100644 --- a/codex-rs/ext/skills/src/extension.rs +++ b/codex-rs/ext/skills/src/extension.rs @@ -6,27 +6,32 @@ use codex_core_skills::SkillInstructions; use codex_core_skills::injection::InjectedHostSkillPrompts; use codex_core_skills::injection::SkillInjection; use codex_extension_api::ConfigContributor; +use codex_extension_api::ContextContributor; use codex_extension_api::ContextualUserFragment; use codex_extension_api::ExtensionData; use codex_extension_api::ExtensionEventSink; use codex_extension_api::ExtensionRegistryBuilder; +use codex_extension_api::PromptFragment; use codex_extension_api::ThreadLifecycleContributor; use codex_extension_api::ThreadStartInput; use codex_extension_api::TurnInputContext; use codex_extension_api::TurnInputContributor; +use codex_protocol::capabilities::SelectedCapabilityRoot; use codex_protocol::protocol::Event; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::WarningEvent; -use crate::catalog::SkillAuthority; use crate::catalog::SkillCatalogEntry; use crate::catalog::SkillReadResult; use crate::catalog::SkillSourceKind; use crate::provider::HostSkillProvider; use crate::provider::SkillListQuery; use crate::provider::SkillReadRequest; +use crate::render::MAX_SKILL_NAME_BYTES; +use crate::render::MAX_SKILL_PATH_BYTES; use crate::render::available_skills_fragment; use crate::render::truncate_main_prompt_contents; +use crate::render::truncate_utf8_to_bytes; use crate::selection::collect_explicit_skill_mentions; use crate::sources::SkillProviders; use crate::state::SkillsExtensionConfig; @@ -42,11 +47,15 @@ struct SkillsExtension { #[async_trait::async_trait] impl ThreadLifecycleContributor for SkillsExtension { async fn on_thread_start(&self, input: ThreadStartInput<'_, Config>) { - input + let selected_roots = input .thread_store - .insert(SkillsThreadState::new(SkillsExtensionConfig::from_config( - input.config, - ))); + .get::>() + .map(|selected_roots| selected_roots.as_ref().clone()) + .unwrap_or_default(); + input.thread_store.insert(SkillsThreadState::new( + SkillsExtensionConfig::from_config(input.config), + selected_roots, + )); } } @@ -62,11 +71,47 @@ impl ConfigContributor for SkillsExtension { if let Some(state) = thread_store.get::() { state.set_config(next_config); } else { - thread_store.insert(SkillsThreadState::new(next_config)); + thread_store.insert(SkillsThreadState::new(next_config, Vec::new())); } } } +impl ContextContributor for SkillsExtension { + fn contribute<'a>( + &'a self, + _session_store: &'a ExtensionData, + thread_store: &'a ExtensionData, + ) -> std::pin::Pin> + Send + 'a>> { + Box::pin(async move { + let Some(thread_state) = thread_store.get::() else { + return Vec::new(); + }; + let config = thread_state.config(); + if !config.include_instructions || thread_state.selected_roots().is_empty() { + return Vec::new(); + } + let catalog = self + .providers + .list_for_turn(SkillListQuery { + turn_id: thread_store.level_id().to_string(), + executor_roots: thread_state.selected_roots().to_vec(), + host: None, + include_host_skills: false, + include_bundled_skills: config.bundled_skills_enabled, + include_remote_skills: false, + }) + .await; + for warning in &catalog.warnings { + self.emit_warning(thread_store.level_id(), warning.clone()); + } + available_skills_fragment(&catalog) + .map(|fragment| PromptFragment::developer_capability(fragment.render())) + .into_iter() + .collect() + }) + } +} + #[async_trait::async_trait] impl TurnInputContributor for SkillsExtension { async fn contribute( @@ -84,16 +129,7 @@ impl TurnInputContributor for SkillsExtension { let host_loaded_skills = turn_store.get::(); let query = SkillListQuery { turn_id: input.turn_id.clone(), - executor_authorities: input - .environments - .iter() - .map(|environment| { - SkillAuthority::new( - SkillSourceKind::Executor, - environment.environment_id.clone(), - ) - }) - .collect(), + executor_roots: thread_state.selected_roots().to_vec(), host: host_loaded_skills.clone(), include_host_skills: true, include_bundled_skills: config.bundled_skills_enabled, @@ -106,10 +142,14 @@ impl TurnInputContributor for SkillsExtension { let selected_entries = collect_explicit_skill_mentions(&input.user_input, &catalog); let mut fragments: Vec> = Vec::new(); - if config.include_instructions - && let Some(fragment) = available_skills_fragment(&catalog) - { - fragments.push(Box::new(fragment)); + if config.include_instructions { + let mut turn_catalog = catalog.clone(); + turn_catalog + .entries + .retain(|entry| entry.authority.kind != SkillSourceKind::Executor); + if let Some(fragment) = available_skills_fragment(&turn_catalog) { + fragments.push(Box::new(fragment)); + } } let mut warnings = catalog.warnings.clone(); @@ -132,14 +172,14 @@ impl TurnInputContributor for SkillsExtension { warnings.push(warning); } let injection = SkillInjection { - name: entry.name.clone(), - path: entry.rendered_path().to_string(), + name: truncate_utf8_to_bytes(&entry.name, MAX_SKILL_NAME_BYTES).0, + path: truncate_utf8_to_bytes(entry.rendered_path(), MAX_SKILL_PATH_BYTES).0, contents, }; fragments.push(Box::new(SkillInstructions::from(&injection))); main_prompts_injected = true; if entry.authority.kind == SkillSourceKind::Host { - injected_host_skill_prompts.insert_path(entry.main_prompt.0.clone()); + injected_host_skill_prompts.insert_path(entry.main_prompt.as_str()); } } Err(message) => { @@ -150,6 +190,23 @@ impl TurnInputContributor for SkillsExtension { } } + if let Some(host_loaded_skills) = &host_loaded_skills { + for entry in selected_entries + .iter() + .filter(|entry| entry.authority.kind != SkillSourceKind::Host) + { + for host_skill in host_loaded_skills + .outcome() + .skills + .iter() + .filter(|host_skill| host_skill.name == entry.name) + { + injected_host_skill_prompts + .insert_path(host_skill.path_to_skills_md.to_string_lossy()); + } + } + } + turn_store.insert(SkillsTurnState { catalog, selected_entries, @@ -206,5 +263,6 @@ pub fn install_with_providers( }); registry.thread_lifecycle_contributor(extension.clone()); registry.config_contributor(extension.clone()); + registry.prompt_contributor(extension.clone()); registry.turn_input_contributor(extension); } diff --git a/codex-rs/ext/skills/src/lib.rs b/codex-rs/ext/skills/src/lib.rs index 3a28c5fbdf..5ab03ddd9e 100644 --- a/codex-rs/ext/skills/src/lib.rs +++ b/codex-rs/ext/skills/src/lib.rs @@ -8,6 +8,8 @@ mod state; pub use extension::install; pub use extension::install_with_providers; +pub use provider::ExecutorSkillProvider; pub use provider::HostSkillProvider; +pub use provider::SkillProvider; pub use sources::SkillProviderSource; pub use sources::SkillProviders; diff --git a/codex-rs/ext/skills/src/provider.rs b/codex-rs/ext/skills/src/provider.rs index a968774afd..8d13a2f981 100644 --- a/codex-rs/ext/skills/src/provider.rs +++ b/codex-rs/ext/skills/src/provider.rs @@ -2,9 +2,11 @@ use std::future::Future; use std::pin::Pin; use std::sync::Arc; +mod executor; mod host; use codex_core_skills::HostLoadedSkills; +use codex_protocol::capabilities::SelectedCapabilityRoot; use crate::catalog::SkillAuthority; use crate::catalog::SkillCatalog; @@ -14,12 +16,13 @@ use crate::catalog::SkillReadResult; use crate::catalog::SkillResourceId; use crate::catalog::SkillSearchResult; +pub use executor::ExecutorSkillProvider; pub use host::HostSkillProvider; #[derive(Clone, Debug)] pub struct SkillListQuery { pub turn_id: String, - pub executor_authorities: Vec, + pub executor_roots: Vec, pub host: Option>, pub include_host_skills: bool, pub include_bundled_skills: bool, diff --git a/codex-rs/ext/skills/src/provider/executor.rs b/codex-rs/ext/skills/src/provider/executor.rs new file mode 100644 index 0000000000..e23e5a95bd --- /dev/null +++ b/codex-rs/ext/skills/src/provider/executor.rs @@ -0,0 +1,197 @@ +use std::path::PathBuf; +use std::sync::Arc; + +use codex_core_skills::SkillMetadata; +use codex_core_skills::filter_skill_load_outcome_for_product; +use codex_core_skills::loader::SkillRoot; +use codex_core_skills::loader::load_skills_from_roots; +use codex_exec_server::EnvironmentManager; +use codex_exec_server::EnvironmentPathRef; +use codex_protocol::capabilities::CapabilityRootLocation; +use codex_protocol::protocol::Product; +use codex_protocol::protocol::SkillScope; +use codex_utils_absolute_path::AbsolutePathBuf; + +use crate::catalog::SkillAuthority; +use crate::catalog::SkillCatalog; +use crate::catalog::SkillCatalogEntry; +use crate::catalog::SkillPackageId; +use crate::catalog::SkillProviderError; +use crate::catalog::SkillReadResult; +use crate::catalog::SkillResourceId; +use crate::catalog::SkillSearchResult; +use crate::catalog::SkillSourceKind; +use crate::provider::SkillListQuery; +use crate::provider::SkillProvider; +use crate::provider::SkillProviderFuture; +use crate::provider::SkillReadRequest; +use crate::provider::SkillSearchRequest; + +/// Discovers and reads skills through the filesystem owned by an execution environment. +#[derive(Clone, Debug)] +pub struct ExecutorSkillProvider { + environment_manager: Arc, + restriction_product: Option, +} + +impl ExecutorSkillProvider { + pub fn new_with_restriction_product( + environment_manager: Arc, + restriction_product: Option, + ) -> Self { + Self { + environment_manager, + restriction_product, + } + } +} + +impl SkillProvider for ExecutorSkillProvider { + fn list(&self, query: SkillListQuery) -> SkillProviderFuture<'_, SkillCatalog> { + Box::pin(async move { + let mut catalog = SkillCatalog::default(); + for selected_root in query.executor_roots { + let selected_root_id = selected_root.id; + let CapabilityRootLocation::Environment { + environment_id, + path, + } = selected_root.location; + let authority = + SkillAuthority::new(SkillSourceKind::Executor, selected_root_id.clone()); + let Some(environment) = self.environment_manager.get_environment(&environment_id) + else { + catalog.warnings.push(format!( + "Selected capability root `{selected_root_id}` references unavailable environment `{environment_id}`." + )); + continue; + }; + let root_path = match executor_absolute_path(&path) { + Ok(root_path) => root_path, + Err(err) => { + catalog.warnings.push(format!( + "Selected capability root `{selected_root_id}` has invalid path `{path}`: {err}" + )); + continue; + } + }; + let file_system = environment.get_filesystem(); + let outcome = filter_skill_load_outcome_for_product( + load_skills_from_roots([SkillRoot { + path: root_path.clone(), + scope: SkillScope::User, + file_system: Arc::clone(&file_system), + plugin_id: None, + plugin_root: None, + }]) + .await, + self.restriction_product, + ); + catalog.warnings.extend(outcome.errors.iter().map(|err| { + format!( + "Failed to load executor skill at {}: {}", + err.path.display(), + err.message + ) + })); + for (skill, enabled) in outcome.skills_with_enabled() { + catalog.push_entry(catalog_entry_from_skill( + skill, + enabled, + authority.clone(), + &selected_root_id, + Arc::clone(&file_system), + )); + } + } + + Ok(catalog) + }) + } + + fn read(&self, request: SkillReadRequest) -> SkillProviderFuture<'_, SkillReadResult> { + Box::pin(async move { + if request.authority.kind != SkillSourceKind::Executor { + return Err(SkillProviderError::new(format!( + "executor skill provider cannot read {} resources", + request.authority.kind + ))); + } + if request.package.0 != request.resource.as_str() { + return Err(SkillProviderError::new( + "executor skill resource does not match its package", + )); + } + let Some(resource_path) = request.resource.environment_path() else { + return Err(SkillProviderError::new( + "executor skill resource is not bound to an environment", + )); + }; + let contents = resource_path + .read_to_string(/*sandbox*/ None) + .await + .map_err(|err| { + SkillProviderError::new(format!( + "failed to read executor skill resource {}: {err}", + request.resource.as_str() + )) + })?; + + Ok(SkillReadResult { + resource: request.resource, + contents, + }) + }) + } + + fn search(&self, _request: SkillSearchRequest) -> SkillProviderFuture<'_, SkillSearchResult> { + Box::pin(async { Ok(SkillSearchResult::default()) }) + } +} + +fn catalog_entry_from_skill( + skill: &SkillMetadata, + enabled: bool, + authority: SkillAuthority, + selected_root_id: &str, + file_system: Arc, +) -> SkillCatalogEntry { + let skill_path = skill.path_to_skills_md.to_string_lossy().into_owned(); + let normalized_path = skill_path.replace('\\', "/"); + let display_path = format!( + "skill://{selected_root_id}/{}", + normalized_path.trim_start_matches('/') + ); + let mut entry = SkillCatalogEntry::new( + SkillPackageId(display_path.clone()), + authority, + skill.name.clone(), + skill.description.clone(), + SkillResourceId::environment( + display_path.clone(), + EnvironmentPathRef::new(file_system, skill.path_to_skills_md.clone()), + ), + ) + .with_short_description(skill.short_description.clone()) + .with_display_path(display_path) + .with_dependencies(skill.dependencies.clone()); + + if !enabled { + entry = entry.disabled(); + } + if !skill.allows_implicit_invocation() { + entry = entry.hidden_from_prompt(); + } + + entry +} + +fn executor_absolute_path(path: &str) -> std::io::Result { + let path = PathBuf::from(path); + if !path.is_absolute() { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "executor path must be absolute", + )); + } + AbsolutePathBuf::from_absolute_path_checked(path) +} diff --git a/codex-rs/ext/skills/src/provider/host.rs b/codex-rs/ext/skills/src/provider/host.rs index 27293aa1d5..63d0d4588c 100644 --- a/codex-rs/ext/skills/src/provider/host.rs +++ b/codex-rs/ext/skills/src/provider/host.rs @@ -55,12 +55,12 @@ impl SkillProvider for HostSkillProvider { }; let Some(skill) = host_loaded_skills.outcome().skills.iter().find(|skill| { let skill_path = skill.path_to_skills_md.to_string_lossy(); - skill_path == request.resource.0.as_str() - || skill_path.replace('\\', "/") == request.resource.0 + skill_path == request.resource.as_str() + || skill_path.replace('\\', "/") == request.resource.as_str() }) else { return Err(SkillProviderError::new(format!( "host skill resource is not loaded: {}", - request.resource.0 + request.resource.as_str() ))); }; @@ -70,7 +70,7 @@ impl SkillProvider for HostSkillProvider { .map_err(|err| { SkillProviderError::new(format!( "failed to read host skill resource {}: {err}", - request.resource.0 + request.resource.as_str() )) })?; @@ -117,7 +117,7 @@ fn catalog_entry_from_skill(skill: &SkillMetadata, enabled: bool) -> SkillCatalo SkillAuthority::new(SkillSourceKind::Host, HOST_AUTHORITY_ID), skill.name.clone(), skill.description.clone(), - SkillResourceId(skill_path), + SkillResourceId::new(skill_path), ) .with_short_description(skill.short_description.clone()) .with_display_path(display_path) diff --git a/codex-rs/ext/skills/src/render.rs b/codex-rs/ext/skills/src/render.rs index 157ae4bff3..6b77d02140 100644 --- a/codex-rs/ext/skills/src/render.rs +++ b/codex-rs/ext/skills/src/render.rs @@ -2,11 +2,14 @@ use codex_core_skills::render_available_skills_body; use codex_extension_api::ContextualUserFragment; use codex_protocol::protocol::SKILLS_INSTRUCTIONS_CLOSE_TAG; use codex_protocol::protocol::SKILLS_INSTRUCTIONS_OPEN_TAG; +use codex_utils_string::take_bytes_at_char_boundary; use crate::catalog::SkillCatalog; -const MAX_AVAILABLE_SKILLS_CHARS: usize = 8_000; -const MAX_MAIN_PROMPT_CHARS: usize = 40_000; +const MAX_AVAILABLE_SKILLS_BYTES: usize = 8_000; +const MAX_MAIN_PROMPT_BYTES: usize = 8_000; +pub(crate) const MAX_SKILL_NAME_BYTES: usize = 256; +pub(crate) const MAX_SKILL_PATH_BYTES: usize = 1_024; #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct AvailableSkillsFragment { @@ -32,7 +35,7 @@ impl ContextualUserFragment for AvailableSkillsFragment { } pub(crate) fn available_skills_fragment(catalog: &SkillCatalog) -> Option { - let mut total_chars = 0usize; + let mut total_bytes = 0usize; let mut omitted = 0usize; let mut skill_lines = Vec::new(); @@ -46,12 +49,12 @@ pub(crate) fn available_skills_fragment(catalog: &SkillCatalog) -> Option MAX_AVAILABLE_SKILLS_CHARS { + let next_bytes = total_bytes.saturating_add(line.len()); + if next_bytes > MAX_AVAILABLE_SKILLS_BYTES { omitted = omitted.saturating_add(1); continue; } - total_chars = next_chars; + total_bytes = next_bytes; skill_lines.push(line); } @@ -79,12 +82,10 @@ fn render_skill_line(name: &str, description: &str, path: &str) -> String { } pub(crate) fn truncate_main_prompt_contents(contents: &str) -> (String, bool) { - let mut chars = 0usize; - for (index, _) in contents.char_indices() { - if chars == MAX_MAIN_PROMPT_CHARS { - return (contents[..index].to_string(), true); - } - chars = chars.saturating_add(1); - } - (contents.to_string(), false) + truncate_utf8_to_bytes(contents, MAX_MAIN_PROMPT_BYTES) +} + +pub(crate) fn truncate_utf8_to_bytes(contents: &str, max_bytes: usize) -> (String, bool) { + let truncated = take_bytes_at_char_boundary(contents, max_bytes); + (truncated.to_string(), truncated.len() < contents.len()) } diff --git a/codex-rs/ext/skills/src/selection.rs b/codex-rs/ext/skills/src/selection.rs index c4405142e5..cfea2e3abd 100644 --- a/codex-rs/ext/skills/src/selection.rs +++ b/codex-rs/ext/skills/src/selection.rs @@ -93,12 +93,12 @@ fn push_selected( } fn entry_matches_path(entry: &SkillCatalogEntry, path: &str) -> bool { - entry.main_prompt.0 == path + entry.main_prompt.as_str() == path || entry.id.0 == path || entry .display_path .as_deref() - .is_some_and(|display_path| display_path == path) + .is_some_and(|display_path| normalize_skill_path(display_path) == path) } fn path_is_skill(path: &str) -> bool { diff --git a/codex-rs/ext/skills/src/sources.rs b/codex-rs/ext/skills/src/sources.rs index 9049fcb550..5ed51b878e 100644 --- a/codex-rs/ext/skills/src/sources.rs +++ b/codex-rs/ext/skills/src/sources.rs @@ -46,7 +46,7 @@ impl SkillProviderSource { fn should_list(&self, query: &SkillListQuery) -> bool { match &self.kind { SkillSourceKind::Host => query.include_host_skills, - SkillSourceKind::Executor => !query.executor_authorities.is_empty(), + SkillSourceKind::Executor => !query.executor_roots.is_empty(), SkillSourceKind::Remote => query.include_remote_skills, SkillSourceKind::Custom(_) => true, } diff --git a/codex-rs/ext/skills/src/state.rs b/codex-rs/ext/skills/src/state.rs index 4a639eb5f0..aa77cce822 100644 --- a/codex-rs/ext/skills/src/state.rs +++ b/codex-rs/ext/skills/src/state.rs @@ -1,4 +1,5 @@ use codex_core::config::Config; +use codex_protocol::capabilities::SelectedCapabilityRoot; use std::sync::Mutex; use crate::catalog::SkillCatalog; @@ -22,12 +23,17 @@ impl SkillsExtensionConfig { #[derive(Debug)] pub(crate) struct SkillsThreadState { config: Mutex, + selected_roots: Vec, } impl SkillsThreadState { - pub(crate) fn new(config: SkillsExtensionConfig) -> Self { + pub(crate) fn new( + config: SkillsExtensionConfig, + selected_roots: Vec, + ) -> Self { Self { config: Mutex::new(config), + selected_roots, } } @@ -44,6 +50,10 @@ impl SkillsThreadState { .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) = config; } + + pub(crate) fn selected_roots(&self) -> &[SelectedCapabilityRoot] { + &self.selected_roots + } } #[derive(Clone, Debug, Default, PartialEq, Eq)] diff --git a/codex-rs/ext/skills/tests/executor_file_system_authority.rs b/codex-rs/ext/skills/tests/executor_file_system_authority.rs new file mode 100644 index 0000000000..026c3f3348 --- /dev/null +++ b/codex-rs/ext/skills/tests/executor_file_system_authority.rs @@ -0,0 +1,337 @@ +use std::io; +use std::path::Path; +use std::sync::Arc; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; + +use async_trait::async_trait; +use codex_core_skills::HostLoadedSkills; +use codex_core_skills::loader::SkillRoot; +use codex_core_skills::loader::load_skills_from_roots; +use codex_exec_server::CopyOptions; +use codex_exec_server::CreateDirectoryOptions; +use codex_exec_server::EnvironmentManager; +use codex_exec_server::ExecutorFileSystem; +use codex_exec_server::FileMetadata; +use codex_exec_server::FileSystemResult; +use codex_exec_server::FileSystemSandboxContext; +use codex_exec_server::ReadDirectoryEntry; +use codex_exec_server::RemoveOptions; +use codex_protocol::capabilities::CapabilityRootLocation; +use codex_protocol::capabilities::SelectedCapabilityRoot; +use codex_protocol::protocol::SkillScope; +use codex_skills_extension::ExecutorSkillProvider; +use codex_skills_extension::catalog::SkillReadResult; +use codex_skills_extension::provider::SkillListQuery; +use codex_skills_extension::provider::SkillProvider; +use codex_skills_extension::provider::SkillReadRequest; +use codex_utils_absolute_path::AbsolutePathBuf; +use pretty_assertions::assert_eq; + +const SKILL_CONTENTS: &str = + "---\nname: synthetic\ndescription: Synthetic executor skill.\n---\n\nEXECUTOR_ONLY_BODY\n"; +static NEXT_TEST_ROOT_ID: AtomicUsize = AtomicUsize::new(0); + +struct SyntheticFileSystem { + alias_root: AbsolutePathBuf, + canonical_root: AbsolutePathBuf, +} + +impl SyntheticFileSystem { + fn metadata(&self, path: &AbsolutePathBuf) -> io::Result { + let skill_dir = self.canonical_root.join("skill"); + let skill_path = skill_dir.join("SKILL.md"); + let (is_directory, is_file) = if path == &self.canonical_root || path == &skill_dir { + (true, false) + } else if path == &skill_path { + (false, true) + } else { + return Err(io::Error::new(io::ErrorKind::NotFound, "not found")); + }; + Ok(FileMetadata { + is_directory, + is_file, + is_symlink: false, + created_at_ms: 0, + modified_at_ms: 0, + }) + } +} + +#[async_trait] +impl ExecutorFileSystem for SyntheticFileSystem { + async fn canonicalize( + &self, + path: &AbsolutePathBuf, + _sandbox: Option<&FileSystemSandboxContext>, + ) -> FileSystemResult { + if path == &self.alias_root { + return Ok(self.canonical_root.clone()); + } + self.metadata(path)?; + Ok(path.clone()) + } + + async fn join( + &self, + base_path: &AbsolutePathBuf, + path: &Path, + ) -> FileSystemResult { + Ok(base_path.join(path)) + } + + async fn parent(&self, path: &AbsolutePathBuf) -> FileSystemResult> { + Ok(path.parent()) + } + + async fn read_file( + &self, + path: &AbsolutePathBuf, + _sandbox: Option<&FileSystemSandboxContext>, + ) -> FileSystemResult> { + if path == &self.canonical_root.join("skill/SKILL.md") { + Ok(SKILL_CONTENTS.as_bytes().to_vec()) + } else { + Err(io::Error::new(io::ErrorKind::NotFound, "not found")) + } + } + + async fn write_file( + &self, + _path: &AbsolutePathBuf, + _contents: Vec, + _sandbox: Option<&FileSystemSandboxContext>, + ) -> FileSystemResult<()> { + Err(io::Error::new(io::ErrorKind::Unsupported, "read only")) + } + + async fn create_directory( + &self, + _path: &AbsolutePathBuf, + _options: CreateDirectoryOptions, + _sandbox: Option<&FileSystemSandboxContext>, + ) -> FileSystemResult<()> { + Err(io::Error::new(io::ErrorKind::Unsupported, "read only")) + } + + async fn get_metadata( + &self, + path: &AbsolutePathBuf, + _sandbox: Option<&FileSystemSandboxContext>, + ) -> FileSystemResult { + self.metadata(path) + } + + async fn read_directory( + &self, + path: &AbsolutePathBuf, + _sandbox: Option<&FileSystemSandboxContext>, + ) -> FileSystemResult> { + if path == &self.canonical_root { + Ok(vec![ReadDirectoryEntry { + file_name: "skill".to_string(), + is_directory: true, + is_file: false, + }]) + } else if path == &self.canonical_root.join("skill") { + Ok(vec![ReadDirectoryEntry { + file_name: "SKILL.md".to_string(), + is_directory: false, + is_file: true, + }]) + } else { + Err(io::Error::new(io::ErrorKind::NotFound, "not found")) + } + } + + async fn remove( + &self, + _path: &AbsolutePathBuf, + _options: RemoveOptions, + _sandbox: Option<&FileSystemSandboxContext>, + ) -> FileSystemResult<()> { + Err(io::Error::new(io::ErrorKind::Unsupported, "read only")) + } + + async fn copy( + &self, + _source_path: &AbsolutePathBuf, + _destination_path: &AbsolutePathBuf, + _options: CopyOptions, + _sandbox: Option<&FileSystemSandboxContext>, + ) -> FileSystemResult<()> { + Err(io::Error::new(io::ErrorKind::Unsupported, "read only")) + } +} + +#[tokio::test] +async fn skill_loading_and_reads_use_the_supplied_executor_file_system() { + let test_root = + std::env::temp_dir().join(format!("codex-executor-skill-fs-{}", std::process::id())); + let alias_root = AbsolutePathBuf::from_absolute_path_checked(test_root.join("alias")) + .expect("absolute path"); + let canonical_root = AbsolutePathBuf::from_absolute_path_checked(test_root.join("canonical")) + .expect("absolute path"); + assert!(!alias_root.as_path().exists()); + assert!(!canonical_root.as_path().exists()); + + let outcome = load_skills_from_roots([SkillRoot { + path: alias_root.clone(), + scope: SkillScope::User, + file_system: Arc::new(SyntheticFileSystem { + alias_root, + canonical_root: canonical_root.clone(), + }), + plugin_id: None, + plugin_root: None, + }]) + .await; + assert_eq!(outcome.errors, Vec::new()); + assert_eq!(outcome.skills.len(), 1); + + let skill = outcome.skills[0].clone(); + assert_eq!(skill.name, "synthetic"); + assert_eq!( + skill.path_to_skills_md, + canonical_root.join("skill/SKILL.md") + ); + let loaded = HostLoadedSkills::new(Arc::new(outcome)); + assert_eq!( + loaded.read_skill_text(&skill).await.expect("skill body"), + SKILL_CONTENTS + ); +} + +#[tokio::test] +async fn executor_provider_reads_from_the_environment_instance_used_for_listing() { + let test_root = create_local_skill_root("bound-instance").expect("create local skill root"); + let root_path = test_root.to_string_lossy().into_owned(); + let environment_manager = Arc::new(EnvironmentManager::default_for_tests()); + let provider = ExecutorSkillProvider::new_with_restriction_product( + Arc::clone(&environment_manager), + /*restriction_product*/ None, + ); + let catalog = provider + .list(SkillListQuery { + turn_id: "turn-1".to_string(), + executor_roots: vec![SelectedCapabilityRoot { + id: "root-a".to_string(), + location: CapabilityRootLocation::Environment { + environment_id: "local".to_string(), + path: root_path, + }, + }], + host: None, + include_host_skills: false, + include_bundled_skills: true, + include_remote_skills: false, + }) + .await + .expect("list executor skills"); + let entry = catalog + .entries + .into_iter() + .next() + .expect("listed executor skill"); + let resource = entry.main_prompt.clone(); + + environment_manager + .upsert_environment("local".to_string(), "http://127.0.0.1:1".to_string()) + .expect("replace environment"); + + assert_eq!( + provider + .read(SkillReadRequest { + authority: entry.authority, + package: entry.id, + resource: resource.clone(), + host: None, + }) + .await + .expect("read bound executor skill"), + SkillReadResult { + resource, + contents: SKILL_CONTENTS.to_string(), + } + ); + + std::fs::remove_dir_all(test_root).expect("remove skill directory"); +} + +#[tokio::test] +async fn selected_root_id_distinguishes_identical_executor_paths() { + let test_root = create_local_skill_root("root-identity").expect("create local skill root"); + let root_path = test_root.to_string_lossy().into_owned(); + let canonical_root = AbsolutePathBuf::from_absolute_path_checked(&test_root) + .expect("absolute skill root") + .canonicalize() + .expect("canonicalize skill root") + .to_string_lossy() + .replace('\\', "/"); + let provider = ExecutorSkillProvider::new_with_restriction_product( + Arc::new(EnvironmentManager::default_for_tests()), + /*restriction_product*/ None, + ); + + let catalog = provider + .list(SkillListQuery { + turn_id: "turn-1".to_string(), + executor_roots: ["root-a", "root-b"] + .into_iter() + .map(|id| SelectedCapabilityRoot { + id: id.to_string(), + location: CapabilityRootLocation::Environment { + environment_id: "local".to_string(), + path: root_path.clone(), + }, + }) + .collect(), + host: None, + include_host_skills: false, + include_bundled_skills: true, + include_remote_skills: false, + }) + .await + .expect("list executor skills"); + + assert_eq!( + catalog + .entries + .iter() + .map(|entry| ( + entry.authority.id.clone(), + entry.display_path.clone().expect("display path"), + )) + .collect::>(), + vec![ + ( + "root-a".to_string(), + format!( + "skill://root-a/{}/skill/SKILL.md", + canonical_root.trim_start_matches('/') + ), + ), + ( + "root-b".to_string(), + format!( + "skill://root-b/{}/skill/SKILL.md", + canonical_root.trim_start_matches('/') + ), + ), + ] + ); + + std::fs::remove_dir_all(test_root).expect("remove skill directory"); +} + +fn create_local_skill_root(label: &str) -> io::Result { + let id = NEXT_TEST_ROOT_ID.fetch_add(1, Ordering::Relaxed); + let test_root = std::env::temp_dir().join(format!( + "codex-executor-skill-{label}-{}-{id}", + std::process::id() + )); + let skill_dir = test_root.join("skill"); + std::fs::create_dir_all(&skill_dir)?; + std::fs::write(skill_dir.join("SKILL.md"), SKILL_CONTENTS)?; + Ok(test_root) +} diff --git a/codex-rs/ext/skills/tests/skills_extension.rs b/codex-rs/ext/skills/tests/skills_extension.rs index 1e2a2f0b60..dc1aa01407 100644 --- a/codex-rs/ext/skills/tests/skills_extension.rs +++ b/codex-rs/ext/skills/tests/skills_extension.rs @@ -14,7 +14,8 @@ use codex_extension_api::ExtensionData; use codex_extension_api::ExtensionRegistryBuilder; use codex_extension_api::ThreadStartInput; use codex_extension_api::TurnInputContext; -use codex_extension_api::TurnInputEnvironment; +use codex_protocol::capabilities::CapabilityRootLocation; +use codex_protocol::capabilities::SelectedCapabilityRoot; use codex_protocol::protocol::SKILLS_INSTRUCTIONS_OPEN_TAG; use codex_protocol::protocol::SessionSource; use codex_protocol::user_input::UserInput; @@ -112,6 +113,7 @@ async fn installed_extension_loads_host_skills_from_legacy_roots() -> TestResult .await; assert_eq!(2, fragments.len()); + assert_eq!("developer", fragments[0].role()); assert!(fragments[0].render().contains("demo")); assert!(fragments[0].render().contains(&skill_prompt_path)); assert_eq!("user", fragments[1].role()); @@ -128,42 +130,35 @@ async fn installed_extension_loads_host_skills_from_legacy_roots() -> TestResult } #[tokio::test] -async fn installed_extension_injects_available_catalog_and_selected_entrypoint() -> TestResult { - let host_read_requests = Arc::new(Mutex::new(Vec::new())); - let remote_read_requests = Arc::new(Mutex::new(Vec::new())); - let host_provider = Arc::new(StaticSkillProvider { +async fn selected_executor_catalog_is_context_and_selected_entrypoint_is_turn_input() -> TestResult +{ + let read_requests = Arc::new(Mutex::new(Vec::new())); + let executor_provider = Arc::new(StaticSkillProvider { catalog: SkillCatalog { entries: vec![test_entry( - SkillSourceKind::Host, - "host", - "host/lint-fix", + SkillSourceKind::Executor, + "env-1", + "executor/lint-fix", "lint-fix/SKILL.md", )], warnings: Vec::new(), }, - read_requests: Arc::clone(&host_read_requests), + read_requests: Arc::clone(&read_requests), }); - let remote_provider = Arc::new(StaticSkillProvider { - catalog: SkillCatalog { - entries: vec![test_entry( - SkillSourceKind::Remote, - "remote", - "remote/lint-fix", - "lint-fix/SKILL.md", - )], - warnings: Vec::new(), - }, - read_requests: Arc::clone(&remote_read_requests), - }); - let providers = SkillProviders::new() - .with_host_provider(host_provider) - .with_remote_provider(remote_provider); + let providers = SkillProviders::new().with_executor_provider(executor_provider); let mut builder = ExtensionRegistryBuilder::new(); install_with_providers(&mut builder, providers); let registry = builder.build(); let session_store = ExtensionData::new("session"); let thread_store = ExtensionData::new("thread"); + thread_store.insert(vec![SelectedCapabilityRoot { + id: "lint-fix".to_string(), + location: CapabilityRootLocation::Environment { + environment_id: "env-1".to_string(), + path: "/skills/lint-fix".to_string(), + }, + }]); let session_source = SessionSource::Cli; let config = default_config().await?; registry.thread_lifecycle_contributors()[0] @@ -176,6 +171,17 @@ async fn installed_extension_injects_available_catalog_and_selected_entrypoint() }) .await; + let prompt_fragments = registry.context_contributors()[0] + .contribute(&session_store, &thread_store) + .await; + assert_eq!(1, prompt_fragments.len()); + assert!( + prompt_fragments[0] + .text() + .starts_with(SKILLS_INSTRUCTIONS_OPEN_TAG) + ); + assert!(prompt_fragments[0].text().contains("lint-fix")); + let turn_store = ExtensionData::new("turn-1"); let fragments = registry.turn_input_contributors()[0] .contribute( @@ -185,11 +191,7 @@ async fn installed_extension_injects_available_catalog_and_selected_entrypoint() text: "$lint-fix please".to_string(), text_elements: Vec::new(), }], - environments: vec![TurnInputEnvironment { - environment_id: "env-1".to_string(), - cwd: std::env::temp_dir(), - is_primary: true, - }], + environments: Vec::new(), }, &session_store, &thread_store, @@ -197,31 +199,23 @@ async fn installed_extension_injects_available_catalog_and_selected_entrypoint() ) .await; - assert_eq!(2, fragments.len()); - assert_eq!("developer", fragments[0].role()); - assert!( - fragments[0] - .render() - .starts_with(SKILLS_INSTRUCTIONS_OPEN_TAG) - ); - assert!(fragments[0].render().contains("lint-fix")); - assert_eq!("user", fragments[1].role()); - assert!(fragments[1].render().contains("lint-fix")); - assert!(fragments[1].render().contains("# Lint Fix")); + assert_eq!(1, fragments.len()); + assert_eq!("user", fragments[0].role()); + assert!(fragments[0].render().contains("lint-fix")); + assert!(fragments[0].render().contains("# Lint Fix")); assert_eq!( vec![( - SkillAuthority::new(SkillSourceKind::Host, "host"), - SkillPackageId("host/lint-fix".to_string()), - SkillResourceId("lint-fix/SKILL.md".to_string()), + SkillAuthority::new(SkillSourceKind::Executor, "env-1"), + SkillPackageId("executor/lint-fix".to_string()), + SkillResourceId::new("lint-fix/SKILL.md"), )], - read_request_keys(&host_read_requests) - ); - assert!( - remote_read_requests - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .is_empty() + read_request_keys(&read_requests) ); + let rebuilt_prompt_fragments = registry.context_contributors()[0] + .contribute(&session_store, &thread_store) + .await; + assert_eq!(1, rebuilt_prompt_fragments.len()); + assert!(rebuilt_prompt_fragments[0].text().contains("lint-fix")); let next_turn_store = ExtensionData::new("turn-2"); let next_fragments = registry.turn_input_contributors()[0] @@ -240,9 +234,91 @@ async fn installed_extension_injects_available_catalog_and_selected_entrypoint() ) .await; - assert_eq!(1, next_fragments.len()); - assert_eq!("developer", next_fragments[0].role()); - assert!(next_fragments[0].render().contains("lint-fix")); + assert!(next_fragments.is_empty()); + + Ok(()) +} + +#[tokio::test] +async fn root_qualified_locator_selects_only_the_matching_executor_skill() -> TestResult { + let read_requests = Arc::new(Mutex::new(Vec::new())); + let root_a_locator = "skill://root-a/shared/lint-fix/SKILL.md"; + let root_b_locator = "skill://root-b/shared/lint-fix/SKILL.md"; + let executor_provider = Arc::new(StaticSkillProvider { + catalog: SkillCatalog { + entries: [("root-a", root_a_locator), ("root-b", root_b_locator)] + .into_iter() + .map(|(root_id, locator)| { + SkillCatalogEntry::new( + SkillPackageId(locator.to_string()), + SkillAuthority::new(SkillSourceKind::Executor, root_id), + "lint-fix", + "Fix lint errors.", + SkillResourceId::new(locator), + ) + .with_display_path(locator) + }) + .collect(), + warnings: Vec::new(), + }, + read_requests: Arc::clone(&read_requests), + }); + let providers = SkillProviders::new().with_executor_provider(executor_provider); + let mut builder = ExtensionRegistryBuilder::new(); + install_with_providers(&mut builder, providers); + let registry = builder.build(); + let session_store = ExtensionData::new("session"); + let thread_store = ExtensionData::new("thread"); + thread_store.insert( + [("root-a", "/skills/root-a"), ("root-b", "/skills/root-b")] + .into_iter() + .map(|(id, path)| SelectedCapabilityRoot { + id: id.to_string(), + location: CapabilityRootLocation::Environment { + environment_id: "env-1".to_string(), + path: path.to_string(), + }, + }) + .collect::>(), + ); + let session_source = SessionSource::Cli; + let config = default_config().await?; + registry.thread_lifecycle_contributors()[0] + .on_thread_start(ThreadStartInput { + config: &config, + session_source: &session_source, + persistent_thread_state_available: true, + session_store: &session_store, + thread_store: &thread_store, + }) + .await; + + let fragments = registry.turn_input_contributors()[0] + .contribute( + TurnInputContext { + turn_id: "turn-1".to_string(), + user_input: vec![UserInput::Mention { + name: "lint-fix".to_string(), + path: root_b_locator.to_string(), + }], + environments: Vec::new(), + }, + &session_store, + &thread_store, + &ExtensionData::new("turn-1"), + ) + .await; + + assert_eq!(1, fragments.len()); + assert!(fragments[0].render().contains(root_b_locator)); + assert_eq!( + vec![( + SkillAuthority::new(SkillSourceKind::Executor, "root-b"), + SkillPackageId(root_b_locator.to_string()), + SkillResourceId::new(root_b_locator), + )], + read_request_keys(&read_requests) + ); Ok(()) } @@ -306,15 +382,14 @@ async fn prompt_hidden_skill_can_still_be_invoked() -> TestResult { .await; assert_eq!(2, fragments.len()); - let catalog_fragment = fragments[0].render(); - assert!(catalog_fragment.contains("visible-skill")); - assert!(!catalog_fragment.contains("hidden-skill")); + assert!(fragments[0].render().contains("visible-skill")); + assert!(!fragments[0].render().contains("hidden-skill")); assert!(fragments[1].render().contains("hidden-skill")); assert_eq!( vec![( SkillAuthority::new(SkillSourceKind::Host, "host"), SkillPackageId("host/hidden-skill".to_string()), - SkillResourceId("hidden-skill/SKILL.md".to_string()), + SkillResourceId::new("hidden-skill/SKILL.md"), )], read_request_keys(&read_requests) ); @@ -329,13 +404,9 @@ struct StaticSkillProvider { } impl SkillProvider for StaticSkillProvider { - fn list(&self, query: SkillListQuery) -> SkillProviderFuture<'_, SkillCatalog> { + fn list(&self, _query: SkillListQuery) -> SkillProviderFuture<'_, SkillCatalog> { let catalog = self.catalog.clone(); - Box::pin(async move { - assert!(query.include_host_skills); - assert!(query.include_bundled_skills); - Ok(catalog) - }) + Box::pin(async move { Ok(catalog) }) } fn read(&self, request: SkillReadRequest) -> SkillProviderFuture<'_, SkillReadResult> { @@ -369,7 +440,7 @@ fn test_entry( SkillAuthority::new(kind, authority_id), name, "Fix lint errors.", - SkillResourceId(main_prompt.to_string()), + SkillResourceId::new(main_prompt), ) .with_display_path(format!("skill://{package_id}/SKILL.md")) } diff --git a/codex-rs/memories/write/src/runtime.rs b/codex-rs/memories/write/src/runtime.rs index 9c15ff04e2..d524a2e224 100644 --- a/codex-rs/memories/write/src/runtime.rs +++ b/codex-rs/memories/write/src/runtime.rs @@ -251,6 +251,7 @@ impl MemoryStartupContext { metrics_service_name: None, parent_trace: None, environments, + thread_extension_init: Default::default(), }) .await?; diff --git a/codex-rs/protocol/src/capabilities.rs b/codex-rs/protocol/src/capabilities.rs new file mode 100644 index 0000000000..cfdead7b28 --- /dev/null +++ b/codex-rs/protocol/src/capabilities.rs @@ -0,0 +1,30 @@ +use schemars::JsonSchema; +use serde::Deserialize; +use serde::Serialize; +use ts_rs::TS; + +/// A user-selected root that can expose one or more runtime capabilities. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct SelectedCapabilityRoot { + /// Stable identifier supplied by the capability selection platform. + pub id: String, + /// Where the selected root can be resolved. + pub location: CapabilityRootLocation, +} + +/// Location used to resolve a selected capability root. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema, TS)] +#[serde(tag = "type", rename_all = "camelCase")] +#[ts(tag = "type")] +#[ts(export_to = "v2/")] +pub enum CapabilityRootLocation { + /// A path owned by an execution environment. + Environment { + #[serde(rename = "environmentId")] + #[ts(rename = "environmentId")] + environment_id: String, + path: String, + }, +} diff --git a/codex-rs/protocol/src/lib.rs b/codex-rs/protocol/src/lib.rs index 63053159c6..3be16d08b7 100644 --- a/codex-rs/protocol/src/lib.rs +++ b/codex-rs/protocol/src/lib.rs @@ -9,6 +9,7 @@ pub use session_id::SessionId; pub use thread_id::ThreadId; pub use tool_name::ToolName; pub mod approvals; +pub mod capabilities; pub mod config_types; pub mod dynamic_tools; pub mod error; From 99da697e4c5c1fc908732a58b6548bf9cc227f83 Mon Sep 17 00:00:00 2001 From: "Adam Perry @ OpenAI" Date: Tue, 9 Jun 2026 11:30:24 -0700 Subject: [PATCH 03/47] [codex] Test extension API contracts (#26835) ## Why `codex-extension-api` defines contracts shared by extension crates and their hosts, but it had no direct test suite. Host and feature tests cover downstream behavior, while regressions in the API crate's own typed state, registry ordering, and capability adapters could go unnoticed. ## What - Add public-surface integration tests for `ExtensionData`, including concurrent initialization and poison recovery. - Cover contributor registration order, approval short-circuiting, event sink retention, no-op response injection, and closure-based agent spawning. - Add the test-only dependencies used by the suite. ## Validation - `just test -p codex-extension-api` - `just argument-comment-lint -p codex-extension-api` - `just bazel-lock-check` --- codex-rs/Cargo.lock | 2 + codex-rs/ext/extension-api/Cargo.toml | 4 + .../ext/extension-api/tests/capabilities.rs | 56 +++ codex-rs/ext/extension-api/tests/registry.rs | 370 ++++++++++++++++++ codex-rs/ext/extension-api/tests/state.rs | 109 ++++++ 5 files changed, 541 insertions(+) create mode 100644 codex-rs/ext/extension-api/tests/capabilities.rs create mode 100644 codex-rs/ext/extension-api/tests/registry.rs create mode 100644 codex-rs/ext/extension-api/tests/state.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 064d9ad944..8783f0e16c 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -2923,6 +2923,8 @@ dependencies = [ "codex-context-fragments", "codex-protocol", "codex-tools", + "pretty_assertions", + "tokio", ] [[package]] diff --git a/codex-rs/ext/extension-api/Cargo.toml b/codex-rs/ext/extension-api/Cargo.toml index 85c7d8f98e..5497dc5439 100644 --- a/codex-rs/ext/extension-api/Cargo.toml +++ b/codex-rs/ext/extension-api/Cargo.toml @@ -18,3 +18,7 @@ async-trait = { workspace = true } codex-context-fragments = { workspace = true } codex-protocol = { workspace = true } codex-tools = { workspace = true } + +[dev-dependencies] +pretty_assertions = { workspace = true } +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/codex-rs/ext/extension-api/tests/capabilities.rs b/codex-rs/ext/extension-api/tests/capabilities.rs new file mode 100644 index 0000000000..c01b8f5723 --- /dev/null +++ b/codex-rs/ext/extension-api/tests/capabilities.rs @@ -0,0 +1,56 @@ +use std::sync::Arc; +use std::sync::Mutex; + +use codex_extension_api::AgentSpawnFuture; +use codex_extension_api::AgentSpawner; +use codex_extension_api::NoopResponseItemInjector; +use codex_extension_api::ResponseItemInjector; +use codex_protocol::ThreadId; +use codex_protocol::models::ContentItem; +use codex_protocol::models::ResponseInputItem; +use pretty_assertions::assert_eq; + +#[tokio::test] +async fn noop_response_item_injector_returns_original_items() { + let items = vec![ResponseInputItem::Message { + role: "user".to_string(), + content: vec![ContentItem::InputText { + text: "keep this input".to_string(), + }], + phase: None, + }]; + + let returned_items = NoopResponseItemInjector + .inject_response_items(items.clone()) + .await + .expect_err("noop injector should reject same-turn injection"); + + assert_eq!(returned_items, items); +} + +#[tokio::test] +async fn closure_agent_spawner_forwards_arguments_and_result() { + let calls = Arc::new(Mutex::new(Vec::new())); + let recorded_calls = Arc::clone(&calls); + let spawner = move |thread_id: ThreadId, + request: String| + -> AgentSpawnFuture<'static, usize, &'static str> { + recorded_calls + .lock() + .expect("agent spawn calls lock") + .push((thread_id, request.clone())); + Box::pin(async move { Ok(request.len()) }) + }; + let thread_id = + ThreadId::from_string("11111111-1111-4111-8111-111111111111").expect("valid thread id"); + + let spawned = spawner + .spawn_subagent(thread_id, "delegate this".to_string()) + .await; + + assert_eq!(spawned, Ok(13)); + assert_eq!( + calls.lock().expect("agent spawn calls lock").as_slice(), + [(thread_id, "delegate this".to_string())] + ); +} diff --git a/codex-rs/ext/extension-api/tests/registry.rs b/codex-rs/ext/extension-api/tests/registry.rs new file mode 100644 index 0000000000..d26f1010cb --- /dev/null +++ b/codex-rs/ext/extension-api/tests/registry.rs @@ -0,0 +1,370 @@ +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; +use std::sync::Mutex; + +use codex_extension_api::ApprovalReviewContributor; +use codex_extension_api::ConfigContributor; +use codex_extension_api::ContextContributor; +use codex_extension_api::ContextualUserFragment; +use codex_extension_api::ExtensionData; +use codex_extension_api::ExtensionEventSink; +use codex_extension_api::ExtensionRegistryBuilder; +use codex_extension_api::PromptFragment; +use codex_extension_api::ThreadLifecycleContributor; +use codex_extension_api::TokenUsageContributor; +use codex_extension_api::ToolCall; +use codex_extension_api::ToolContributor; +use codex_extension_api::ToolExecutor; +use codex_extension_api::ToolLifecycleContributor; +use codex_extension_api::TurnInputContext; +use codex_extension_api::TurnInputContributor; +use codex_extension_api::TurnItemContributor; +use codex_extension_api::TurnLifecycleContributor; +use codex_extension_api::empty_extension_registry; +use codex_protocol::items::HookPromptItem; +use codex_protocol::items::TurnItem; +use codex_protocol::protocol::Event; +use codex_protocol::protocol::EventMsg; +use codex_protocol::protocol::ReviewDecision; +use codex_protocol::protocol::WarningEvent; +use pretty_assertions::assert_eq; + +struct AllContributors; + +impl ContextContributor for AllContributors { + fn contribute<'a>( + &'a self, + _session_store: &'a ExtensionData, + _thread_store: &'a ExtensionData, + ) -> Pin> + Send + 'a>> { + Box::pin(std::future::ready(Vec::new())) + } +} + +#[async_trait::async_trait] +impl ThreadLifecycleContributor<()> for AllContributors {} + +#[async_trait::async_trait] +impl TurnLifecycleContributor for AllContributors {} + +impl ConfigContributor<()> for AllContributors {} + +#[async_trait::async_trait] +impl TokenUsageContributor for AllContributors {} + +#[async_trait::async_trait] +impl TurnInputContributor for AllContributors { + async fn contribute( + &self, + _input: TurnInputContext, + _session_store: &ExtensionData, + _thread_store: &ExtensionData, + _turn_store: &ExtensionData, + ) -> Vec> { + Vec::new() + } +} + +impl ToolContributor for AllContributors { + fn tools( + &self, + _session_store: &ExtensionData, + _thread_store: &ExtensionData, + ) -> Vec>> { + Vec::new() + } +} + +impl ToolLifecycleContributor for AllContributors {} + +#[async_trait::async_trait] +impl TurnItemContributor for AllContributors { + async fn contribute( + &self, + _thread_store: &ExtensionData, + _turn_store: &ExtensionData, + _item: &mut TurnItem, + ) -> Result<(), String> { + Ok(()) + } +} + +#[async_trait::async_trait] +impl ApprovalReviewContributor for AllContributors { + async fn contribute( + &self, + _session_store: &ExtensionData, + _thread_store: &ExtensionData, + _prompt: &str, + ) -> Option { + Some(ReviewDecision::ApprovedForSession) + } +} + +#[tokio::test] +async fn build_round_trips_every_contributor_category() { + let contributor = Arc::new(AllContributors); + let mut builder = ExtensionRegistryBuilder::<()>::new(); + builder.thread_lifecycle_contributor(contributor.clone()); + builder.turn_lifecycle_contributor(contributor.clone()); + builder.config_contributor(contributor.clone()); + builder.token_usage_contributor(contributor.clone()); + builder.prompt_contributor(contributor.clone()); + builder.turn_input_contributor(contributor.clone()); + builder.tool_contributor(contributor.clone()); + builder.tool_lifecycle_contributor(contributor.clone()); + builder.turn_item_contributor(contributor.clone()); + builder.approval_review_contributor(contributor); + let registry = builder.build(); + + assert_eq!(registry.thread_lifecycle_contributors().len(), 1); + assert_eq!(registry.turn_lifecycle_contributors().len(), 1); + assert_eq!(registry.config_contributors().len(), 1); + assert_eq!(registry.token_usage_contributors().len(), 1); + assert_eq!(registry.context_contributors().len(), 1); + assert_eq!(registry.turn_input_contributors().len(), 1); + assert_eq!(registry.tool_contributors().len(), 1); + assert_eq!(registry.tool_lifecycle_contributors().len(), 1); + assert_eq!(registry.turn_item_contributors().len(), 1); + assert_eq!( + registry + .approval_review( + &ExtensionData::new("session"), + &ExtensionData::new("thread"), + "review this", + ) + .await, + Some(ReviewDecision::ApprovedForSession) + ); +} + +struct NamedContextContributor(&'static str); + +impl ContextContributor for NamedContextContributor { + fn contribute<'a>( + &'a self, + _session_store: &'a ExtensionData, + _thread_store: &'a ExtensionData, + ) -> Pin> + Send + 'a>> { + Box::pin(std::future::ready(vec![PromptFragment::developer_policy( + self.0, + )])) + } +} + +struct RecordingTurnItemContributor { + name: &'static str, + calls: Arc>>, +} + +#[async_trait::async_trait] +impl TurnItemContributor for RecordingTurnItemContributor { + async fn contribute( + &self, + _thread_store: &ExtensionData, + _turn_store: &ExtensionData, + _item: &mut TurnItem, + ) -> Result<(), String> { + self.calls + .lock() + .unwrap_or_else(|error| panic!("turn item calls lock poisoned: {error}")) + .push(self.name); + Ok(()) + } +} + +#[tokio::test] +async fn contributors_preserve_registration_order() { + let turn_item_calls = Arc::new(Mutex::new(Vec::new())); + let mut builder = ExtensionRegistryBuilder::<()>::new(); + builder.prompt_contributor(Arc::new(NamedContextContributor("first"))); + builder.prompt_contributor(Arc::new(NamedContextContributor("second"))); + for name in ["first", "second"] { + builder.turn_item_contributor(Arc::new(RecordingTurnItemContributor { + name, + calls: Arc::clone(&turn_item_calls), + })); + } + let registry = builder.build(); + let session_store = ExtensionData::new("session"); + let thread_store = ExtensionData::new("thread"); + let turn_store = ExtensionData::new("turn"); + + let mut fragments = Vec::new(); + for contributor in registry.context_contributors() { + fragments.extend(contributor.contribute(&session_store, &thread_store).await); + } + let mut item = TurnItem::HookPrompt(HookPromptItem { + id: "item".to_string(), + fragments: Vec::new(), + }); + for contributor in registry.turn_item_contributors() { + contributor + .contribute(&thread_store, &turn_store, &mut item) + .await + .expect("turn item contribution should succeed"); + } + + assert_eq!( + fragments, + vec![ + PromptFragment::developer_policy("first"), + PromptFragment::developer_policy("second"), + ] + ); + assert_eq!( + turn_item_calls + .lock() + .expect("turn item calls lock") + .as_slice(), + ["first", "second"] + ); +} + +#[derive(Debug, PartialEq, Eq)] +struct ApprovalCall { + contributor: &'static str, + session_id: String, + thread_id: String, + prompt: String, +} + +struct RecordingApprovalContributor { + name: &'static str, + decision: Option, + calls: Arc>>, +} + +#[async_trait::async_trait] +impl ApprovalReviewContributor for RecordingApprovalContributor { + async fn contribute( + &self, + session_store: &ExtensionData, + thread_store: &ExtensionData, + prompt: &str, + ) -> Option { + self.calls + .lock() + .unwrap_or_else(|error| panic!("approval calls lock poisoned: {error}")) + .push(ApprovalCall { + contributor: self.name, + session_id: session_store.level_id().to_string(), + thread_id: thread_store.level_id().to_string(), + prompt: prompt.to_string(), + }); + self.decision.clone() + } +} + +#[tokio::test] +async fn approval_review_returns_first_claim_and_short_circuits() { + let calls = Arc::new(Mutex::new(Vec::new())); + let mut builder = ExtensionRegistryBuilder::<()>::new(); + for (name, decision) in [ + ("first", None), + ("second", Some(ReviewDecision::Approved)), + ("third", Some(ReviewDecision::Denied)), + ] { + builder.approval_review_contributor(Arc::new(RecordingApprovalContributor { + name, + decision, + calls: Arc::clone(&calls), + })); + } + let registry = builder.build(); + + let decision = registry + .approval_review( + &ExtensionData::new("session-1"), + &ExtensionData::new("thread-1"), + "allow command?", + ) + .await; + + assert_eq!(decision, Some(ReviewDecision::Approved)); + assert_eq!( + calls.lock().expect("approval calls lock").as_slice(), + [ + ApprovalCall { + contributor: "first", + session_id: "session-1".to_string(), + thread_id: "thread-1".to_string(), + prompt: "allow command?".to_string(), + }, + ApprovalCall { + contributor: "second", + session_id: "session-1".to_string(), + thread_id: "thread-1".to_string(), + prompt: "allow command?".to_string(), + }, + ] + ); +} + +#[derive(Default)] +struct RecordingEventSink { + events: Mutex>, +} + +impl ExtensionEventSink for RecordingEventSink { + fn emit(&self, event: Event) { + let EventMsg::Warning(warning) = event.msg else { + panic!("test sink only accepts warning events"); + }; + self.events + .lock() + .unwrap_or_else(|error| panic!("recording event sink lock poisoned: {error}")) + .push((event.id, warning.message)); + } +} + +#[test] +fn custom_event_sink_survives_registry_build() { + let sink = Arc::new(RecordingEventSink::default()); + let builder = ExtensionRegistryBuilder::<()>::with_event_sink(sink.clone()); + + builder + .event_sink() + .emit(warning_event("builder", "before")); + let registry = builder.build(); + registry + .event_sink() + .emit(warning_event("registry", "after")); + + assert_eq!( + sink.events + .lock() + .expect("recording event sink lock") + .as_slice(), + [ + ("builder".to_string(), "before".to_string()), + ("registry".to_string(), "after".to_string()), + ] + ); +} + +#[tokio::test] +async fn empty_registry_does_not_claim_approval_review() { + let registry = empty_extension_registry::<()>(); + + assert_eq!( + registry + .approval_review( + &ExtensionData::new("session"), + &ExtensionData::new("thread"), + "unclaimed", + ) + .await, + None + ); +} + +fn warning_event(id: &str, message: &str) -> Event { + Event { + id: id.to_string(), + msg: EventMsg::Warning(WarningEvent { + message: message.to_string(), + }), + } +} diff --git a/codex-rs/ext/extension-api/tests/state.rs b/codex-rs/ext/extension-api/tests/state.rs new file mode 100644 index 0000000000..31b643c29b --- /dev/null +++ b/codex-rs/ext/extension-api/tests/state.rs @@ -0,0 +1,109 @@ +use std::panic::AssertUnwindSafe; +use std::sync::Arc; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; + +use codex_extension_api::ExtensionData; +use pretty_assertions::assert_eq; + +#[test] +fn typed_values_can_be_inserted_replaced_and_removed() { + let data = ExtensionData::new("thread-1"); + + assert_eq!(data.insert(/*value*/ 41_u64), None); + assert_eq!(data.insert("alpha".to_string()), None); + assert_eq!(data.get::().as_deref(), Some(&41)); + assert_eq!( + data.get::().map(|value| value.as_str().to_string()), + Some("alpha".to_string()) + ); + + assert_eq!(data.insert(/*value*/ 42_u64).as_deref(), Some(&41)); + assert_eq!(data.get::().as_deref(), Some(&42)); + assert_eq!( + data.remove::() + .map(|value| value.as_str().to_string()), + Some("alpha".to_string()) + ); + assert_eq!(data.get::(), None); + assert_eq!(data.get::().as_deref(), Some(&42)); +} + +#[test] +fn get_or_init_initializes_once_and_returns_shared_value() { + const CALLER_COUNT: usize = 8; + + #[derive(Debug, PartialEq, Eq)] + struct SharedValue(usize); + + let data = Arc::new(ExtensionData::new("session")); + let callers_started = Arc::new(AtomicUsize::new(0)); + let initialization_count = Arc::new(AtomicUsize::new(0)); + + let handles: [_; CALLER_COUNT] = std::array::from_fn(|_| { + let data = Arc::clone(&data); + let callers_started = Arc::clone(&callers_started); + let initialization_count = Arc::clone(&initialization_count); + std::thread::spawn(move || { + callers_started.fetch_add(1, Ordering::SeqCst); + data.get_or_init(|| { + initialization_count.fetch_add(1, Ordering::SeqCst); + // Keep the first initializer active until every worker has attempted + // get_or_init, forcing callers to overlap on the same missing entry. + while callers_started.load(Ordering::SeqCst) < CALLER_COUNT { + std::thread::yield_now(); + } + SharedValue(7) + }) + }) + }); + let values = handles + .into_iter() + .map(|handle| handle.join().expect("initializer thread should succeed")) + .collect::>(); + + assert_eq!(initialization_count.load(Ordering::SeqCst), 1); + assert_eq!( + values.iter().map(Arc::as_ref).collect::>(), + vec![&SharedValue(7); CALLER_COUNT] + ); + assert!( + values + .iter() + .skip(1) + .all(|value| Arc::ptr_eq(&values[0], value)) + ); +} + +#[test] +fn stores_are_isolated_and_preserve_level_id() { + let session_data = ExtensionData::new("root-1"); + let thread_data = ExtensionData::new("root-1"); + + session_data.insert(/*value*/ 17_u32); + thread_data.insert("thread value".to_string()); + + assert_eq!(session_data.level_id(), "root-1"); + assert_eq!(thread_data.level_id(), "root-1"); + assert_eq!(session_data.get::().as_deref(), Some(&17)); + assert_eq!(session_data.get::(), None); + assert_eq!(thread_data.get::(), None); + assert_eq!( + thread_data + .get::() + .map(|value| value.as_str().to_string()), + Some("thread value".to_string()) + ); +} + +#[test] +fn store_remains_usable_after_panicking_initializer() { + let data = ExtensionData::new("turn-1"); + + let result = std::panic::catch_unwind(AssertUnwindSafe(|| { + data.get_or_init::(|| panic!("initializer failed")); + })); + + assert!(result.is_err()); + assert_eq!(*data.get_or_init(|| 99_u64), 99); +} From a71e040df51ec0e1f5523bf0ff5f1bef39858128 Mon Sep 17 00:00:00 2001 From: marksteinbrick-oai Date: Tue, 9 Jun 2026 12:27:10 -0700 Subject: [PATCH 04/47] [codex-analytics] add extensible feature thread sources (#27063) ## Why - `ThreadSource` currently defines a closed set of core-owned values - Product features also create threads for background or scheduled work - Adding every product-specific value to the core enum would require repeated `codex-rs` protocol changes - Feature-backed values let product callers provide precise attribution while preserving the existing core classifications ## What Changed - Adds `ThreadSource::Feature(String)` for app-owned thread source values - Represents all app-server v2 thread sources as scalar strings, so a feature source is supplied as `"automation"` - Persists and emits the feature's plain string label, so `"automation"` produces `thread_source="automation"` in analytics - Keeps `user`, `subagent`, and `memory_consolidation` as explicit core-owned values and regenerates the app-server schemas and TypeScript bindings ## Verification - `just write-app-server-schema` - `cargo check --workspace` - `just test -p codex-protocol feature_thread_source_serializes_as_its_app_owned_label` - `just test -p codex-app-server-protocol thread_sources_round_trip_as_scalar_labels` - `cargo test -p codex-analytics thread_initialized_event_serializes_expected_shape` - `just fmt` --- .../analytics/src/analytics_client_tests.rs | 4 +- codex-rs/analytics/src/reducer.rs | 10 ++--- .../schema/json/ClientRequest.json | 5 --- .../schema/json/ServerNotification.json | 5 --- .../codex_app_server_protocol.schemas.json | 5 --- .../codex_app_server_protocol.v2.schemas.json | 5 --- .../schema/json/v2/ThreadForkParams.json | 5 --- .../schema/json/v2/ThreadForkResponse.json | 5 --- .../schema/json/v2/ThreadListResponse.json | 5 --- .../json/v2/ThreadMetadataUpdateResponse.json | 5 --- .../schema/json/v2/ThreadReadResponse.json | 5 --- .../schema/json/v2/ThreadResumeResponse.json | 5 --- .../json/v2/ThreadRollbackResponse.json | 5 --- .../schema/json/v2/ThreadStartParams.json | 5 --- .../schema/json/v2/ThreadStartResponse.json | 5 --- .../json/v2/ThreadStartedNotification.json | 5 --- .../json/v2/ThreadUnarchiveResponse.json | 5 --- .../schema/typescript/v2/ThreadSource.ts | 2 +- .../src/protocol/v2/tests.rs | 24 +++++++++++ .../src/protocol/v2/thread_data.rs | 36 ++++++++++++++-- .../request_processors/thread_processor.rs | 4 +- codex-rs/core/src/session/review.rs | 4 +- codex-rs/core/src/session/session.rs | 6 +-- codex-rs/core/src/session/turn_context.rs | 6 +-- codex-rs/exec/src/lib.rs | 4 +- codex-rs/protocol/src/protocol.rs | 43 ++++++++++++++++--- codex-rs/state/src/extract.rs | 2 +- codex-rs/state/src/model/thread_metadata.rs | 2 +- codex-rs/state/src/runtime/threads.rs | 2 + codex-rs/thread-store/src/in_memory.rs | 6 +-- .../src/local/update_thread_metadata.rs | 2 +- .../thread-store/src/thread_metadata_sync.rs | 4 +- 32 files changed, 123 insertions(+), 113 deletions(-) diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs index 7172de6898..02779c6609 100644 --- a/codex-rs/analytics/src/analytics_client_tests.rs +++ b/codex-rs/analytics/src/analytics_client_tests.rs @@ -1374,7 +1374,7 @@ fn thread_initialized_event_serializes_expected_shape() { }, model: "gpt-5".to_string(), ephemeral: true, - thread_source: Some(ThreadSource::User), + thread_source: Some(ThreadSource::Feature("automation".to_string())), initialization_mode: ThreadInitializationMode::New, subagent_source: None, parent_thread_id: None, @@ -1407,7 +1407,7 @@ fn thread_initialized_event_serializes_expected_shape() { }, "model": "gpt-5", "ephemeral": true, - "thread_source": "user", + "thread_source": "automation", "initialization_mode": "new", "subagent_source": null, "parent_thread_id": null, diff --git a/codex-rs/analytics/src/reducer.rs b/codex-rs/analytics/src/reducer.rs index 0cceb2b46d..f8abe36e4e 100644 --- a/codex-rs/analytics/src/reducer.rs +++ b/codex-rs/analytics/src/reducer.rs @@ -1263,7 +1263,7 @@ impl AnalyticsReducer { thread_metadata.session_id.clone(), connection_state.app_server_client.clone(), connection_state.runtime.clone(), - thread_metadata.thread_source, + thread_metadata.thread_source.clone(), thread_metadata.subagent_source.clone(), thread_metadata.parent_thread_id.clone(), ), @@ -1368,7 +1368,7 @@ impl AnalyticsReducer { accepted_turn_id, app_server_client: connection_state.app_server_client.clone(), runtime: connection_state.runtime.clone(), - thread_source: thread_metadata.thread_source, + thread_source: thread_metadata.thread_source.clone(), subagent_source: thread_metadata.subagent_source.clone(), parent_thread_id: thread_metadata.parent_thread_id.clone(), num_input_images: pending_request.num_input_images, @@ -1411,7 +1411,7 @@ impl AnalyticsReducer { review_id: pending_review.review_id, app_server_client: connection_state.app_server_client.clone(), runtime: connection_state.runtime.clone(), - thread_source: thread_metadata.thread_source, + thread_source: thread_metadata.thread_source.clone(), subagent_source: thread_metadata.subagent_source.clone(), parent_thread_id: thread_metadata.parent_thread_id.clone(), subject_kind: pending_review.subject_kind, @@ -1979,7 +1979,7 @@ fn tool_item_base( item_id, app_server_client: context.connection_state.app_server_client.clone(), runtime: context.connection_state.runtime.clone(), - thread_source: thread_metadata.thread_source, + thread_source: thread_metadata.thread_source.clone(), subagent_source: thread_metadata.subagent_source.clone(), parent_thread_id: thread_metadata.parent_thread_id.clone(), tool_name, @@ -2459,7 +2459,7 @@ fn codex_turn_event_params( runtime, submission_type, ephemeral, - thread_source: thread_metadata.thread_source, + thread_source: thread_metadata.thread_source.clone(), initialization_mode: thread_metadata.initialization_mode, subagent_source: thread_metadata.subagent_source.clone(), parent_thread_id: thread_metadata.parent_thread_id.clone(), diff --git a/codex-rs/app-server-protocol/schema/json/ClientRequest.json b/codex-rs/app-server-protocol/schema/json/ClientRequest.json index c499376547..b360d4bddd 100644 --- a/codex-rs/app-server-protocol/schema/json/ClientRequest.json +++ b/codex-rs/app-server-protocol/schema/json/ClientRequest.json @@ -3847,11 +3847,6 @@ "type": "string" }, "ThreadSource": { - "enum": [ - "user", - "subagent", - "memory_consolidation" - ], "type": "string" }, "ThreadSourceKind": { diff --git a/codex-rs/app-server-protocol/schema/json/ServerNotification.json b/codex-rs/app-server-protocol/schema/json/ServerNotification.json index 01f6f58b09..e0651d1727 100644 --- a/codex-rs/app-server-protocol/schema/json/ServerNotification.json +++ b/codex-rs/app-server-protocol/schema/json/ServerNotification.json @@ -4615,11 +4615,6 @@ "type": "object" }, "ThreadSource": { - "enum": [ - "user", - "subagent", - "memory_consolidation" - ], "type": "string" }, "ThreadStartedNotification": { diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index 475154b38a..d65c043a7d 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -18109,11 +18109,6 @@ "type": "string" }, "ThreadSource": { - "enum": [ - "user", - "subagent", - "memory_consolidation" - ], "type": "string" }, "ThreadSourceKind": { diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index 8ed40282a9..8271dc485d 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -15926,11 +15926,6 @@ "type": "string" }, "ThreadSource": { - "enum": [ - "user", - "subagent", - "memory_consolidation" - ], "type": "string" }, "ThreadSourceKind": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkParams.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkParams.json index d9a543e939..e72522a26a 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkParams.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkParams.json @@ -73,11 +73,6 @@ "type": "string" }, "ThreadSource": { - "enum": [ - "user", - "subagent", - "memory_consolidation" - ], "type": "string" } }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json index 0cea90749b..fa1277b0ef 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json @@ -1830,11 +1830,6 @@ ] }, "ThreadSource": { - "enum": [ - "user", - "subagent", - "memory_consolidation" - ], "type": "string" }, "ThreadStatus": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json index d0b3351a4e..1b53d4ca2e 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json @@ -1645,11 +1645,6 @@ ] }, "ThreadSource": { - "enum": [ - "user", - "subagent", - "memory_consolidation" - ], "type": "string" }, "ThreadStatus": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json index e2850b3dd9..c98fa1f2a5 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json @@ -1645,11 +1645,6 @@ ] }, "ThreadSource": { - "enum": [ - "user", - "subagent", - "memory_consolidation" - ], "type": "string" }, "ThreadStatus": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json index 01da3165b1..345d84d794 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json @@ -1645,11 +1645,6 @@ ] }, "ThreadSource": { - "enum": [ - "user", - "subagent", - "memory_consolidation" - ], "type": "string" }, "ThreadStatus": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json index d766d7a024..98ef79517b 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json @@ -1830,11 +1830,6 @@ ] }, "ThreadSource": { - "enum": [ - "user", - "subagent", - "memory_consolidation" - ], "type": "string" }, "ThreadStatus": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json index 6448ca6517..f63dfe6ffb 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json @@ -1645,11 +1645,6 @@ ] }, "ThreadSource": { - "enum": [ - "user", - "subagent", - "memory_consolidation" - ], "type": "string" }, "ThreadStatus": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartParams.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartParams.json index 7f0d4ad9a4..57e6e0ab1b 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartParams.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartParams.json @@ -159,11 +159,6 @@ "type": "object" }, "ThreadSource": { - "enum": [ - "user", - "subagent", - "memory_consolidation" - ], "type": "string" }, "ThreadStartSource": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json index a19a40f522..052023361a 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json @@ -1830,11 +1830,6 @@ ] }, "ThreadSource": { - "enum": [ - "user", - "subagent", - "memory_consolidation" - ], "type": "string" }, "ThreadStatus": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json index 0eb79075f6..cdeda64b7c 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json @@ -1645,11 +1645,6 @@ ] }, "ThreadSource": { - "enum": [ - "user", - "subagent", - "memory_consolidation" - ], "type": "string" }, "ThreadStatus": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json index 71064d5ea2..2a7281fcbb 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json @@ -1645,11 +1645,6 @@ ] }, "ThreadSource": { - "enum": [ - "user", - "subagent", - "memory_consolidation" - ], "type": "string" }, "ThreadStatus": { diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadSource.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadSource.ts index 8f55524801..f27154ab6e 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadSource.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadSource.ts @@ -2,4 +2,4 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. -export type ThreadSource = "user" | "subagent" | "memory_consolidation"; +export type ThreadSource = string; diff --git a/codex-rs/app-server-protocol/src/protocol/v2/tests.rs b/codex-rs/app-server-protocol/src/protocol/v2/tests.rs index 91bfbcc091..c570b87ce4 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/tests.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/tests.rs @@ -57,6 +57,30 @@ fn test_absolute_path() -> AbsolutePathBuf { absolute_path("readable") } +#[test] +fn thread_sources_round_trip_as_scalar_labels() { + for (source, label) in [ + (ThreadSource::User, "user"), + (ThreadSource::Subagent, "subagent"), + ( + ThreadSource::Feature("automation".to_string()), + "automation", + ), + (ThreadSource::MemoryConsolidation, "memory_consolidation"), + ] { + let value = serde_json::to_value(&source).expect("serialize thread source"); + + assert_eq!(value, json!(label)); + assert_eq!( + serde_json::from_value::(value).expect("deserialize thread source"), + source + ); + + let core_source: codex_protocol::protocol::ThreadSource = source.clone().into(); + assert_eq!(ThreadSource::from(core_source), source); + } +} + #[test] fn approvals_reviewer_serializes_auto_review_and_accepts_legacy_guardian_subagent() { assert_eq!( diff --git a/codex-rs/app-server-protocol/src/protocol/v2/thread_data.rs b/codex-rs/app-server-protocol/src/protocol/v2/thread_data.rs index 35b6183871..fc0762361f 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/thread_data.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/thread_data.rs @@ -7,6 +7,8 @@ use codex_protocol::protocol::SubAgentSource as CoreSubAgentSource; use codex_protocol::protocol::ThreadSource as CoreThreadSource; use codex_utils_absolute_path::AbsolutePathBuf; use schemars::JsonSchema; +use schemars::r#gen::SchemaGenerator; +use schemars::schema::Schema; use serde::Deserialize; use serde::Serialize; use std::path::PathBuf; @@ -61,20 +63,47 @@ impl From for CoreSessionSource { } } -#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, JsonSchema, TS)] -#[serde(rename_all = "snake_case")] -#[ts(rename_all = "snake_case", export_to = "v2/")] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, TS)] +#[serde(try_from = "String", into = "String")] +#[ts(type = "string")] +#[ts(export_to = "v2/")] pub enum ThreadSource { User, Subagent, + Feature(String), MemoryConsolidation, } +impl JsonSchema for ThreadSource { + fn schema_name() -> String { + "ThreadSource".to_string() + } + + fn json_schema(generator: &mut SchemaGenerator) -> Schema { + String::json_schema(generator) + } +} + +impl TryFrom for ThreadSource { + type Error = String; + + fn try_from(value: String) -> Result { + value.parse::().map(Into::into) + } +} + +impl From for String { + fn from(value: ThreadSource) -> Self { + CoreThreadSource::from(value).into() + } +} + impl From for ThreadSource { fn from(value: CoreThreadSource) -> Self { match value { CoreThreadSource::User => ThreadSource::User, CoreThreadSource::Subagent => ThreadSource::Subagent, + CoreThreadSource::Feature(feature) => ThreadSource::Feature(feature), CoreThreadSource::MemoryConsolidation => ThreadSource::MemoryConsolidation, } } @@ -85,6 +114,7 @@ impl From for CoreThreadSource { match value { ThreadSource::User => CoreThreadSource::User, ThreadSource::Subagent => CoreThreadSource::Subagent, + ThreadSource::Feature(feature) => CoreThreadSource::Feature(feature), ThreadSource::MemoryConsolidation => CoreThreadSource::MemoryConsolidation, } } diff --git a/codex-rs/app-server/src/request_processors/thread_processor.rs b/codex-rs/app-server/src/request_processors/thread_processor.rs index be5cffb49a..fd6432531d 100644 --- a/codex-rs/app-server/src/request_processors/thread_processor.rs +++ b/codex-rs/app-server/src/request_processors/thread_processor.rs @@ -4154,7 +4154,7 @@ fn summary_from_thread_metadata(metadata: &ThreadMetadata) -> ConversationSummar metadata.cwd.clone(), metadata.cli_version.clone(), metadata.source.clone(), - metadata.thread_source, + metadata.thread_source.clone(), metadata.agent_nickname.clone(), metadata.agent_role.clone(), metadata.git_sha.clone(), @@ -4243,7 +4243,7 @@ fn build_thread_from_snapshot( agent_nickname: config_snapshot.session_source.get_nickname(), agent_role: config_snapshot.session_source.get_agent_role(), source: config_snapshot.session_source.clone().into(), - thread_source: config_snapshot.thread_source.map(Into::into), + thread_source: config_snapshot.thread_source.clone().map(Into::into), git_info: None, name: None, turns: Vec::new(), diff --git a/codex-rs/core/src/session/review.rs b/codex-rs/core/src/session/review.rs index a3b23c6a78..00a89b2b5c 100644 --- a/codex-rs/core/src/session/review.rs +++ b/codex-rs/core/src/session/review.rs @@ -91,7 +91,7 @@ pub(super) async fn spawn_review_thread( forked_from_thread_id, parent_turn_context.parent_thread_id, &session_source, - parent_turn_context.thread_source, + parent_turn_context.thread_source.clone(), review_turn_id.clone(), #[allow(deprecated)] parent_turn_context.cwd.clone(), @@ -121,7 +121,7 @@ pub(super) async fn spawn_review_thread( reasoning_summary, session_source, parent_thread_id: parent_turn_context.parent_thread_id, - thread_source: parent_turn_context.thread_source, + thread_source: parent_turn_context.thread_source.clone(), environments: parent_turn_context.environments.clone(), available_models, unified_exec_shell_mode, diff --git a/codex-rs/core/src/session/session.rs b/codex-rs/core/src/session/session.rs index e51a83113b..50ff984231 100644 --- a/codex-rs/core/src/session/session.rs +++ b/codex-rs/core/src/session/session.rs @@ -193,7 +193,7 @@ impl SessionConfiguration { session_source: self.session_source.clone(), forked_from_thread_id: self.forked_from_thread_id, parent_thread_id: self.parent_thread_id, - thread_source: self.thread_source, + thread_source: self.thread_source.clone(), } } @@ -549,7 +549,7 @@ impl Session { forked_from_id, parent_thread_id, source: session_source, - thread_source: session_configuration.thread_source, + thread_source: session_configuration.thread_source.clone(), base_instructions: BaseInstructions { text: session_configuration.base_instructions.clone(), }, @@ -1089,7 +1089,7 @@ impl Session { thread_id, forked_from_id, parent_thread_id, - thread_source: session_configuration.thread_source, + thread_source: session_configuration.thread_source.clone(), thread_name: session_configuration.thread_name.clone(), model: session_configuration.collaboration_mode.model().to_string(), model_provider_id: config.model_provider_id.clone(), diff --git a/codex-rs/core/src/session/turn_context.rs b/codex-rs/core/src/session/turn_context.rs index be6e316a00..e3bd01081a 100644 --- a/codex-rs/core/src/session/turn_context.rs +++ b/codex-rs/core/src/session/turn_context.rs @@ -239,7 +239,7 @@ impl TurnContext { reasoning_summary: self.reasoning_summary, session_source: self.session_source.clone(), parent_thread_id: self.parent_thread_id, - thread_source: self.thread_source, + thread_source: self.thread_source.clone(), environments: self.environments.clone(), #[allow(deprecated)] cwd: self.cwd.clone(), @@ -514,7 +514,7 @@ impl Session { session_configuration.forked_from_thread_id, session_configuration.parent_thread_id, &session_configuration.session_source, - session_configuration.thread_source, + session_configuration.thread_source.clone(), sub_id.clone(), cwd.clone(), &session_configuration.permission_profile(), @@ -538,7 +538,7 @@ impl Session { reasoning_summary, session_source, parent_thread_id: session_configuration.parent_thread_id, - thread_source: session_configuration.thread_source, + thread_source: session_configuration.thread_source.clone(), environments, #[allow(deprecated)] cwd, diff --git a/codex-rs/exec/src/lib.rs b/codex-rs/exec/src/lib.rs index b8b87a85ec..fbecfe5620 100644 --- a/codex-rs/exec/src/lib.rs +++ b/codex-rs/exec/src/lib.rs @@ -1146,7 +1146,7 @@ fn session_configured_from_thread_start_response( &response.thread.session_id, &response.thread.id, response.thread.parent_thread_id.as_deref(), - response.thread.thread_source.map(Into::into), + response.thread.thread_source.clone().map(Into::into), response.thread.name.clone(), response.thread.path.clone(), response.model.clone(), @@ -1169,7 +1169,7 @@ fn session_configured_from_thread_resume_response( &response.thread.session_id, &response.thread.id, response.thread.parent_thread_id.as_deref(), - response.thread.thread_source.map(Into::into), + response.thread.thread_source.clone().map(Into::into), response.thread.name.clone(), response.thread.path.clone(), response.model.clone(), diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index b141298288..5abc9c6399 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -2480,12 +2480,12 @@ impl InitialHistory { pub fn get_resumed_session_sources(&self) -> Option<(SessionSource, Option)> { let meta = self.get_resumed_session_meta()?; - Some((meta.source.clone(), meta.thread_source)) + Some((meta.source.clone(), meta.thread_source.clone())) } pub fn get_resumed_thread_source(&self) -> Option { self.get_resumed_session_meta() - .and_then(|meta| meta.thread_source) + .and_then(|meta| meta.thread_source.clone()) } pub fn get_resumed_parent_thread_id(&self) -> Option { @@ -2529,20 +2529,23 @@ pub enum SessionSource { Unknown, } -#[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq, JsonSchema, TS)] -#[serde(rename_all = "snake_case")] -#[ts(rename_all = "snake_case")] +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, JsonSchema, TS)] +#[serde(try_from = "String", into = "String")] +#[schemars(with = "String")] +#[ts(type = "string")] pub enum ThreadSource { User, Subagent, + Feature(String), MemoryConsolidation, } impl ThreadSource { - pub fn as_str(self) -> &'static str { + pub fn as_str(&self) -> &str { match self { ThreadSource::User => "user", ThreadSource::Subagent => "subagent", + ThreadSource::Feature(feature) => feature, ThreadSource::MemoryConsolidation => "memory_consolidation", } } @@ -2554,6 +2557,20 @@ impl fmt::Display for ThreadSource { } } +impl TryFrom for ThreadSource { + type Error = String; + + fn try_from(value: String) -> Result { + value.parse() + } +} + +impl From for String { + fn from(value: ThreadSource) -> Self { + value.to_string() + } +} + impl FromStr for ThreadSource { type Err = String; @@ -2562,7 +2579,7 @@ impl FromStr for ThreadSource { "user" => Ok(ThreadSource::User), "subagent" => Ok(ThreadSource::Subagent), "memory_consolidation" => Ok(ThreadSource::MemoryConsolidation), - other => Err(format!("unknown thread source: {other}")), + other => Ok(ThreadSource::Feature(other.to_string())), } } } @@ -4048,6 +4065,18 @@ mod tests { use tempfile::NamedTempFile; use tempfile::TempDir; + #[test] + fn feature_thread_source_serializes_as_its_app_owned_label() -> Result<()> { + let source = ThreadSource::Feature("automation".to_string()); + + assert_eq!(serde_json::to_value(&source)?, json!("automation")); + assert_eq!( + serde_json::from_value::(json!("automation"))?, + source + ); + Ok(()) + } + fn sorted_writable_roots(roots: Vec) -> Vec<(PathBuf, Vec)> { let mut sorted_roots: Vec<(PathBuf, Vec)> = roots .into_iter() diff --git a/codex-rs/state/src/extract.rs b/codex-rs/state/src/extract.rs index 63cb06fea1..47285c5cf5 100644 --- a/codex-rs/state/src/extract.rs +++ b/codex-rs/state/src/extract.rs @@ -50,7 +50,7 @@ fn apply_session_meta_from_item(metadata: &mut ThreadMetadata, meta_line: &Sessi } metadata.id = meta_line.meta.id; metadata.source = enum_to_string(&meta_line.meta.source); - metadata.thread_source = meta_line.meta.thread_source; + metadata.thread_source = meta_line.meta.thread_source.clone(); metadata.agent_nickname = meta_line.meta.agent_nickname.clone(); metadata.agent_role = meta_line.meta.agent_role.clone(); metadata.agent_path = meta_line.meta.agent_path.clone(); diff --git a/codex-rs/state/src/model/thread_metadata.rs b/codex-rs/state/src/model/thread_metadata.rs index cfcd49d5f9..a30f7e479b 100644 --- a/codex-rs/state/src/model/thread_metadata.rs +++ b/codex-rs/state/src/model/thread_metadata.rs @@ -196,7 +196,7 @@ impl ThreadMetadataBuilder { created_at, updated_at, source, - thread_source: self.thread_source, + thread_source: self.thread_source.clone(), agent_nickname: self.agent_nickname.clone(), agent_role: self.agent_role.clone(), agent_path: self diff --git a/codex-rs/state/src/runtime/threads.rs b/codex-rs/state/src/runtime/threads.rs index 4cc7ee4dc1..6f3c45f0e2 100644 --- a/codex-rs/state/src/runtime/threads.rs +++ b/codex-rs/state/src/runtime/threads.rs @@ -521,6 +521,7 @@ ON CONFLICT(id) DO NOTHING .bind( metadata .thread_source + .as_ref() .map(codex_protocol::protocol::ThreadSource::as_str), ) .bind(metadata.agent_nickname.as_deref()) @@ -753,6 +754,7 @@ ON CONFLICT(id) DO UPDATE SET .bind( metadata .thread_source + .as_ref() .map(codex_protocol::protocol::ThreadSource::as_str), ) .bind(metadata.agent_nickname.as_deref()) diff --git a/codex-rs/thread-store/src/in_memory.rs b/codex-rs/thread-store/src/in_memory.rs index beed1d18fe..2ab5103663 100644 --- a/codex-rs/thread-store/src/in_memory.rs +++ b/codex-rs/thread-store/src/in_memory.rs @@ -177,7 +177,7 @@ impl ThreadStore for InMemoryThreadStore { agent_role: params.source.get_agent_role(), agent_path: params.source.get_agent_path().map(Into::into), source: params.source.clone(), - thread_source: params.thread_source, + thread_source: params.thread_source.clone(), model_provider: Some(params.metadata.model_provider.clone()), base_instructions: Some(params.base_instructions.clone()), dynamic_tools: (!params.dynamic_tools.is_empty()).then(|| params.dynamic_tools.clone()), @@ -392,8 +392,8 @@ fn stored_thread_from_state( .and_then(|metadata| metadata.source.clone()) .unwrap_or_else(|| created.source.clone()), thread_source: metadata - .and_then(|metadata| metadata.thread_source) - .unwrap_or(created.thread_source), + .and_then(|metadata| metadata.thread_source.clone()) + .unwrap_or_else(|| created.thread_source.clone()), agent_nickname: metadata.and_then(|metadata| metadata.agent_nickname.clone().flatten()), agent_role: metadata.and_then(|metadata| metadata.agent_role.clone().flatten()), agent_path: metadata.and_then(|metadata| metadata.agent_path.clone().flatten()), diff --git a/codex-rs/thread-store/src/local/update_thread_metadata.rs b/codex-rs/thread-store/src/local/update_thread_metadata.rs index dc616e4da2..7754b1a841 100644 --- a/codex-rs/thread-store/src/local/update_thread_metadata.rs +++ b/codex-rs/thread-store/src/local/update_thread_metadata.rs @@ -221,7 +221,7 @@ async fn apply_metadata_update( patch.source.clone().unwrap_or(SessionSource::Unknown), ); builder.model_provider = patch.model_provider.clone(); - builder.thread_source = patch.thread_source.flatten(); + builder.thread_source = patch.thread_source.clone().flatten(); builder.agent_nickname = patch.agent_nickname.clone().flatten(); builder.agent_role = patch.agent_role.clone().flatten(); builder.agent_path = patch.agent_path.clone().flatten(); diff --git a/codex-rs/thread-store/src/thread_metadata_sync.rs b/codex-rs/thread-store/src/thread_metadata_sync.rs index 4fb9f398c1..3db827e3a2 100644 --- a/codex-rs/thread-store/src/thread_metadata_sync.rs +++ b/codex-rs/thread-store/src/thread_metadata_sync.rs @@ -66,7 +66,7 @@ impl ThreadMetadataSync { created_at: Some(created_at), updated_at: Some(created_at), source: Some(params.source.clone()), - thread_source: Some(params.thread_source), + thread_source: Some(params.thread_source.clone()), agent_nickname: Some(params.source.get_nickname()), agent_role: Some(params.source.get_agent_role()), agent_path: Some(params.source.get_agent_path().map(Into::into)), @@ -201,7 +201,7 @@ impl ThreadMetadataSync { RolloutItem::SessionMeta(meta_line) if meta_line.meta.id == self.thread_id => { update.created_at = parse_session_timestamp(meta_line.meta.timestamp.as_str()); update.source = Some(meta_line.meta.source.clone()); - update.thread_source = Some(meta_line.meta.thread_source); + update.thread_source = Some(meta_line.meta.thread_source.clone()); update.agent_nickname = Some(meta_line.meta.agent_nickname.clone()); update.agent_role = Some(meta_line.meta.agent_role.clone()); update.agent_path = Some(meta_line.meta.agent_path.clone()); From 6e7ab529297ac3e8ef9110718e1eab688d4391ef Mon Sep 17 00:00:00 2001 From: "Adam Perry @ OpenAI" Date: Tue, 9 Jun 2026 12:48:04 -0700 Subject: [PATCH 05/47] [codex] Speed up local nextest runs (#26479) ## Why `just test` currently uses the CI-oriented nextest profile, which serializes app-server integration tests even on developer machines that can run several safely. Bounded local parallelism substantially shortens this common iteration loop without changing CI behavior. Eight-worker experiments were faster, but keeping them reliable required relaxing several test deadlines. Four workers for integration tests is a solid tradeoff that speeds up local testing without needing to change test logic. ## What changed - Add a `local` nextest profile that inherits the existing defaults. - Allow up to four app-server integration tests to run concurrently under that profile. - Make `just test` select the local profile on Unix and Windows. - Keep the default CI profile serialized and leave all test deadlines unchanged. The tests use separate processes, randomized temporary `CODEX_HOME` directories, and ephemeral ports. The remaining shared constraints are system resources; each app-server also uses a multi-thread Tokio runtime, and fuzzy-search tests can create additional worker threads, so the local cap remains intentionally conservative. ## Performance and validation All measurements below are warm, execution-only app-server runs with nextest retries disabled. On the current rebased branch, an AMD EPYC 7763 machine with 16 logical CPUs and 62 GiB RAM completed three consecutive runs: | Run | Nextest time | Wall time | Result | | --- | ---: | ---: | --- | | 1 | 142.941s | 145.17s | 836/836 passed | | 2 | 143.402s | 145.59s | 836/836 passed | | 3 | 142.870s | 145.08s | 836/836 passed | The mean wall time was 145.28s. The slow-inventory, approval replay, and zsh-fork tests all passed with their original deadlines. Earlier measurements on the same Linux machine, before the suite grew, showed the scaling that motivated the change: | App-server concurrency | Nextest time | Result | | --- | ---: | --- | | 1 | 369.5s | 572/572 passed | | 2 | 194.5s | 572/572 passed | | 4 | 111.0s mean over 3 runs | 3/3 clean | Four workers reduced that execution time by about 70%, a roughly 3.3x speedup over serialization. --- codex-rs/.config/nextest.toml | 14 ++++++++++++++ justfile | 4 ++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/codex-rs/.config/nextest.toml b/codex-rs/.config/nextest.toml index 01f4a98dc8..f3f3ca2607 100644 --- a/codex-rs/.config/nextest.toml +++ b/codex-rs/.config/nextest.toml @@ -8,12 +8,20 @@ retries = 1 [profile.default.junit] path = "junit.xml" +[profile.local] +inherits = "default" + [test-groups.app_server_protocol_codegen] max-threads = 1 [test-groups.app_server_integration] max-threads = 1 +# Higher concurrency causes integration test timeouts under resource contention +# on common developer machines. +[test-groups.app_server_integration_local] +max-threads = 4 + [test-groups.core_apply_patch_cli_integration] max-threads = 1 @@ -42,6 +50,12 @@ test-group = 'app_server_protocol_codegen' filter = 'package(codex-app-server) & kind(test)' test-group = 'app_server_integration' +[[profile.local.overrides]] +# Use up to four app-server subprocesses locally. The global nextest pool still +# limits this to the machine's logical CPU count. +filter = 'package(codex-app-server) & kind(test)' +test-group = 'app_server_integration_local' + [[profile.default.overrides]] # These tests exercise full Codex turns and apply_patch execution, and they are # sensitive to Windows runner process-startup stalls when many cases launch at once. diff --git a/justfile b/justfile index 3b993b767d..fa32a19e7d 100644 --- a/justfile +++ b/justfile @@ -75,11 +75,11 @@ install: # there should be no need to add `--all-features`. [unix] test *args: - RUST_MIN_STACK={{ rust_min_stack }} cargo nextest run --no-fail-fast "$@" + RUST_MIN_STACK={{ rust_min_stack }} NEXTEST_PROFILE=local cargo nextest run --no-fail-fast "$@" [windows] test *args: - $env:RUST_MIN_STACK = "{{ rust_min_stack }}"; cargo nextest run --no-fail-fast @($args | Select-Object -Skip 1) + $env:RUST_MIN_STACK = "{{ rust_min_stack }}"; $env:NEXTEST_PROFILE = "local"; cargo nextest run --no-fail-fast @($args | Select-Object -Skip 1) # Run from the repository root so scripts that resolve paths from `cwd` see # the same layout they use in GitHub Actions. From 7a7cee1be4e82733f393c95180242d11b50064d5 Mon Sep 17 00:00:00 2001 From: Eric Ning Date: Tue, 9 Jun 2026 12:52:05 -0700 Subject: [PATCH 06/47] fix: use plugin service route for remote uninstall (#27223) --- .../tests/suite/v2/plugin_uninstall.rs | 18 +++++++++--------- codex-rs/core-plugins/src/remote.rs | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/codex-rs/app-server/tests/suite/v2/plugin_uninstall.rs b/codex-rs/app-server/tests/suite/v2/plugin_uninstall.rs index 46299d1186..5d31d8390f 100644 --- a/codex-rs/app-server/tests/suite/v2/plugin_uninstall.rs +++ b/codex-rs/app-server/tests/suite/v2/plugin_uninstall.rs @@ -206,7 +206,7 @@ async fn plugin_uninstall_writes_remote_plugin_to_cloud_when_remote_plugin_enabl Mock::given(method("POST")) .and(path(format!( - "/backend-api/plugins/{REMOTE_PLUGIN_ID}/uninstall" + "/backend-api/ps/plugins/{REMOTE_PLUGIN_ID}/uninstall" ))) .and(header("authorization", "Bearer chatgpt-token")) .and(header("chatgpt-account-id", "account-123")) @@ -249,7 +249,7 @@ async fn plugin_uninstall_writes_remote_plugin_to_cloud_when_remote_plugin_enabl wait_for_remote_plugin_request_count( &server, "POST", - &format!("/plugins/{REMOTE_PLUGIN_ID}/uninstall"), + &format!("/ps/plugins/{REMOTE_PLUGIN_ID}/uninstall"), /*expected_count*/ 1, ) .await?; @@ -278,7 +278,7 @@ async fn plugin_uninstall_uses_detail_scope_for_cache_namespace() -> Result<()> Mock::given(method("POST")) .and(path(format!( - "/backend-api/plugins/{REMOTE_PLUGIN_ID}/uninstall" + "/backend-api/ps/plugins/{REMOTE_PLUGIN_ID}/uninstall" ))) .and(header("authorization", "Bearer chatgpt-token")) .and(header("chatgpt-account-id", "account-123")) @@ -321,7 +321,7 @@ async fn plugin_uninstall_uses_detail_scope_for_cache_namespace() -> Result<()> wait_for_remote_plugin_request_count( &server, "POST", - &format!("/plugins/{REMOTE_PLUGIN_ID}/uninstall"), + &format!("/ps/plugins/{REMOTE_PLUGIN_ID}/uninstall"), /*expected_count*/ 1, ) .await?; @@ -357,7 +357,7 @@ async fn plugin_uninstall_accepts_workspace_remote_plugin_id_shape() -> Result<( Mock::given(method("POST")) .and(path(format!( - "/backend-api/plugins/{WORKSPACE_REMOTE_PLUGIN_ID}/uninstall" + "/backend-api/ps/plugins/{WORKSPACE_REMOTE_PLUGIN_ID}/uninstall" ))) .and(header("authorization", "Bearer chatgpt-token")) .and(header("chatgpt-account-id", "account-123")) @@ -395,7 +395,7 @@ async fn plugin_uninstall_accepts_workspace_remote_plugin_id_shape() -> Result<( wait_for_remote_plugin_request_count( &server, "POST", - &format!("/plugins/{WORKSPACE_REMOTE_PLUGIN_ID}/uninstall"), + &format!("/ps/plugins/{WORKSPACE_REMOTE_PLUGIN_ID}/uninstall"), /*expected_count*/ 1, ) .await?; @@ -451,7 +451,7 @@ async fn plugin_uninstall_rejects_before_post_when_remote_detail_fetch_fails() - wait_for_remote_plugin_request_count( &server, "POST", - &format!("/plugins/{REMOTE_PLUGIN_ID}/uninstall"), + &format!("/ps/plugins/{REMOTE_PLUGIN_ID}/uninstall"), /*expected_count*/ 0, ) .await?; @@ -487,7 +487,7 @@ async fn plugin_uninstall_rejects_remote_plugin_id_with_spaces_before_network_ca wait_for_remote_plugin_request_count( &server, "POST", - "/plugins/sample plugin/uninstall", + "/ps/plugins/sample plugin/uninstall", /*expected_count*/ 0, ) .await?; @@ -522,7 +522,7 @@ async fn plugin_uninstall_rejects_invalid_remote_plugin_id_before_network_call() wait_for_remote_plugin_request_count( &server, "POST", - "/plugins/linear/../../oops/uninstall", + "/ps/plugins/linear/../../oops/uninstall", /*expected_count*/ 0, ) .await?; diff --git a/codex-rs/core-plugins/src/remote.rs b/codex-rs/core-plugins/src/remote.rs index 1885318727..2beeed2a1b 100644 --- a/codex-rs/core-plugins/src/remote.rs +++ b/codex-rs/core-plugins/src/remote.rs @@ -1124,7 +1124,7 @@ pub async fn uninstall_remote_plugin( let plugin_name = plugin.name; let base_url = config.chatgpt_base_url.trim_end_matches('/'); - let url = format!("{base_url}/plugins/{plugin_id}/uninstall"); + let url = format!("{base_url}/ps/plugins/{plugin_id}/uninstall"); let client = build_reqwest_client(); let request = authenticated_request(client.post(&url), auth)?; let response: RemotePluginMutationResponse = send_and_decode(request, &url).await?; From 472619f9fc6739d5c4fa499b3ef10007655b031d Mon Sep 17 00:00:00 2001 From: Winston Howes Date: Tue, 9 Jun 2026 13:08:01 -0700 Subject: [PATCH 07/47] Add SOCKS5 TCP MITM coverage (#22685) ## Summary - reuse the MITM HTTPS serving path for raw SOCKS5 TCP streams - route limited-mode and hooked SOCKS5 TCP requests through MITM before dialing upstream - keep SOCKS5 UDP limited-mode behavior unchanged ## Validation - `just fmt` - `just test -p codex-network-proxy` - `just fix -p codex-network-proxy` - `git diff --check` --- codex-rs/network-proxy/README.md | 9 +- codex-rs/network-proxy/src/config.rs | 2 +- codex-rs/network-proxy/src/mitm.rs | 26 +- codex-rs/network-proxy/src/socks5.rs | 510 +++++++++++++++++++++------ 4 files changed, 419 insertions(+), 128 deletions(-) diff --git a/codex-rs/network-proxy/README.md b/codex-rs/network-proxy/README.md index 300cd3fcb0..d5d0730227 100644 --- a/codex-rs/network-proxy/README.md +++ b/codex-rs/network-proxy/README.md @@ -107,9 +107,9 @@ When a request is blocked, the proxy responds with `403` and includes: - `blocked-by-method-policy` - `blocked-by-policy` -In "limited" mode, only `GET`, `HEAD`, and `OPTIONS` are allowed. HTTPS `CONNECT` requests require -MITM to enforce limited-mode method policy; otherwise they are blocked. SOCKS5 remains blocked in -limited mode. +In "limited" mode, only `GET`, `HEAD`, and `OPTIONS` are allowed. HTTPS `CONNECT` requests and +HTTPS SOCKS5 TCP targets on `:443` require MITM to enforce limited-mode method policy; otherwise +they are blocked. SOCKS5 UDP and non-HTTPS SOCKS5 TCP remain blocked in limited mode. Websocket clients typically tunnel `wss://` through HTTPS `CONNECT`; those CONNECT targets still go through the same host allowlist/denylist checks. @@ -215,7 +215,8 @@ what it can reasonably guarantee. allowlisted (best-effort DNS lookup). - Limited mode enforcement: - only `GET`, `HEAD`, and `OPTIONS` are allowed - - HTTPS `CONNECT` remains a tunnel; limited-mode method enforcement does not apply to HTTPS + - HTTPS `CONNECT` requests and HTTPS SOCKS5 TCP targets on `:443` require MITM so the proxy can + enforce limited-mode method policy; SOCKS5 UDP and non-HTTPS SOCKS5 TCP remain blocked - Listener safety defaults: - the HTTP proxy listener clamps non-loopback binds unless explicitly enabled via `dangerously_allow_non_loopback_proxy` diff --git a/codex-rs/network-proxy/src/config.rs b/codex-rs/network-proxy/src/config.rs index 0b0a8233f5..d9cef46b15 100644 --- a/codex-rs/network-proxy/src/config.rs +++ b/codex-rs/network-proxy/src/config.rs @@ -276,7 +276,7 @@ impl NetworkProxySettings { pub enum NetworkMode { /// Limited (read-only) access: only GET/HEAD/OPTIONS are allowed for HTTP. HTTPS CONNECT is /// blocked unless MITM is enabled so the proxy can enforce method policy on inner requests. - /// SOCKS5 remains blocked in limited mode. + /// SOCKS5 UDP and non-HTTPS SOCKS5 TCP remain blocked in limited mode. Limited, /// Full network access: all HTTP methods are allowed. HTTPS CONNECTs are tunneled directly. /// MITM hooks do not currently make full mode enter MITM. diff --git a/codex-rs/network-proxy/src/mitm.rs b/codex-rs/network-proxy/src/mitm.rs index 9ee5bfc253..345c5b5032 100644 --- a/codex-rs/network-proxy/src/mitm.rs +++ b/codex-rs/network-proxy/src/mitm.rs @@ -21,10 +21,12 @@ use rama_core::Layer; use rama_core::Service; use rama_core::bytes::Bytes; use rama_core::error::BoxError; +use rama_core::extensions::ExtensionsMut; use rama_core::extensions::ExtensionsRef; -use rama_core::futures::stream::Stream; +use rama_core::futures::stream::Stream as FuturesStream; use rama_core::rt::Executor; use rama_core::service::service_fn; +use rama_core::stream::Stream; use rama_http::Body; use rama_http::BodyDataStream; use rama_http::HeaderMap; @@ -135,17 +137,25 @@ impl MitmState { /// Terminate the upgraded CONNECT stream with a generated leaf cert and proxy inner HTTPS traffic. pub(crate) async fn mitm_tunnel(upgraded: Upgraded) -> Result<()> { - let mitm = upgraded + mitm_stream(upgraded).await +} + +/// Terminate a raw client stream with a generated leaf cert and proxy inner HTTPS traffic. +pub(crate) async fn mitm_stream(stream: S) -> Result<()> +where + S: Stream + Unpin + ExtensionsMut, +{ + let mitm = stream .extensions() .get::>() .cloned() .context("missing MITM state")?; - let app_state = upgraded + let app_state = stream .extensions() .get::>() .cloned() .context("missing app state")?; - let target = upgraded + let target = stream .extensions() .get::() .context("missing proxy target")? @@ -154,7 +164,7 @@ pub(crate) async fn mitm_tunnel(upgraded: Upgraded) -> Result<()> { let target_host = normalize_host(&target.host.to_string()); let target_port = target.port; let acceptor_data = mitm.tls_acceptor_data_for_host(&target_host)?; - let mode = upgraded + let mode = stream .extensions() .get::() .copied() @@ -169,7 +179,7 @@ pub(crate) async fn mitm_tunnel(upgraded: Upgraded) -> Result<()> { mitm, }); - let executor = upgraded + let executor = stream .extensions() .get::() .cloned() @@ -194,7 +204,7 @@ pub(crate) async fn mitm_tunnel(upgraded: Upgraded) -> Result<()> { .into_layer(http_service); https_service - .serve(upgraded) + .serve(stream) .await .map_err(|err| anyhow!("MITM serve error: {err}"))?; Ok(()) @@ -456,7 +466,7 @@ struct InspectStream { max_body_bytes: usize, } -impl Stream for InspectStream { +impl FuturesStream for InspectStream { type Item = Result; fn poll_next(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll> { diff --git a/codex-rs/network-proxy/src/socks5.rs b/codex-rs/network-proxy/src/socks5.rs index 8369c4da80..d9a8a559fd 100644 --- a/codex-rs/network-proxy/src/socks5.rs +++ b/codex-rs/network-proxy/src/socks5.rs @@ -1,5 +1,6 @@ use crate::config::NetworkMode; use crate::connect_policy::TargetCheckedTcpConnector; +use crate::mitm; use crate::network_policy::BlockDecisionAuditEventArgs; use crate::network_policy::NetworkDecision; use crate::network_policy::NetworkDecisionSource; @@ -24,10 +25,17 @@ use anyhow::Result; use rama_core::Layer; use rama_core::Service; use rama_core::error::BoxError; +use rama_core::extensions::Extensions; +use rama_core::extensions::ExtensionsMut; use rama_core::extensions::ExtensionsRef; use rama_core::layer::AddInputExtensionLayer; use rama_core::service::service_fn; +use rama_net::address::HostWithPort; use rama_net::client::EstablishedClientConnection; +use rama_net::proxy::ProxyRequest; +use rama_net::proxy::ProxyTarget; +use rama_net::proxy::StreamForwardService; +use rama_net::stream::Socket; use rama_net::stream::SocketInfo; use rama_socks5::Socks5Acceptor; use rama_socks5::server::DefaultConnector; @@ -40,8 +48,14 @@ use rama_tcp::server::TcpListener; use std::io; use std::net::SocketAddr; use std::net::TcpListener as StdTcpListener; +use std::pin::Pin; use std::sync::Arc; +use std::task::Context as TaskContext; +use std::task::Poll; use std::time::Instant; +use tokio::io::AsyncRead; +use tokio::io::AsyncWrite; +use tokio::io::ReadBuf; use tracing::error; use tracing::info; use tracing::warn; @@ -88,7 +102,9 @@ async fn run_socks5_with_listener( match state.network_mode().await { Ok(NetworkMode::Limited) => { - info!("SOCKS5 is blocked in limited mode; set mode=\"full\" to allow SOCKS5"); + info!( + "SOCKS5 UDP and non-HTTPS SOCKS5 TCP are blocked in limited mode; HTTPS SOCKS5 TCP requires MITM inspection" + ); } Ok(NetworkMode::Full) => {} Err(err) => { @@ -106,7 +122,10 @@ async fn run_socks5_with_listener( } }); - let socks_connector = DefaultConnector::default().with_connector(policy_tcp_connector); + let socks_proxy = service_fn(|request| async move { proxy_socks5_tcp(request).await }); + let socks_connector = DefaultConnector::default() + .with_connector(policy_tcp_connector) + .with_service(socks_proxy); let base = Socks5Acceptor::new().with_connector(socks_connector); if enable_socks5_udp { @@ -135,7 +154,7 @@ async fn handle_socks5_tcp( req: TcpRequest, tcp_connector: TargetCheckedTcpConnector, policy_decider: Option>, -) -> Result, BoxError> { +) -> Result, BoxError> { let app_state = req .extensions() .get::>() @@ -144,6 +163,7 @@ async fn handle_socks5_tcp( let host = normalize_host(&req.authority.host.to_string()); let port = req.authority.port; + let target = req.authority.clone(); if host.is_empty() { return Err(io::Error::new(io::ErrorKind::InvalidInput, "invalid host").into()); } @@ -196,94 +216,52 @@ async fn handle_socks5_tcp( } } - match app_state.network_mode().await { - Ok(NetworkMode::Limited) => { - emit_socks_block_decision_audit_event( - &app_state, - NetworkDecisionSource::ModeGuard, - REASON_METHOD_NOT_ALLOWED, - NetworkProtocol::Socks5Tcp, - host.as_str(), - port, - client.as_deref(), - ); - let details = PolicyDecisionDetails { - decision: NetworkPolicyDecision::Deny, - reason: REASON_METHOD_NOT_ALLOWED, - source: NetworkDecisionSource::ModeGuard, - protocol: NetworkProtocol::Socks5Tcp, - host: &host, - port, - }; - let _ = app_state - .record_blocked(BlockedRequest::new(BlockedRequestArgs { - host: host.clone(), - reason: REASON_METHOD_NOT_ALLOWED.to_string(), - client: client.clone(), - method: None, - mode: Some(NetworkMode::Limited), - protocol: "socks5".to_string(), - decision: Some(details.decision.as_str().to_string()), - source: Some(details.source.as_str().to_string()), - port: Some(port), - })) - .await; - let client = client.as_deref().unwrap_or_default(); - warn!( - "SOCKS blocked by method policy (client={client}, host={host}, mode=limited, allowed_methods=GET, HEAD, OPTIONS)" - ); - return Err(policy_denied_error(REASON_METHOD_NOT_ALLOWED, &details).into()); - } - Ok(NetworkMode::Full) => {} + let mode = match app_state.network_mode().await { + Ok(mode) => mode, Err(err) => { error!("failed to evaluate method policy: {err}"); return Err(io::Error::other("proxy error").into()); } - } - - match app_state.host_has_mitm_hooks(&host).await { - Ok(true) => { - emit_socks_block_decision_audit_event( - &app_state, - NetworkDecisionSource::ModeGuard, - REASON_MITM_REQUIRED, - NetworkProtocol::Socks5Tcp, - host.as_str(), - port, - client.as_deref(), - ); - let details = PolicyDecisionDetails { - decision: NetworkPolicyDecision::Deny, - reason: REASON_MITM_REQUIRED, - source: NetworkDecisionSource::ModeGuard, - protocol: NetworkProtocol::Socks5Tcp, - host: &host, - port, - }; - let _ = app_state - .record_blocked(BlockedRequest::new(BlockedRequestArgs { - host: host.clone(), - reason: REASON_MITM_REQUIRED.to_string(), - client: client.clone(), - method: None, - mode: Some(NetworkMode::Full), - protocol: "socks5".to_string(), - decision: Some(details.decision.as_str().to_string()), - source: Some(details.source.as_str().to_string()), - port: Some(port), - })) - .await; - let client = client.as_deref().unwrap_or_default(); - warn!( - "SOCKS blocked; MITM required to enforce HTTPS policy (client={client}, host={host}, mode=full)" - ); - return Err(policy_denied_error(REASON_MITM_REQUIRED, &details).into()); - } - Ok(false) => {} - Err(err) => { - error!("failed to inspect MITM hooks for {host}: {err}"); - return Err(io::Error::other("proxy error").into()); - } + }; + // SOCKS5 only exposes host and port, so only the default HTTPS port is identifiable as a + // TLS stream that the HTTPS MITM path can safely terminate. + let socks5_tcp_target_is_https = port == 443; + if mode == NetworkMode::Limited && !socks5_tcp_target_is_https { + emit_socks_block_decision_audit_event( + &app_state, + NetworkDecisionSource::ModeGuard, + REASON_METHOD_NOT_ALLOWED, + NetworkProtocol::Socks5Tcp, + host.as_str(), + port, + client.as_deref(), + ); + let details = PolicyDecisionDetails { + decision: NetworkPolicyDecision::Deny, + reason: REASON_METHOD_NOT_ALLOWED, + source: NetworkDecisionSource::ModeGuard, + protocol: NetworkProtocol::Socks5Tcp, + host: &host, + port, + }; + let _ = app_state + .record_blocked(BlockedRequest::new(BlockedRequestArgs { + host: host.clone(), + reason: REASON_METHOD_NOT_ALLOWED.to_string(), + client: client.clone(), + method: None, + mode: Some(NetworkMode::Limited), + protocol: "socks5".to_string(), + decision: Some(details.decision.as_str().to_string()), + source: Some(details.source.as_str().to_string()), + port: Some(port), + })) + .await; + let client = client.as_deref().unwrap_or_default(); + warn!( + "SOCKS blocked; limited mode only supports HTTPS MITM (client={client}, host={host}, port={port})" + ); + return Err(policy_denied_error(REASON_METHOD_NOT_ALLOWED, &details).into()); } let request = NetworkPolicyRequest::new(NetworkPolicyRequestArgs { @@ -337,9 +315,85 @@ async fn handle_socks5_tcp( } } + let host_has_mitm_hooks = match app_state.host_has_mitm_hooks(&host).await { + Ok(has_hooks) => has_hooks, + Err(err) => { + error!("failed to inspect MITM hooks for {host}: {err}"); + return Err(io::Error::other("proxy error").into()); + } + }; + let mitm_state = match app_state.mitm_state().await { + Ok(state) => state, + Err(err) => { + error!("failed to load MITM state: {err}"); + return Err(io::Error::other("proxy error").into()); + } + }; + let socks_needs_mitm = + socks5_tcp_target_is_https && (mode == NetworkMode::Limited || host_has_mitm_hooks); + if (host_has_mitm_hooks && !socks5_tcp_target_is_https) + || (socks_needs_mitm && mitm_state.is_none()) + { + emit_socks_block_decision_audit_event( + &app_state, + NetworkDecisionSource::ModeGuard, + REASON_MITM_REQUIRED, + NetworkProtocol::Socks5Tcp, + host.as_str(), + port, + client.as_deref(), + ); + let details = PolicyDecisionDetails { + decision: NetworkPolicyDecision::Deny, + reason: REASON_MITM_REQUIRED, + source: NetworkDecisionSource::ModeGuard, + protocol: NetworkProtocol::Socks5Tcp, + host: &host, + port, + }; + let _ = app_state + .record_blocked(BlockedRequest::new(BlockedRequestArgs { + host: host.clone(), + reason: REASON_MITM_REQUIRED.to_string(), + client: client.clone(), + method: None, + mode: Some(mode), + protocol: "socks5".to_string(), + decision: Some(details.decision.as_str().to_string()), + source: Some(details.source.as_str().to_string()), + port: Some(port), + })) + .await; + let client = client.as_deref().unwrap_or_default(); + warn!( + "SOCKS blocked; MITM required to enforce HTTPS policy (client={client}, host={host}, mode={mode:?}, hooked_host={host_has_mitm_hooks}, https_target={socks5_tcp_target_is_https})" + ); + return Err(policy_denied_error(REASON_MITM_REQUIRED, &details).into()); + } + + if socks_needs_mitm && let Some(mitm_state) = mitm_state { + let client = client.as_deref().unwrap_or_default(); + info!("SOCKS MITM enabled (client={client}, host={host}, port={port}, mode={mode:?})"); + return Ok(EstablishedClientConnection { + input: req, + conn: Socks5TcpConnection::Mitm { + target, + mode, + mitm: mitm_state, + extensions: Extensions::new(), + }, + }); + } + info!("SOCKS upstream dial started (host={host}, port={port})"); let connect_started_at = Instant::now(); - let result = tcp_connector.serve(req).await; + let result = tcp_connector.serve(req).await.map(|connection| { + let EstablishedClientConnection { input, conn } = connection; + EstablishedClientConnection { + input, + conn: Socks5TcpConnection::Direct(conn), + } + }); match &result { Ok(_) => info!( "SOCKS upstream dial established (host={host}, port={port}, elapsed_ms={})", @@ -353,6 +407,113 @@ async fn handle_socks5_tcp( result } +/// Internal connector output for SOCKS5 TCP. MITM requests do not dial upstream before the +/// inner HTTPS request is inspected, so they carry the target metadata instead of a socket. +#[derive(Debug)] +enum Socks5TcpConnection { + Direct(TcpStream), + Mitm { + target: HostWithPort, + mode: NetworkMode, + mitm: Arc, + extensions: Extensions, + }, +} + +impl AsyncRead for Socks5TcpConnection { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut TaskContext<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + match self.get_mut() { + Self::Direct(stream) => Pin::new(stream).poll_read(cx, buf), + Self::Mitm { .. } => Poll::Ready(Ok(())), + } + } +} + +impl AsyncWrite for Socks5TcpConnection { + fn poll_write( + self: Pin<&mut Self>, + cx: &mut TaskContext<'_>, + buf: &[u8], + ) -> Poll> { + match self.get_mut() { + Self::Direct(stream) => Pin::new(stream).poll_write(cx, buf), + Self::Mitm { .. } => Poll::Ready(Ok(buf.len())), + } + } + + fn poll_flush(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll> { + match self.get_mut() { + Self::Direct(stream) => Pin::new(stream).poll_flush(cx), + Self::Mitm { .. } => Poll::Ready(Ok(())), + } + } + + fn poll_shutdown(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll> { + match self.get_mut() { + Self::Direct(stream) => Pin::new(stream).poll_shutdown(cx), + Self::Mitm { .. } => Poll::Ready(Ok(())), + } + } +} + +impl Socket for Socks5TcpConnection { + fn local_addr(&self) -> io::Result { + match self { + Self::Direct(stream) => stream.local_addr(), + Self::Mitm { .. } => Ok(SocketAddr::from(([0, 0, 0, 0], 0))), + } + } + + fn peer_addr(&self) -> io::Result { + match self { + Self::Direct(stream) => stream.peer_addr(), + Self::Mitm { .. } => Ok(SocketAddr::from(([0, 0, 0, 0], 0))), + } + } +} + +impl ExtensionsRef for Socks5TcpConnection { + fn extensions(&self) -> &Extensions { + match self { + Self::Direct(stream) => stream.extensions(), + Self::Mitm { extensions, .. } => extensions, + } + } +} + +impl ExtensionsMut for Socks5TcpConnection { + fn extensions_mut(&mut self) -> &mut Extensions { + match self { + Self::Direct(stream) => stream.extensions_mut(), + Self::Mitm { extensions, .. } => extensions, + } + } +} + +async fn proxy_socks5_tcp( + request: ProxyRequest, +) -> Result<(), BoxError> { + let ProxyRequest { mut source, target } = request; + match target { + Socks5TcpConnection::Direct(target) => StreamForwardService::default() + .serve(ProxyRequest { source, target }) + .await + .map_err(Into::into), + Socks5TcpConnection::Mitm { + target, mode, mitm, .. + } => { + source.extensions_mut().insert(ProxyTarget(target)); + source.extensions_mut().insert(mode); + source.extensions_mut().insert(mitm); + mitm::mitm_stream(source).await.map_err(Into::into) + } + } +} + async fn inspect_socks5_udp( request: RelayRequest, state: Arc, @@ -554,7 +715,6 @@ mod tests { use crate::network_policy::test_support::find_event_by_name; use crate::runtime::ConfigReloader; use crate::runtime::ConfigState; - use crate::runtime::network_proxy_state_for_policy; use crate::state::NetworkProxyConstraints; use crate::state::build_config_state; use async_trait::async_trait; @@ -567,6 +727,11 @@ mod tests { use std::net::IpAddr; use std::net::Ipv4Addr; use std::sync::Arc; + use std::sync::Mutex; + + // Managed MITM CA files live under the shared test CODEX_HOME, so MITM-enabled config state + // must be materialized one test at a time. + static MITM_CONFIG_STATE_LOCK: Mutex<()> = Mutex::new(()); #[derive(Clone)] struct StaticReloader { @@ -590,6 +755,10 @@ mod tests { fn state_for_settings(network: NetworkProxySettings) -> Arc { let config = NetworkProxyConfig { network }; + let _mitm_config_state_guard = config + .network + .mitm + .then(|| MITM_CONFIG_STATE_LOCK.lock().unwrap()); let state = build_config_state(config, NetworkProxyConstraints::default()).unwrap(); let reloader = Arc::new(StaticReloader { state: state.clone(), @@ -639,43 +808,56 @@ mod tests { } #[tokio::test(flavor = "current_thread")] - async fn handle_socks5_tcp_blocks_hooked_host_in_full_mode() { - let state = Arc::new(network_proxy_state_for_policy(NetworkProxySettings { + async fn handle_socks5_tcp_uses_mitm_in_limited_mode() { + let mut settings = NetworkProxySettings { enabled: true, - mode: NetworkMode::Full, + mode: NetworkMode::Limited, mitm: true, - mitm_hooks: vec![MitmHookConfig { - host: "api.github.com".to_string(), - matcher: MitmHookMatchConfig { - methods: vec!["GET".to_string()], - path_prefixes: vec!["/".to_string()], - ..MitmHookMatchConfig::default() - }, - ..MitmHookConfig::default() - }], ..NetworkProxySettings::default() - })); + }; + settings.set_allowed_domains(vec!["example.com".to_string()]); + let state = state_for_settings(settings); let mut request = - TcpRequest::new(HostWithPort::try_from("api.github.com:443").expect("valid authority")); + TcpRequest::new(HostWithPort::try_from("example.com:443").expect("valid authority")); + request.extensions_mut().insert(state.clone()); + + let result = handle_socks5_tcp( + request, + TargetCheckedTcpConnector::new(state), + /*policy_decider*/ None, + ) + .await + .expect("limited-mode HTTPS should use MITM"); + + assert!(matches!(result.conn, Socks5TcpConnection::Mitm { .. })); + } + + #[tokio::test(flavor = "current_thread")] + async fn handle_socks5_tcp_blocks_non_https_in_limited_mode() { + let mut settings = NetworkProxySettings { + enabled: true, + mode: NetworkMode::Limited, + ..NetworkProxySettings::default() + }; + settings.set_allowed_domains(vec!["example.com".to_string()]); + let state = state_for_settings(settings); + let mut request = + TcpRequest::new(HostWithPort::try_from("example.com:80").expect("valid authority")); request.extensions_mut().insert(state.clone()); let (result, events) = capture_events(|| async { handle_socks5_tcp( request, - TargetCheckedTcpConnector::new(state.clone()), + TargetCheckedTcpConnector::new(state), /*policy_decider*/ None, ) .await }) .await; - assert!(result.is_err(), "hooked host should require MITM"); - - let blocked = state.drain_blocked().await.unwrap(); - assert_eq!(blocked.len(), 1); - assert_eq!(blocked[0].reason, REASON_MITM_REQUIRED); - assert_eq!(blocked[0].host, "api.github.com"); - assert_eq!(blocked[0].port, Some(443)); - assert_eq!(blocked[0].protocol, "socks5"); + assert!( + result.is_err(), + "limited-mode non-HTTPS SOCKS should be denied" + ); let event = find_event_by_name(&events, POLICY_DECISION_EVENT_NAME) .expect("expected policy decision event"); @@ -684,18 +866,116 @@ mod tests { assert_eq!(event.field("network.policy.source"), Some("mode_guard")); assert_eq!( event.field("network.policy.reason"), - Some(REASON_MITM_REQUIRED) + Some(REASON_METHOD_NOT_ALLOWED) ); assert_eq!( event.field("network.transport.protocol"), Some("socks5_tcp") ); - assert_eq!(event.field("server.address"), Some("api.github.com")); - assert_eq!(event.field("server.port"), Some("443")); + assert_eq!(event.field("server.address"), Some("example.com")); + assert_eq!(event.field("server.port"), Some("80")); assert_eq!(event.field("http.request.method"), Some("none")); assert_eq!(event.field("client.address"), Some("unknown")); } + #[tokio::test(flavor = "current_thread")] + async fn handle_socks5_tcp_blocks_limited_mode_without_mitm_state() { + let mut settings = NetworkProxySettings { + enabled: true, + mode: NetworkMode::Limited, + ..NetworkProxySettings::default() + }; + settings.set_allowed_domains(vec!["example.com".to_string()]); + let state = state_for_settings(settings); + let mut request = + TcpRequest::new(HostWithPort::try_from("example.com:443").expect("valid authority")); + request.extensions_mut().insert(state.clone()); + + let err = handle_socks5_tcp( + request, + TargetCheckedTcpConnector::new(state), + /*policy_decider*/ None, + ) + .await + .expect_err("limited-mode HTTPS requires MITM"); + + assert!( + format!("{err:?}").contains("MITM required"), + "unexpected error: {err:?}" + ); + } + + #[tokio::test(flavor = "current_thread")] + async fn handle_socks5_tcp_uses_mitm_for_hooked_host_in_full_mode() { + let mut settings = NetworkProxySettings { + enabled: true, + mode: NetworkMode::Full, + mitm: true, + mitm_hooks: vec![MitmHookConfig { + host: "api.github.com".to_string(), + matcher: MitmHookMatchConfig { + methods: vec!["POST".to_string()], + path_prefixes: vec!["/repos/openai/".to_string()], + ..MitmHookMatchConfig::default() + }, + ..MitmHookConfig::default() + }], + ..NetworkProxySettings::default() + }; + settings.set_allowed_domains(vec!["api.github.com".to_string()]); + let state = state_for_settings(settings); + let mut request = + TcpRequest::new(HostWithPort::try_from("api.github.com:443").expect("valid authority")); + request.extensions_mut().insert(state.clone()); + + let result = handle_socks5_tcp( + request, + TargetCheckedTcpConnector::new(state), + /*policy_decider*/ None, + ) + .await + .expect("hooked HTTPS should use MITM"); + + assert!(matches!(result.conn, Socks5TcpConnection::Mitm { .. })); + } + + #[tokio::test(flavor = "current_thread")] + async fn handle_socks5_tcp_blocks_hooked_non_https_host_in_full_mode() { + let mut settings = NetworkProxySettings { + enabled: true, + mode: NetworkMode::Full, + mitm: true, + mitm_hooks: vec![MitmHookConfig { + host: "api.github.com".to_string(), + matcher: MitmHookMatchConfig { + methods: vec!["POST".to_string()], + path_prefixes: vec!["/repos/openai/".to_string()], + ..MitmHookMatchConfig::default() + }, + ..MitmHookConfig::default() + }], + ..NetworkProxySettings::default() + }; + settings.set_allowed_domains(vec!["api.github.com".to_string()]); + let state = state_for_settings(settings); + let mut request = + TcpRequest::new(HostWithPort::try_from("api.github.com:80").expect("valid authority")); + request.extensions_mut().insert(state.clone()); + + let err = handle_socks5_tcp( + request, + TargetCheckedTcpConnector::new(state), + /*policy_decider*/ None, + ) + .await + .expect_err("hooked non-HTTPS SOCKS should require MITM"); + + assert!( + format!("{err:?}").contains("MITM required"), + "unexpected error: {err:?}" + ); + } + #[tokio::test(flavor = "current_thread")] async fn inspect_socks5_udp_emits_block_decision_for_mode_guard_deny() { let state = state_for_settings(NetworkProxySettings { From 36bf63a5cf02dafb1574b2396597ed1ef16bb5d7 Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Tue, 9 Jun 2026 13:23:31 -0700 Subject: [PATCH 08/47] Allow creating a new goal after completion (#26681) ## Why Users have indicated that they want an agent to be able to create a new goal for itself after completing the previous goal. Currently, that's not possible because agents cannot overwrite an existing goal even if it's complete. This PR removes this limitation and allows `create_goal` to overwrite an existing goal if it is in the `complete` state. ## What changed `create_goal` now replaces the existing goal only when its status is `complete`. The replacement is performed atomically in the goal store, creates a fresh active goal with reset usage, and continues to reject creation while any unfinished goal exists. App server clients see a single `thread/goal/updated` event when the previous goal is replaced with the new one. The tool description and error message now reflect these semantics. ## What didn't change Agents are not allowed to create a new goal (overwrite their existing goal) if an existing goal is still active, blocked, paused, or in any other state other than "completed". --- codex-rs/ext/goal/src/spec.rs | 4 +-- codex-rs/ext/goal/src/tool.rs | 2 +- .../ext/goal/tests/goal_extension_backend.rs | 25 +++++++++++++++++-- codex-rs/state/src/runtime/goals.rs | 11 +++++++- 4 files changed, 36 insertions(+), 6 deletions(-) diff --git a/codex-rs/ext/goal/src/spec.rs b/codex-rs/ext/goal/src/spec.rs index 2c92c03848..6b97a24e9c 100644 --- a/codex-rs/ext/goal/src/spec.rs +++ b/codex-rs/ext/goal/src/spec.rs @@ -27,7 +27,7 @@ pub fn create_create_goal_tool() -> ToolSpec { ( "objective".to_string(), JsonSchema::string(Some( - "Required. The concrete objective to start pursuing. This starts a new active goal only when no goal is currently defined; if a goal already exists, this tool fails." + "Required. The concrete objective to start pursuing. This starts a new active goal when no goal exists or replaces the current goal when it is complete." .to_string(), )), ), @@ -44,7 +44,7 @@ pub fn create_create_goal_tool() -> ToolSpec { name: CREATE_GOAL_TOOL_NAME.to_string(), description: format!( r#"Create a goal only when explicitly requested by the user or system/developer instructions; do not infer goals from ordinary tasks. -Set token_budget only when an explicit token budget is requested. Fails if a goal exists; use {UPDATE_GOAL_TOOL_NAME} only for status."# +Set token_budget only when an explicit token budget is requested. Fails if an unfinished goal exists; use {UPDATE_GOAL_TOOL_NAME} only for status."# ), strict: false, defer_loading: None, diff --git a/codex-rs/ext/goal/src/tool.rs b/codex-rs/ext/goal/src/tool.rs index 17947aa343..8c0f513004 100644 --- a/codex-rs/ext/goal/src/tool.rs +++ b/codex-rs/ext/goal/src/tool.rs @@ -191,7 +191,7 @@ impl GoalToolExecutor { .map_err(|err| FunctionCallError::RespondToModel(format!("failed to create goal: {err}")))? .ok_or_else(|| { FunctionCallError::RespondToModel( - "cannot create a new goal because this thread already has a goal; use update_goal only when the existing goal is complete" + "cannot create a new goal because this thread has an unfinished goal; complete the existing goal first" .to_string(), ) })?; diff --git a/codex-rs/ext/goal/tests/goal_extension_backend.rs b/codex-rs/ext/goal/tests/goal_extension_backend.rs index f89a08ebbb..b922f5332d 100644 --- a/codex-rs/ext/goal/tests/goal_extension_backend.rs +++ b/codex-rs/ext/goal/tests/goal_extension_backend.rs @@ -124,7 +124,7 @@ async fn goal_tools_hidden_for_review_subagents() -> anyhow::Result<()> { } #[tokio::test] -async fn installed_goal_tools_reject_duplicate_goal_creation() -> anyhow::Result<()> { +async fn installed_goal_tools_only_replace_complete_goal() -> anyhow::Result<()> { let runtime = test_runtime().await?; let thread_id = test_thread_id()?; seed_thread_metadata(runtime.as_ref(), thread_id).await?; @@ -152,10 +152,31 @@ async fn installed_goal_tools_reject_duplicate_goal_creation() -> anyhow::Result assert_eq!( err, FunctionCallError::RespondToModel( - "cannot create a new goal because this thread already has a goal; use update_goal only when the existing goal is complete" + "cannot create a new goal because this thread has an unfinished goal; complete the existing goal first" .to_string() ) ); + + let update_tool = tool_by_name(&tools, "update_goal"); + update_tool + .handle(tool_call( + "update_goal", + "call-complete-goal", + json!({ "status": "complete" }), + )) + .await?; + + let invocation = tool_call( + "create_goal", + "call-create-goal-3", + json!({ "objective": "replacement goal" }), + ); + let output = create_tool.handle(invocation.clone()).await?; + let result = output.code_mode_result(&invocation.payload); + + assert_eq!(json!("replacement goal"), result["goal"]["objective"]); + assert_eq!(json!("active"), result["goal"]["status"]); + assert_eq!(json!(0), result["goal"]["tokensUsed"]); Ok(()) } diff --git a/codex-rs/state/src/runtime/goals.rs b/codex-rs/state/src/runtime/goals.rs index aa0e90462b..50e8cf068e 100644 --- a/codex-rs/state/src/runtime/goals.rs +++ b/codex-rs/state/src/runtime/goals.rs @@ -141,7 +141,16 @@ INSERT INTO thread_goals ( created_at_ms, updated_at_ms ) VALUES (?, ?, ?, ?, ?, 0, 0, ?, ?) -ON CONFLICT(thread_id) DO NOTHING +ON CONFLICT(thread_id) DO UPDATE SET + goal_id = excluded.goal_id, + objective = excluded.objective, + status = excluded.status, + token_budget = excluded.token_budget, + tokens_used = 0, + time_used_seconds = 0, + created_at_ms = excluded.created_at_ms, + updated_at_ms = excluded.updated_at_ms +WHERE thread_goals.status = 'complete' RETURNING thread_id, goal_id, From 8e69d29521488506182d2b5814851cdbb90c1354 Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Tue, 9 Jun 2026 13:26:00 -0700 Subject: [PATCH 09/47] Reduce TUI legacy core dependencies (#26711) ## Why The TUI still reached through `app-server-client::legacy_core` for thread-name normalization and project-instruction filename details. In particular, checking the TUI's local filesystem for `/init` is incorrect for remote app-server sessions, where the server owns the working directory and instruction discovery. ## What changed - use the instruction source paths supplied by the app server to decide whether `/init` should avoid overwriting project instructions - keep the small thread-name normalization helper local to the TUI - remove the now-unused instruction filename constants, utility module, and other unused `legacy_core` re-exports - make status helper tests independent of concrete instruction filenames ## Verification - `just test -p codex-app-server-client` - `just test -p codex-tui slash_init_skips_when_project_instructions_are_loaded` - `just test -p codex-tui` ran 2,799 tests; 2,797 passed and two unrelated guardian feature-flag tests failed reproducibly in untouched code ### Manual test Started an app server over WebSocket with a remote workspace containing `AGENTS.md`, then connected the TUI using `--remote`. After confirming `thread/start` returned the file in `instructionSources`, deleted `AGENTS.md` and ran `/init` in the existing session. The TUI still reported that project instructions already existed and skipped `/init`. The trace contained no `turn/start` request, confirming the decision came from app-server session state rather than a new client-local filesystem check. --- codex-rs/app-server-client/src/lib.rs | 20 ------------ codex-rs/tui/prompt_for_init_command.md | 1 + codex-rs/tui/src/chatwidget.rs | 6 +++- codex-rs/tui/src/chatwidget/interaction.rs | 2 +- codex-rs/tui/src/chatwidget/slash_dispatch.rs | 10 +----- .../src/chatwidget/tests/slash_commands.rs | 31 +++---------------- codex-rs/tui/src/status/helpers.rs | 12 +++---- 7 files changed, 18 insertions(+), 64 deletions(-) diff --git a/codex-rs/app-server-client/src/lib.rs b/codex-rs/app-server-client/src/lib.rs index c75915629c..11684c6360 100644 --- a/codex-rs/app-server-client/src/lib.rs +++ b/codex-rs/app-server-client/src/lib.rs @@ -70,13 +70,9 @@ pub use crate::remote::RemoteAppServerEndpoint; /// module exists so clients can remove a direct `codex-core` dependency /// while legacy startup/config paths are migrated to RPCs. pub mod legacy_core { - pub use codex_core::DEFAULT_AGENTS_MD_FILENAME; - pub use codex_core::LOCAL_AGENTS_MD_FILENAME; - pub use codex_core::McpManager; pub use codex_core::check_execpolicy_for_warnings; pub use codex_core::format_exec_policy_error_with_source; pub use codex_core::grant_read_root_non_elevated; - pub use codex_core::web_search_detail; pub mod config { pub use codex_core::config::*; @@ -86,10 +82,6 @@ pub mod legacy_core { } } - pub mod connectors { - pub use codex_core::connectors::*; - } - pub mod otel_init { pub use codex_core::otel_init::*; } @@ -98,22 +90,10 @@ pub mod legacy_core { pub use codex_core::personality_migration::*; } - pub mod review_format { - pub use codex_core::review_format::*; - } - - pub mod review_prompts { - pub use codex_core::review_prompts::*; - } - pub mod test_support { pub use codex_core::test_support::*; } - pub mod util { - pub use codex_core::util::*; - } - pub mod windows_sandbox { pub use codex_core::windows_sandbox::*; } diff --git a/codex-rs/tui/prompt_for_init_command.md b/codex-rs/tui/prompt_for_init_command.md index b8fd3886b3..e96b002df7 100644 --- a/codex-rs/tui/prompt_for_init_command.md +++ b/codex-rs/tui/prompt_for_init_command.md @@ -1,4 +1,5 @@ Generate a file named AGENTS.md that serves as a contributor guide for this repository. +Before writing, check whether AGENTS.md already exists in the current working directory. If it does, do not overwrite or modify it. Your goal is to produce a clear, concise, and well-structured document with descriptive headings and actionable explanations for each section. Follow the outline below, but adapt as needed — add sections if relevant, and omit those that do not apply to this project. diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index c9827a4e71..6fd95e8dd5 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -58,7 +58,6 @@ use crate::bottom_pane::TerminalTitleItem; use crate::bottom_pane::TerminalTitleSetupView; use crate::diff_model::FileChange; use crate::git_action_directives::parse_assistant_markdown; -use crate::legacy_core::DEFAULT_AGENTS_MD_FILENAME; use crate::legacy_core::config::Config; use crate::legacy_core::config::Constrained; use crate::legacy_core::config::ConstraintResult; @@ -257,6 +256,11 @@ fn queued_message_edit_hint_binding( .or_else(|| bindings.first().copied()) } +fn normalize_thread_name(name: &str) -> Option { + let trimmed = name.trim(); + (!trimmed.is_empty()).then(|| trimmed.to_string()) +} + use crate::app_event::AppEvent; use crate::app_event::ExitMode; use crate::app_event::PermissionProfileSelection; diff --git a/codex-rs/tui/src/chatwidget/interaction.rs b/codex-rs/tui/src/chatwidget/interaction.rs index 7a3385034d..49fdd438f8 100644 --- a/codex-rs/tui/src/chatwidget/interaction.rs +++ b/codex-rs/tui/src/chatwidget/interaction.rs @@ -302,7 +302,7 @@ impl ChatWidget { /*initial_text*/ existing_name.unwrap_or_default().to_string(), /*context_label*/ None, Box::new(move |name: String| { - let Some(name) = crate::legacy_core::util::normalize_thread_name(&name) else { + let Some(name) = normalize_thread_name(&name) else { tx.send(AppEvent::InsertHistoryCell(Box::new( history_cell::new_error_event("Thread name cannot be empty.".to_string()), ))); diff --git a/codex-rs/tui/src/chatwidget/slash_dispatch.rs b/codex-rs/tui/src/chatwidget/slash_dispatch.rs index 48caf8f54d..e7776a3007 100644 --- a/codex-rs/tui/src/chatwidget/slash_dispatch.rs +++ b/codex-rs/tui/src/chatwidget/slash_dispatch.rs @@ -213,14 +213,6 @@ impl ChatWidget { .send(AppEvent::OpenDesktopThread { thread_id }); } SlashCommand::Init => { - let init_target = self.config.cwd.join(DEFAULT_AGENTS_MD_FILENAME); - if init_target.exists() { - let message = format!( - "{DEFAULT_AGENTS_MD_FILENAME} already exists here. Skipping /init to avoid overwriting it." - ); - self.add_info_message(message, /*hint*/ None); - return; - } const INIT_PROMPT: &str = include_str!("../../prompt_for_init_command.md"); self.submit_user_message(INIT_PROMPT.to_string().into()); } @@ -662,7 +654,7 @@ impl ChatWidget { } self.session_telemetry .counter("codex.thread.rename", /*inc*/ 1, &[]); - let Some(name) = crate::legacy_core::util::normalize_thread_name(&args) else { + let Some(name) = normalize_thread_name(&args) else { self.add_error_message("Thread name cannot be empty.".to_string()); return; }; diff --git a/codex-rs/tui/src/chatwidget/tests/slash_commands.rs b/codex-rs/tui/src/chatwidget/tests/slash_commands.rs index 9c2802fb60..02faef89f4 100644 --- a/codex-rs/tui/src/chatwidget/tests/slash_commands.rs +++ b/codex-rs/tui/src/chatwidget/tests/slash_commands.rs @@ -565,35 +565,14 @@ async fn ctrl_d_with_modal_open_does_not_quit() { } #[tokio::test] -async fn slash_init_skips_when_project_doc_exists() { - let (mut chat, mut rx, mut op_rx) = make_chatwidget_manual(/*model_override*/ None).await; - let tempdir = tempdir().unwrap(); - let existing_path = tempdir.path().join(DEFAULT_AGENTS_MD_FILENAME); - std::fs::write(&existing_path, "existing instructions").unwrap(); - chat.config.cwd = tempdir.path().to_path_buf().abs(); +async fn slash_init_does_not_depend_on_loaded_instruction_sources() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + chat.instruction_source_paths = vec![chat.config.cwd.join("project-instructions.md")]; submit_composer_text(&mut chat, "/init"); - match op_rx.try_recv() { - Err(TryRecvError::Empty) => {} - other => panic!("expected no Codex op to be sent, got {other:?}"), - } - - let cells = drain_insert_history(&mut rx); - assert_eq!(cells.len(), 1, "expected one info message"); - let rendered = lines_to_single_string(&cells[0]); - assert!( - rendered.contains(DEFAULT_AGENTS_MD_FILENAME), - "info message should mention the existing file: {rendered:?}" - ); - assert!( - rendered.contains("Skipping /init"), - "info message should explain why /init was skipped: {rendered:?}" - ); - assert_eq!( - std::fs::read_to_string(existing_path).unwrap(), - "existing instructions" - ); + assert_eq!(chat.input_queue.queued_user_messages.len(), 1); + assert!(drain_insert_history(&mut rx).is_empty()); assert_eq!(recall_latest_after_clearing(&mut chat), "/init"); } diff --git a/codex-rs/tui/src/status/helpers.rs b/codex-rs/tui/src/status/helpers.rs index 27292e1364..f3ceeb435f 100644 --- a/codex-rs/tui/src/status/helpers.rs +++ b/codex-rs/tui/src/status/helpers.rs @@ -185,8 +185,6 @@ fn title_case(s: &str) -> String { #[cfg(test)] mod tests { use super::*; - use crate::legacy_core::DEFAULT_AGENTS_MD_FILENAME; - use crate::legacy_core::LOCAL_AGENTS_MD_FILENAME; use crate::legacy_core::config::ConfigBuilder; use codex_utils_absolute_path::test_support::PathBufExt; use pretty_assertions::assert_eq; @@ -227,7 +225,7 @@ mod tests { async fn compose_agents_summary_includes_global_agents_path() { let codex_home = TempDir::new().expect("temp codex home"); let cwd = TempDir::new().expect("temp cwd"); - let global_agents_path = codex_home.path().join(DEFAULT_AGENTS_MD_FILENAME); + let global_agents_path = codex_home.path().join("global.md"); let config = test_config(&codex_home, &cwd).await; assert_eq!( @@ -240,7 +238,7 @@ mod tests { async fn compose_agents_summary_names_global_agents_override() { let codex_home = TempDir::new().expect("temp codex home"); let cwd = TempDir::new().expect("temp cwd"); - let override_path = codex_home.path().join(LOCAL_AGENTS_MD_FILENAME); + let override_path = codex_home.path().join("override.md"); let config = test_config(&codex_home, &cwd).await; assert_eq!( @@ -253,8 +251,8 @@ mod tests { async fn compose_agents_summary_orders_global_before_project_agents() { let codex_home = TempDir::new().expect("temp codex home"); let cwd = TempDir::new().expect("temp cwd"); - let global_agents_path = codex_home.path().join(DEFAULT_AGENTS_MD_FILENAME); - let project_agents_path = cwd.path().join(DEFAULT_AGENTS_MD_FILENAME); + let global_agents_path = codex_home.path().join("global.md"); + let project_agents_path = cwd.path().join("project.md"); let config = test_config(&codex_home, &cwd).await; let summary = compose_agents_summary( @@ -270,7 +268,7 @@ mod tests { Some(format_directory_display(&global_agents_path, /*max_width*/ None).as_str()) ); let project_path = paths.next().expect("project agents path"); - assert!(project_path.ends_with(DEFAULT_AGENTS_MD_FILENAME)); + assert!(project_path.ends_with("project.md")); assert_eq!(paths.next(), None); } } From 5a0f9134267e0fed2406fce0ea0f14e30894513e Mon Sep 17 00:00:00 2001 From: Ahmed Ibrahim Date: Tue, 9 Jun 2026 13:35:29 -0700 Subject: [PATCH 10/47] [1/4] Add Python goal routing foundation (#27110) ## Why Goal continuation turns are emitted by the existing runtime as separate physical turns. The Python SDK needs private thread-scoped routing before it can present those notifications as one logical operation, without changing ordinary turn routing or the app-server protocol. ## What - add private goal operation state and thread-scoped notification routing - add internal wrappers for the existing `thread/goal/clear` and `thread/goal/set` RPCs - include existing goal notifications in the SDK notification union - preserve ordinary turn-ID routing unchanged - add focused routing coverage This PR does not expose a public goal API. It is the first PR in the Python goal operations stack. ## Test plan - online CI, including the Python SDK suite - focused typed-notification routing coverage --- sdk/python/src/openai_codex/_goal.py | 156 ++++++++++++++++++ .../src/openai_codex/_message_router.py | 71 ++++++-- sdk/python/src/openai_codex/async_client.py | 39 +++++ sdk/python/src/openai_codex/client.py | 47 ++++++ sdk/python/src/openai_codex/models.py | 4 + sdk/python/tests/test_client_rpc_methods.py | 26 +++ 6 files changed, 328 insertions(+), 15 deletions(-) create mode 100644 sdk/python/src/openai_codex/_goal.py diff --git a/sdk/python/src/openai_codex/_goal.py b/sdk/python/src/openai_codex/_goal.py new file mode 100644 index 0000000000..dc1fe358c2 --- /dev/null +++ b/sdk/python/src/openai_codex/_goal.py @@ -0,0 +1,156 @@ +import queue +import threading +import time +from dataclasses import dataclass, field + +from .generated.v2_all import ( + ThreadGoalClearedNotification, + ThreadGoalStatus, + ThreadGoalUpdatedNotification, + Turn, + TurnCompletedNotification, + TurnStartedNotification, +) +from .models import Notification + + +class _GoalStreamClosed(Exception): + """Wake a notification reader after its logical stream closes.""" + + +def _terminal_goal_status(status: ThreadGoalStatus | None) -> bool: + return status in { + ThreadGoalStatus.paused, + ThreadGoalStatus.blocked, + ThreadGoalStatus.usage_limited, + ThreadGoalStatus.budget_limited, + ThreadGoalStatus.complete, + } + + +@dataclass(slots=True) +class _GoalOperationState: + """Private state for one goal operation exposed as a logical turn.""" + + thread_id: str + logical_turn_id: str | None = None + current_turn_id: str | None = None + status: ThreadGoalStatus | None = None + started_turn: Turn | None = None + completed_turn: Turn | None = None + interrupted: bool = False + interrupt_requested: bool = False + cleared: bool = False + _condition: threading.Condition = field(default_factory=threading.Condition) + _notifications: queue.Queue[Notification | BaseException] = field(default_factory=queue.Queue) + _failure: BaseException | None = None + _finished: bool = False + + def observe(self, notification: Notification) -> None: + payload = notification.payload + with self._condition: + if isinstance(payload, TurnStartedNotification): + if self.logical_turn_id is None: + self.logical_turn_id = payload.turn.id + self.current_turn_id = payload.turn.id + if self.started_turn is None: + self.started_turn = payload.turn + elif isinstance(payload, TurnCompletedNotification): + self.completed_turn = payload.turn + if self.current_turn_id == payload.turn.id: + self.current_turn_id = None + elif isinstance(payload, ThreadGoalUpdatedNotification): + self.status = payload.goal.status + if self.status == ThreadGoalStatus.active: + self.cleared = False + elif isinstance(payload, ThreadGoalClearedNotification): + self.cleared = True + if ( + self.current_turn_id is None + and self.completed_turn is not None + and (self.cleared or _terminal_goal_status(self.status)) + ): + self._finished = True + self._condition.notify_all() + self._notifications.put(notification) + + def wait_for_start(self, timeout: float) -> str | None: + """Wait for the runtime-generated first turn without consuming its event.""" + deadline = time.monotonic() + timeout + with self._condition: + while self.started_turn is None or self.logical_turn_id is None: + if self._failure is not None: + raise self._failure + remaining = deadline - time.monotonic() + if remaining <= 0: + return None + self._condition.wait(remaining) + return self.logical_turn_id + + def fail(self, exc: BaseException) -> None: + with self._condition: + self._failure = exc + self._condition.notify_all() + self._notifications.put(exc) + + def next_notification(self) -> Notification: + item = self._notifications.get() + if isinstance(item, BaseException): + raise item + return item + + def finish(self) -> None: + """Mark the logical operation inactive and wake waiting controls.""" + with self._condition: + self._finished = True + self.current_turn_id = None + self._condition.notify_all() + + def is_finished(self) -> bool: + with self._condition: + return self._finished + + def begin_interrupt(self) -> bool: + with self._condition: + if self._finished: + return False + self.interrupt_requested = True + return True + + def confirm_interrupt(self) -> None: + with self._condition: + self.interrupted = True + self.interrupt_requested = False + + def cancel_interrupt(self) -> None: + with self._condition: + self.interrupt_requested = False + + def explicit_interrupt(self, status: ThreadGoalStatus | None) -> bool: + with self._condition: + return self.interrupted or ( + self.interrupt_requested and status == ThreadGoalStatus.paused + ) + + def active_turn(self, *, after: str | None = None) -> str | None: + """Wait for the current turn, or return None once the goal has ended.""" + with self._condition: + while True: + if self._failure is not None: + raise self._failure + if self._finished: + return None + if self.current_turn_id is not None and self.current_turn_id != after: + return self.current_turn_id + if self.cleared or _terminal_goal_status(self.status): + return None + self._condition.wait() + + def current_turn(self) -> str | None: + """Return the current physical turn without waiting for rollover.""" + with self._condition: + return self.current_turn_id + + def wake_notification_reader(self) -> None: + """Release a reader blocked after its stream has been closed.""" + self._notifications.put(_GoalStreamClosed()) diff --git a/sdk/python/src/openai_codex/_message_router.py b/sdk/python/src/openai_codex/_message_router.py index b500f16c56..e71544a0ef 100644 --- a/sdk/python/src/openai_codex/_message_router.py +++ b/sdk/python/src/openai_codex/_message_router.py @@ -4,6 +4,7 @@ import queue import threading from collections import deque +from ._goal import _GoalOperationState from .errors import CodexError, map_jsonrpc_error from .generated.notification_registry import notification_turn_id from .generated.v2_all import AccountLoginCompletedNotification @@ -30,6 +31,7 @@ class MessageRouter: self._pending_login_notifications: dict[str, deque[Notification]] = {} self._turn_notifications: dict[str, queue.Queue[NotificationQueueItem]] = {} self._pending_turn_notifications: dict[str, deque[Notification]] = {} + self._goal_operations: dict[str, _GoalOperationState] = {} self._global_notifications: queue.Queue[NotificationQueueItem] = queue.Queue() def create_response_waiter(self, request_id: str) -> queue.Queue[ResponseQueueItem]: @@ -116,6 +118,21 @@ class MessageRouter: raise item return item + def register_goal(self, thread_id: str) -> _GoalOperationState: + """Register one thread-scoped logical goal operation before it starts.""" + state = _GoalOperationState(thread_id=thread_id) + with self._lock: + if thread_id in self._goal_operations: + raise RuntimeError(f"thread {thread_id!r} already has an active goal operation") + self._goal_operations[thread_id] = state + return state + + def unregister_goal(self, state: _GoalOperationState) -> None: + """Stop routing notifications to a completed logical goal operation.""" + with self._lock: + if self._goal_operations.get(state.thread_id) is state: + self._goal_operations.pop(state.thread_id) + def route_response(self, msg: dict[str, JsonValue]) -> None: """Deliver a JSON-RPC response or error to its request waiter.""" @@ -157,6 +174,17 @@ class MessageRouter: return turn_id = self._notification_turn_id(notification) + thread_id = self._notification_thread_id(notification) + if thread_id is not None: + with self._lock: + goal_state = self._goal_operations.get(thread_id) + if goal_state is not None and ( + turn_id is not None or notification.method.startswith("thread/goal/") + ): + goal_state.observe(notification) + if goal_state.is_finished(): + self.unregister_goal(goal_state) + return if turn_id is None: self._global_notifications.put(notification) return @@ -182,6 +210,8 @@ class MessageRouter: self._pending_login_notifications.clear() turn_queues = list(self._turn_notifications.values()) self._pending_turn_notifications.clear() + goal_operations = list(self._goal_operations.values()) + self._goal_operations.clear() # Put the same transport failure into every queue so no SDK call blocks # forever waiting for a response that cannot arrive. for waiter in response_waiters: @@ -190,8 +220,34 @@ class MessageRouter: login_queue.put(exc) for turn_queue in turn_queues: turn_queue.put(exc) + for goal_operation in goal_operations: + goal_operation.fail(exc) self._global_notifications.put(exc) + def _notification_turn_id(self, notification: Notification) -> str | None: + """Extract routing ids from generated metadata or raw unknown payloads.""" + payload = notification.payload + if isinstance(payload, UnknownNotification): + raw_turn_id = payload.params.get("turnId") + if isinstance(raw_turn_id, str): + return raw_turn_id + raw_turn = payload.params.get("turn") + if isinstance(raw_turn, dict): + raw_nested_turn_id = raw_turn.get("id") + if isinstance(raw_nested_turn_id, str): + return raw_nested_turn_id + return None + return notification_turn_id(payload) + + def _notification_thread_id(self, notification: Notification) -> str | None: + """Extract thread ids from typed payloads or raw unknown payloads.""" + payload = notification.payload + if isinstance(payload, UnknownNotification): + raw_thread_id = payload.params.get("threadId") + return raw_thread_id if isinstance(raw_thread_id, str) else None + thread_id = getattr(payload, "thread_id", None) + return thread_id if isinstance(thread_id, str) else None + def _notification_login_id(self, notification: Notification) -> str | None: """Extract the login attempt id from completion notifications.""" if notification.method != "account/login/completed": @@ -205,18 +261,3 @@ class MessageRouter: if isinstance(raw_login_id, str): return raw_login_id return None - - def _notification_turn_id(self, notification: Notification) -> str | None: - """Extract routing ids from known generated payloads or raw unknown payloads.""" - payload = notification.payload - if isinstance(payload, UnknownNotification): - raw_turn_id = payload.params.get("turnId") - if isinstance(raw_turn_id, str): - return raw_turn_id - raw_turn = payload.params.get("turn") - if isinstance(raw_turn, dict): - raw_nested_turn_id = raw_turn.get("id") - if isinstance(raw_nested_turn_id, str): - return raw_nested_turn_id - return None - return notification_turn_id(payload) diff --git a/sdk/python/src/openai_codex/async_client.py b/sdk/python/src/openai_codex/async_client.py index f0b12b99ed..ff12f16884 100644 --- a/sdk/python/src/openai_codex/async_client.py +++ b/sdk/python/src/openai_codex/async_client.py @@ -6,6 +6,7 @@ from typing import AsyncIterator, Callable, ParamSpec, TypeVar from pydantic import BaseModel +from ._goal import _GoalOperationState from .client import CodexClient, CodexConfig from .generated.v2_all import ( AccountLoginCompletedNotification, @@ -21,6 +22,9 @@ from .generated.v2_all import ( ThreadCompactStartResponse, ThreadForkParams as V2ThreadForkParams, ThreadForkResponse, + ThreadGoalClearResponse, + ThreadGoalSetResponse, + ThreadGoalStatus, ThreadListParams as V2ThreadListParams, ThreadListResponse, ThreadReadResponse, @@ -107,6 +111,14 @@ class AsyncCodexClient: """Unregister a turn notification queue on the wrapped sync client.""" self._sync.unregister_turn_notifications(turn_id) + def register_goal_operation(self, thread_id: str) -> _GoalOperationState: + """Register a logical goal route on the wrapped sync client.""" + return self._sync.register_goal_operation(thread_id) + + def unregister_goal_operation(self, state: _GoalOperationState) -> None: + """Release one logical goal route.""" + self._sync.unregister_goal_operation(state) + async def request( self, method: str, @@ -192,6 +204,29 @@ class AsyncCodexClient: """Start thread compaction using the wrapped sync client.""" return await self._call_sync(self._sync.thread_compact, thread_id) + async def thread_goal_clear(self, thread_id: str) -> ThreadGoalClearResponse: + """Clear the persisted goal through the wrapped sync client.""" + return await self._call_sync(self._sync.thread_goal_clear, thread_id) + + async def thread_goal_set( + self, + thread_id: str, + *, + objective: str | None = None, + status: ThreadGoalStatus | None = None, + ) -> ThreadGoalSetResponse: + """Create or update a persisted goal through the wrapped sync client.""" + return await self._call_sync( + self._sync.thread_goal_set, + thread_id, + objective=objective, + status=status, + ) + + async def pause_goal(self, thread_id: str) -> ThreadGoalSetResponse: + """Pause the active goal through the wrapped sync client.""" + return await self._call_sync(self._sync.pause_goal, thread_id) + async def turn_start( self, thread_id: str, @@ -256,6 +291,10 @@ class AsyncCodexClient: """Wait for the next notification routed to one turn.""" return await self._call_sync(self._sync.next_turn_notification, turn_id) + async def next_goal_notification(self, state: _GoalOperationState) -> Notification: + """Wait for the next notification in a logical goal turn.""" + return await self._call_sync(self._sync.next_goal_notification, state) + async def wait_for_login_completed( self, login_id: str, diff --git a/sdk/python/src/openai_codex/client.py b/sdk/python/src/openai_codex/client.py index 9951d6d1f3..951f4f95f0 100644 --- a/sdk/python/src/openai_codex/client.py +++ b/sdk/python/src/openai_codex/client.py @@ -10,6 +10,7 @@ from typing import Callable, Iterator, TypeVar from pydantic import BaseModel +from ._goal import _GoalOperationState from ._message_router import MessageRouter from ._version import __version__ as SDK_VERSION from .errors import CodexError, TransportClosedError @@ -30,6 +31,9 @@ from .generated.v2_all import ( ThreadCompactStartResponse, ThreadForkParams as V2ThreadForkParams, ThreadForkResponse, + ThreadGoalClearResponse, + ThreadGoalSetResponse, + ThreadGoalStatus, ThreadListParams as V2ThreadListParams, ThreadListResponse, ThreadReadResponse, @@ -352,6 +356,18 @@ class CodexClient: """Return the next routed notification for the requested turn id.""" return self._router.next_turn_notification(turn_id) + def register_goal_operation(self, thread_id: str) -> _GoalOperationState: + """Register a private thread-scoped route for a logical goal turn.""" + return self._router.register_goal(thread_id) + + def unregister_goal_operation(self, state: _GoalOperationState) -> None: + """Release routing state for one logical goal turn.""" + self._router.unregister_goal(state) + + def next_goal_notification(self, state: _GoalOperationState) -> Notification: + """Wait for the next notification in a logical goal turn.""" + return state.next_notification() + def account_login_start( self, params: V2LoginAccountParams | JsonObject, @@ -452,6 +468,37 @@ class CodexClient: response_model=ThreadCompactStartResponse, ) + def thread_goal_clear(self, thread_id: str) -> ThreadGoalClearResponse: + """Clear the persisted goal for a thread before replacing it.""" + return self.request( + "thread/goal/clear", + {"threadId": thread_id}, + response_model=ThreadGoalClearResponse, + ) + + def thread_goal_set( + self, + thread_id: str, + *, + objective: str | None = None, + status: ThreadGoalStatus | None = None, + ) -> ThreadGoalSetResponse: + """Create or update the persisted goal for a thread.""" + payload: JsonObject = {"threadId": thread_id} + if objective is not None: + payload["objective"] = objective + if status is not None: + payload["status"] = status.value + return self.request( + "thread/goal/set", + payload, + response_model=ThreadGoalSetResponse, + ) + + def pause_goal(self, thread_id: str) -> ThreadGoalSetResponse: + """Pause the active goal used by a logical goal turn.""" + return self.thread_goal_set(thread_id, status=ThreadGoalStatus.paused) + def turn_start( self, thread_id: str, diff --git a/sdk/python/src/openai_codex/models.py b/sdk/python/src/openai_codex/models.py index 70c61d44cd..d9d15dc684 100644 --- a/sdk/python/src/openai_codex/models.py +++ b/sdk/python/src/openai_codex/models.py @@ -27,6 +27,8 @@ from .generated.v2_all import ( ReasoningSummaryTextDeltaNotification, ReasoningTextDeltaNotification, TerminalInteractionNotification, + ThreadGoalClearedNotification, + ThreadGoalUpdatedNotification, ThreadNameUpdatedNotification, ThreadStartedNotification, ThreadTokenUsageUpdatedNotification, @@ -70,6 +72,8 @@ NotificationPayload: TypeAlias = ( | ReasoningTextDeltaNotification | TerminalInteractionNotification | ThreadNameUpdatedNotification + | ThreadGoalClearedNotification + | ThreadGoalUpdatedNotification | ThreadStartedNotification | ThreadTokenUsageUpdatedNotification | TurnCompletedNotification diff --git a/sdk/python/tests/test_client_rpc_methods.py b/sdk/python/tests/test_client_rpc_methods.py index 73fdd452ff..15be24f3b4 100644 --- a/sdk/python/tests/test_client_rpc_methods.py +++ b/sdk/python/tests/test_client_rpc_methods.py @@ -185,6 +185,32 @@ def test_turn_notification_router_demuxes_registered_turns() -> None: ] +def test_goal_notification_router_routes_by_thread_id() -> None: + """A goal operation should receive turn notifications across physical turn ids.""" + client = CodexClient() + state = client.register_goal_operation("thread-1") + + client._router.route_notification( + client._coerce_notification( + "item/agentMessage/delta", + { + "delta": "continued", + "itemId": "item-1", + "threadId": "thread-1", + "turnId": "turn-2", + }, + ) + ) + + event = client.next_goal_notification(state) + + assert isinstance(event.payload, AgentMessageDeltaNotification) + assert (event.method, event.payload.delta) == ( + "item/agentMessage/delta", + "continued", + ) + + def test_client_reader_routes_interleaved_turn_notifications_by_turn_id() -> None: """Reader-loop routing should preserve order within each interleaved turn stream.""" client = CodexClient() From 4ec3b8eeea38e9f693568e4f51bb607cc6cd9717 Mon Sep 17 00:00:00 2001 From: jif Date: Tue, 9 Jun 2026 22:44:16 +0200 Subject: [PATCH 11/47] Route hosted Apps MCP through extensions (#27191) ## Stack - Base: #27184 - This PR is the second vertical and should be reviewed against `jif/external-plugins-1`, not `main`. ## Why CCA is moving toward a split runtime where the orchestrator may have no filesystem or executor, but it still needs to activate remotely hosted plugin components. HTTP MCP servers are the simplest complete example: they need configuration and host authentication, but they do not need an executor process. The Apps MCP endpoint is currently synthesized by a special-purpose loader inside the MCP runtime. That works locally, but it leaves hosted MCP activation outside the extension model being established in #27184. It also makes the Apps path a poor foundation for plugins whose skills, MCP servers, connectors, and hooks may come from different sources or execute in different places. This PR moves that one behavior behind an extension-owned contribution while preserving the existing local fallback. It deliberately does not introduce a generic plugin activation framework. ## What changed ### MCP extension contribution `codex-extension-api` gains an ordered `McpServerContributor` contract. A contributor returns typed `Set` or `Remove` overlays for MCP server configuration; later contributors win for the names they own. The contract stays at the existing MCP configuration boundary. Extensions do not create a second connection manager or transport abstraction. ### Hosted Apps MCP extension A new `codex-mcp-extension` contributes the reserved `codex_apps` server from the existing Apps feature, ChatGPT base URL, path override, and product SKU configuration. When `apps_mcp_path_override` is enabled for `https://chatgpt.com`, the resulting streamable HTTP endpoint is `https://chatgpt.com/backend-api/ps/mcp`. The existing ChatGPT-auth gate remains authoritative, so this server can run in an orchestrator-only process without being exposed for API-key sessions. ### One resolved runtime view `McpManager` now distinguishes three views: - **configured:** config- and plugin-backed servers before extension overlays; - **runtime:** configured servers plus host-installed extension contributions; - **effective:** runtime servers after auth gating and compatibility built-ins. App-server installs the hosted MCP extension and uses the runtime view for thread startup, refresh, status, threadless resource reads, connector discovery, and MCP OAuth lookup. This keeps `mcpServer/oauth/login` consistent with the servers exposed by the other MCP APIs. The hosted Apps server itself continues to use existing ChatGPT host authentication rather than MCP OAuth. ## Compatibility Hosts that do not install the MCP extension retain the existing Apps MCP synthesis path. This preserves current local-only, CLI, and standalone-host behavior while app-server exercises the extension path. Disabling Apps removes the reserved `codex_apps` entry, and losing ChatGPT auth removes it from the effective runtime view. Executor availability is not consulted for this HTTP transport. ## Follow-ups The next vertical will resolve a manifest-declared stdio MCP server from an executor-selected plugin root and execute it in the environment that owns that root. Later verticals can add backend-owned skills, connector metadata, hooks, durable selection semantics, and incremental local convergence without changing the component-specific runtime boundaries introduced here. ## Verification Focused coverage was added for: - contributing the hosted Apps MCP at `/backend-api/ps/mcp` without an executor; - requiring ChatGPT auth in the effective runtime view; - removing a reserved configured Apps server when the Apps feature is disabled. `cargo check -p codex-app-server -p codex-mcp-extension -p codex-extension-api -p codex-mcp` passed. Tests and Clippy were not run locally under the current development instruction; CI provides the full validation pass. --- codex-rs/Cargo.lock | 19 ++++ codex-rs/Cargo.toml | 2 + codex-rs/app-server/Cargo.toml | 1 + codex-rs/app-server/src/extensions.rs | 1 + codex-rs/app-server/src/mcp_refresh.rs | 5 +- codex-rs/app-server/src/request_processors.rs | 1 + .../src/request_processors/apps_processor.rs | 34 +++++-- .../src/request_processors/mcp_processor.rs | 22 +++-- .../src/request_processors/plugins.rs | 11 ++- codex-rs/chatgpt/src/connectors.rs | 1 + codex-rs/codex-mcp/src/lib.rs | 1 + codex-rs/codex-mcp/src/mcp/mod.rs | 36 ++++--- codex-rs/codex-mcp/src/mcp/mod_tests.rs | 11 +-- codex-rs/core/src/config/mod.rs | 1 + codex-rs/core/src/connectors.rs | 27 ++++-- codex-rs/core/src/mcp.rs | 70 +++++++++++++- codex-rs/core/src/mcp_skill_dependencies.rs | 22 +++-- codex-rs/core/src/session/mcp.rs | 6 +- codex-rs/core/src/thread_manager.rs | 5 +- codex-rs/ext/extension-api/Cargo.toml | 1 + .../ext/extension-api/src/contributors.rs | 14 +++ .../ext/extension-api/src/contributors/mcp.rs | 22 +++++ codex-rs/ext/extension-api/src/lib.rs | 2 + codex-rs/ext/extension-api/src/registry.rs | 15 +++ codex-rs/ext/mcp/BUILD.bazel | 6 ++ codex-rs/ext/mcp/Cargo.toml | 29 ++++++ codex-rs/ext/mcp/src/lib.rs | 31 +++++++ codex-rs/ext/mcp/tests/hosted_apps_mcp.rs | 93 +++++++++++++++++++ 28 files changed, 424 insertions(+), 65 deletions(-) create mode 100644 codex-rs/ext/extension-api/src/contributors/mcp.rs create mode 100644 codex-rs/ext/mcp/BUILD.bazel create mode 100644 codex-rs/ext/mcp/Cargo.toml create mode 100644 codex-rs/ext/mcp/src/lib.rs create mode 100644 codex-rs/ext/mcp/tests/hosted_apps_mcp.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 8783f0e16c..cbc237a84c 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1982,6 +1982,7 @@ dependencies = [ "codex-image-generation-extension", "codex-login", "codex-mcp", + "codex-mcp-extension", "codex-memories-extension", "codex-memories-write", "codex-model-provider", @@ -2920,6 +2921,7 @@ name = "codex-extension-api" version = "0.0.0" dependencies = [ "async-trait", + "codex-config", "codex-context-fragments", "codex-protocol", "codex-tools", @@ -3247,6 +3249,23 @@ dependencies = [ "url", ] +[[package]] +name = "codex-mcp-extension" +version = "0.0.0" +dependencies = [ + "async-trait", + "codex-config", + "codex-core", + "codex-core-plugins", + "codex-extension-api", + "codex-features", + "codex-login", + "codex-mcp", + "pretty_assertions", + "tempfile", + "tokio", +] + [[package]] name = "codex-mcp-server" version = "0.0.0" diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index 955d8987f7..b87b0cf39f 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -49,6 +49,7 @@ members = [ "ext/guardian", "ext/image-generation", "ext/memories", + "ext/mcp", "ext/skills", "ext/web-search", "external-agent-migration", @@ -191,6 +192,7 @@ codex-web-search-extension = { path = "ext/web-search" } codex-memories-read = { path = "memories/read" } codex-memories-write = { path = "memories/write" } codex-mcp = { path = "codex-mcp" } +codex-mcp-extension = { path = "ext/mcp" } codex-mcp-server = { path = "mcp-server" } codex-model-provider-info = { path = "model-provider-info" } codex-models-manager = { path = "models-manager" } diff --git a/codex-rs/app-server/Cargo.toml b/codex-rs/app-server/Cargo.toml index d6f71ab82f..55ea78e617 100644 --- a/codex-rs/app-server/Cargo.toml +++ b/codex-rs/app-server/Cargo.toml @@ -61,6 +61,7 @@ codex-memories-extension = { workspace = true } codex-web-search-extension = { workspace = true } codex-memories-write = { workspace = true } codex-mcp = { workspace = true } +codex-mcp-extension = { workspace = true } codex-model-provider = { workspace = true } codex-models-manager = { workspace = true } codex-protocol = { workspace = true } diff --git a/codex-rs/app-server/src/extensions.rs b/codex-rs/app-server/src/extensions.rs index b18331064f..a0a44fd80d 100644 --- a/codex-rs/app-server/src/extensions.rs +++ b/codex-rs/app-server/src/extensions.rs @@ -50,6 +50,7 @@ where } codex_guardian::install(&mut builder, guardian_agent_spawner); codex_memories_extension::install(&mut builder, codex_otel::global()); + codex_mcp_extension::install(&mut builder); codex_web_search_extension::install(&mut builder, auth_manager.clone()); codex_image_generation_extension::install(&mut builder, auth_manager); codex_skills_extension::install_with_providers( diff --git a/codex-rs/app-server/src/mcp_refresh.rs b/codex-rs/app-server/src/mcp_refresh.rs index 821af862d5..2cd1027754 100644 --- a/codex-rs/app-server/src/mcp_refresh.rs +++ b/codex-rs/app-server/src/mcp_refresh.rs @@ -67,10 +67,7 @@ async fn build_refresh_config( let config = config_manager .load_latest_config_for_thread(thread_config.as_ref()) .await?; - let mcp_servers = thread_manager - .mcp_manager() - .configured_servers(&config) - .await; + let mcp_servers = thread_manager.mcp_manager().runtime_servers(&config).await; Ok(McpServerRefreshConfig { mcp_servers: serde_json::to_value(mcp_servers).map_err(io::Error::other)?, mcp_oauth_credentials_store_mode: serde_json::to_value( diff --git a/codex-rs/app-server/src/request_processors.rs b/codex-rs/app-server/src/request_processors.rs index d4e9395627..00f17937ba 100644 --- a/codex-rs/app-server/src/request_processors.rs +++ b/codex-rs/app-server/src/request_processors.rs @@ -280,6 +280,7 @@ use codex_config::types::McpServerTransportConfig; use codex_core::CodexThread; use codex_core::CodexThreadSettingsOverrides; use codex_core::ForkSnapshot; +use codex_core::McpManager; use codex_core::NewThread; #[cfg(test)] use codex_core::SessionMeta; diff --git a/codex-rs/app-server/src/request_processors/apps_processor.rs b/codex-rs/app-server/src/request_processors/apps_processor.rs index 4a665d8ef3..95cf3f452a 100644 --- a/codex-rs/app-server/src/request_processors/apps_processor.rs +++ b/codex-rs/app-server/src/request_processors/apps_processor.rs @@ -88,11 +88,19 @@ impl AppsRequestProcessor { let request = request_id.clone(); let outgoing = Arc::clone(&self.outgoing); let environment_manager = self.thread_manager.environment_manager(); + let mcp_manager = self.thread_manager.mcp_manager(); let shutdown_token = self.shutdown_token.child_token(); tokio::spawn(async move { tokio::select! { _ = shutdown_token.cancelled() => {} - _ = Self::apps_list_task(outgoing, request, params, config, environment_manager) => {} + _ = Self::apps_list_task( + outgoing, + request, + params, + config, + environment_manager, + mcp_manager, + ) => {} } }); Ok(None) @@ -108,11 +116,15 @@ impl AppsRequestProcessor { params: AppsListParams, config: Config, environment_manager: Arc, + mcp_manager: Arc, ) { let retry_params = params.clone(); let retry_config = config.clone(); let retry_environment_manager = Arc::clone(&environment_manager); - let result = Self::apps_list_response(&outgoing, params, config, environment_manager).await; + let retry_mcp_manager = Arc::clone(&mcp_manager); + let result = + Self::apps_list_response(&outgoing, params, config, environment_manager, mcp_manager) + .await; let should_retry = result .as_ref() .is_ok_and(|(_, codex_apps_ready)| !codex_apps_ready); @@ -128,6 +140,7 @@ impl AppsRequestProcessor { retry_params, retry_config, retry_environment_manager, + retry_mcp_manager, ) .await { @@ -141,6 +154,7 @@ impl AppsRequestProcessor { params: AppsListParams, config: Config, environment_manager: Arc, + mcp_manager: Arc, ) -> Result<(AppsListResponse, bool), JSONRPCErrorError> { let AppsListParams { cursor, @@ -167,14 +181,14 @@ impl AppsRequestProcessor { let accessible_config = config.clone(); let accessible_tx = tx.clone(); tokio::spawn(async move { - let result = - connectors::list_accessible_connectors_from_mcp_tools_with_environment_manager( - &accessible_config, - force_refetch, - Arc::clone(&environment_manager), - ) - .await - .map_err(|err| format!("failed to load accessible apps: {err}")); + let result = connectors::list_accessible_connectors_from_mcp_tools_with_mcp_manager( + &accessible_config, + force_refetch, + Arc::clone(&environment_manager), + mcp_manager, + ) + .await + .map_err(|err| format!("failed to load accessible apps: {err}")); let _ = accessible_tx.send(AppListLoadResult::Accessible(result)); }); diff --git a/codex-rs/app-server/src/request_processors/mcp_processor.rs b/codex-rs/app-server/src/request_processors/mcp_processor.rs index ae62e2e785..facd4facc6 100644 --- a/codex-rs/app-server/src/request_processors/mcp_processor.rs +++ b/codex-rs/app-server/src/request_processors/mcp_processor.rs @@ -120,12 +120,16 @@ impl McpRequestProcessor { timeout_secs, } = params; - let configured_servers = self + let auth = self.auth_manager.auth().await; + let effective_servers = self .thread_manager .mcp_manager() - .configured_servers(&config) + .effective_servers(&config, auth.as_ref()) .await; - let Some(server) = configured_servers.get(&name) else { + let Some(server) = effective_servers + .get(&name) + .and_then(codex_mcp::EffectiveMcpServer::configured_config) + else { return Err(invalid_request(format!( "No MCP server named '{name}' found." ))); @@ -210,8 +214,10 @@ impl McpRequestProcessor { } None => self.load_latest_config(/*fallback_cwd*/ None).await?, }; - let mcp_config = config - .to_mcp_config(self.thread_manager.plugins_manager().as_ref()) + let mcp_config = self + .thread_manager + .mcp_manager() + .runtime_config(&config) .await; let auth = self.auth_manager.auth().await; let environment_manager = self.thread_manager.environment_manager(); @@ -361,8 +367,10 @@ impl McpRequestProcessor { } let config = self.load_latest_config(/*fallback_cwd*/ None).await?; - let mcp_config = config - .to_mcp_config(self.thread_manager.plugins_manager().as_ref()) + let mcp_config = self + .thread_manager + .mcp_manager() + .runtime_config(&config) .await; let auth = self.auth_manager.auth().await; let environment_manager = self.thread_manager.environment_manager(); diff --git a/codex-rs/app-server/src/request_processors/plugins.rs b/codex-rs/app-server/src/request_processors/plugins.rs index bc2383c21f..54a5efbe7a 100644 --- a/codex-rs/app-server/src/request_processors/plugins.rs +++ b/codex-rs/app-server/src/request_processors/plugins.rs @@ -1036,6 +1036,7 @@ impl PluginRequestProcessor { &config, &outcome.plugin.apps, Arc::clone(&environment_manager), + self.thread_manager.mcp_manager(), ) .await; let visible_skills = outcome @@ -1118,6 +1119,7 @@ impl PluginRequestProcessor { &config, &plugin_apps, Arc::clone(&environment_manager), + self.thread_manager.mcp_manager(), ) .await; remote_plugin_detail_to_info(remote_detail, app_summaries) @@ -1611,10 +1613,11 @@ impl PluginRequestProcessor { let environment_manager = self.thread_manager.environment_manager(); let (all_connectors_result, accessible_connectors_result) = tokio::join!( connectors::list_all_connectors_with_options(config, /*force_refetch*/ false), - connectors::list_accessible_connectors_from_mcp_tools_with_environment_manager( + connectors::list_accessible_connectors_from_mcp_tools_with_mcp_manager( config, /*force_refetch*/ true, - Arc::clone(&environment_manager) + Arc::clone(&environment_manager), + self.thread_manager.mcp_manager(), ), ); @@ -1881,6 +1884,7 @@ async fn load_plugin_app_summaries( config: &Config, plugin_apps: &[codex_plugin::AppConnectorId], environment_manager: Arc, + mcp_manager: Arc, ) -> Vec { if plugin_apps.is_empty() { return Vec::new(); @@ -1900,10 +1904,11 @@ async fn load_plugin_app_summaries( let plugin_connectors = connectors::connectors_for_plugin_apps(connectors, plugin_apps); let accessible_connectors = - match connectors::list_accessible_connectors_from_mcp_tools_with_environment_manager( + match connectors::list_accessible_connectors_from_mcp_tools_with_mcp_manager( config, /*force_refetch*/ false, environment_manager, + mcp_manager, ) .await { diff --git a/codex-rs/chatgpt/src/connectors.rs b/codex-rs/chatgpt/src/connectors.rs index 63b34e7355..4bbf40813e 100644 --- a/codex-rs/chatgpt/src/connectors.rs +++ b/codex-rs/chatgpt/src/connectors.rs @@ -14,6 +14,7 @@ use codex_connectors::merge::merge_plugin_connectors; use codex_core::config::Config; pub use codex_core::connectors::list_accessible_connectors_from_mcp_tools; pub use codex_core::connectors::list_accessible_connectors_from_mcp_tools_with_environment_manager; +pub use codex_core::connectors::list_accessible_connectors_from_mcp_tools_with_mcp_manager; pub use codex_core::connectors::list_accessible_connectors_from_mcp_tools_with_options; pub use codex_core::connectors::list_accessible_connectors_from_mcp_tools_with_options_and_status; pub use codex_core::connectors::list_cached_accessible_connectors_from_mcp_tools; diff --git a/codex-rs/codex-mcp/src/lib.rs b/codex-rs/codex-mcp/src/lib.rs index 83419175d4..a56365277a 100644 --- a/codex-rs/codex-mcp/src/lib.rs +++ b/codex-rs/codex-mcp/src/lib.rs @@ -24,6 +24,7 @@ pub use auth_elicitation::build_auth_elicitation_plan; pub use auth_elicitation::connector_auth_failure_from_tool_result; pub use codex_apps::CodexAppsToolsCacheKey; pub use codex_apps::codex_apps_tools_cache_key; +pub use mcp::codex_apps_mcp_server_config; pub use mcp::configured_mcp_servers; pub use mcp::effective_mcp_servers; pub use mcp::effective_mcp_servers_from_configured; diff --git a/codex-rs/codex-mcp/src/mcp/mod.rs b/codex-rs/codex-mcp/src/mcp/mod.rs index 9633655846..a74f95bae3 100644 --- a/codex-rs/codex-mcp/src/mcp/mod.rs +++ b/codex-rs/codex-mcp/src/mcp/mod.rs @@ -131,6 +131,11 @@ pub struct McpConfig { /// ChatGPT auth is checked separately at runtime before the host-owned apps /// MCP server is added. pub apps_enabled: bool, + /// Whether to synthesize the legacy host-owned Apps MCP server. + /// + /// Hosts that install an MCP extension for this server disable the legacy + /// loader and contribute the server through the normal runtime overlay. + pub legacy_apps_mcp_loader_enabled: bool, /// Whether model-visible MCP tool namespaces should keep the legacy /// `mcp__` prefix. pub prefix_mcp_tool_names: bool, @@ -218,10 +223,20 @@ pub fn with_codex_apps_mcp( auth: Option<&CodexAuth>, config: &McpConfig, ) -> HashMap { + if !config.legacy_apps_mcp_loader_enabled { + if !host_owned_codex_apps_enabled(config, auth) { + servers.remove(CODEX_APPS_MCP_SERVER_NAME); + } + return servers; + } if host_owned_codex_apps_enabled(config, auth) { servers.insert( CODEX_APPS_MCP_SERVER_NAME.to_string(), - EffectiveMcpServer::configured(codex_apps_mcp_server_config(config)), + EffectiveMcpServer::configured(codex_apps_mcp_server_config( + &config.chatgpt_base_url, + config.apps_mcp_path_override.as_deref(), + config.apps_mcp_product_sku.as_deref(), + )), ); } else { servers.remove(CODEX_APPS_MCP_SERVER_NAME); @@ -381,13 +396,6 @@ pub async fn collect_mcp_server_status_snapshot_with_detail( snapshot } -pub(crate) fn codex_apps_mcp_url(config: &McpConfig) -> String { - codex_apps_mcp_url_for_base_url( - &config.chatgpt_base_url, - config.apps_mcp_path_override.as_deref(), - ) -} - /// The Responses API requires tool names to match `^[a-zA-Z0-9_-]+$`. /// MCP server/tool names are user-controlled, so sanitize the fully-qualified /// name we expose to the model by replacing any disallowed character with `_`. @@ -443,10 +451,14 @@ fn codex_apps_mcp_url_for_base_url(base_url: &str, apps_mcp_path_override: Optio format!("{base_url}/{path}") } -fn codex_apps_mcp_server_config(config: &McpConfig) -> McpServerConfig { - let url = codex_apps_mcp_url(config); - let http_headers = config.apps_mcp_product_sku.as_ref().map(|product_sku| { - HashMap::from([("X-OpenAI-Product-Sku".to_string(), product_sku.clone())]) +pub fn codex_apps_mcp_server_config( + chatgpt_base_url: &str, + apps_mcp_path_override: Option<&str>, + apps_mcp_product_sku: Option<&str>, +) -> McpServerConfig { + let url = codex_apps_mcp_url_for_base_url(chatgpt_base_url, apps_mcp_path_override); + let http_headers = apps_mcp_product_sku.map(|product_sku| { + HashMap::from([("X-OpenAI-Product-Sku".to_string(), product_sku.to_string())]) }); McpServerConfig { diff --git a/codex-rs/codex-mcp/src/mcp/mod_tests.rs b/codex-rs/codex-mcp/src/mcp/mod_tests.rs index b29f7a9e58..fb1af362ca 100644 --- a/codex-rs/codex-mcp/src/mcp/mod_tests.rs +++ b/codex-rs/codex-mcp/src/mcp/mod_tests.rs @@ -27,6 +27,7 @@ fn test_mcp_config(codex_home: PathBuf) -> McpConfig { codex_linux_sandbox_exe: None, use_legacy_landlock: false, apps_enabled: false, + legacy_apps_mcp_loader_enabled: true, prefix_mcp_tool_names: true, client_elicitation_capability: ElicitationCapability::default(), configured_mcp_servers: HashMap::new(), @@ -206,16 +207,6 @@ fn codex_apps_mcp_url_for_base_url_keeps_existing_paths() { ); } -#[test] -fn codex_apps_mcp_url_uses_legacy_codex_apps_path() { - let config = test_mcp_config(PathBuf::from("/tmp")); - - assert_eq!( - codex_apps_mcp_url(&config), - "https://chatgpt.com/backend-api/wham/apps" - ); -} - #[test] fn codex_apps_server_config_uses_legacy_codex_apps_path() { let mut config = test_mcp_config(PathBuf::from("/tmp")); diff --git a/codex-rs/core/src/config/mod.rs b/codex-rs/core/src/config/mod.rs index 93e203053f..5c2a93bc45 100644 --- a/codex-rs/core/src/config/mod.rs +++ b/codex-rs/core/src/config/mod.rs @@ -1428,6 +1428,7 @@ impl Config { codex_linux_sandbox_exe: self.codex_linux_sandbox_exe.clone(), use_legacy_landlock: self.features.use_legacy_landlock(), apps_enabled: self.features.enabled(Feature::Apps), + legacy_apps_mcp_loader_enabled: true, prefix_mcp_tool_names: self.prefix_mcp_tool_names(), client_elicitation_capability: if self.features.enabled(Feature::AuthElicitation) { ElicitationCapability { diff --git a/codex-rs/core/src/connectors.rs b/codex-rs/core/src/connectors.rs index 7a8344b3d7..623cb44216 100644 --- a/codex-rs/core/src/connectors.rs +++ b/codex-rs/core/src/connectors.rs @@ -1,4 +1,3 @@ -use std::collections::HashMap; use std::collections::HashSet; use std::sync::Arc; use std::sync::LazyLock; @@ -41,8 +40,8 @@ use codex_mcp::ToolInfo; use codex_mcp::ToolPluginProvenance; use codex_mcp::codex_apps_tools_cache_key; use codex_mcp::compute_auth_statuses; +use codex_mcp::effective_mcp_servers; use codex_mcp::host_owned_codex_apps_enabled; -use codex_mcp::with_codex_apps_mcp; const CONNECTORS_READY_TIMEOUT_ON_EMPTY_TOOLS: Duration = Duration::from_secs(30); @@ -220,6 +219,23 @@ pub async fn list_accessible_connectors_from_mcp_tools_with_environment_manager( config: &Config, force_refetch: bool, environment_manager: Arc, +) -> anyhow::Result { + let plugins_manager = Arc::new(PluginsManager::new(config.codex_home.to_path_buf())); + let mcp_manager = Arc::new(McpManager::new(plugins_manager)); + list_accessible_connectors_from_mcp_tools_with_mcp_manager( + config, + force_refetch, + environment_manager, + mcp_manager, + ) + .await +} + +pub async fn list_accessible_connectors_from_mcp_tools_with_mcp_manager( + config: &Config, + force_refetch: bool, + environment_manager: Arc, + mcp_manager: Arc, ) -> anyhow::Result { let auth_manager = AuthManager::shared_from_config(config, /*enable_codex_api_key_env*/ false).await; @@ -234,8 +250,6 @@ pub async fn list_accessible_connectors_from_mcp_tools_with_environment_manager( }); } let cache_key = accessible_connectors_cache_key(config, auth.as_ref()); - let plugins_manager = Arc::new(PluginsManager::new(config.codex_home.to_path_buf())); - let mcp_manager = McpManager::new(Arc::clone(&plugins_manager)); let tool_plugin_provenance = mcp_manager.tool_plugin_provenance(config).await; if !force_refetch && let Some(cached_connectors) = read_cached_accessible_connectors(&cache_key) { @@ -250,8 +264,9 @@ pub async fn list_accessible_connectors_from_mcp_tools_with_environment_manager( }); } - let mcp_config = config.to_mcp_config(plugins_manager.as_ref()).await; - let mcp_servers = with_codex_apps_mcp(HashMap::new(), auth.as_ref(), &mcp_config); + let mcp_config = mcp_manager.runtime_config(config).await; + let mut mcp_servers = effective_mcp_servers(&mcp_config, auth.as_ref()); + mcp_servers.retain(|name, _| name == CODEX_APPS_MCP_SERVER_NAME); let host_owned_codex_apps_enabled = host_owned_codex_apps_enabled(&mcp_config, auth.as_ref()); if mcp_servers.is_empty() { return Ok(AccessibleConnectorsStatus { diff --git a/codex-rs/core/src/mcp.rs b/codex-rs/core/src/mcp.rs index 91ebce4ad8..1b693d70fd 100644 --- a/codex-rs/core/src/mcp.rs +++ b/codex-rs/core/src/mcp.rs @@ -4,8 +4,12 @@ use std::sync::Arc; use crate::config::Config; use codex_config::McpServerConfig; use codex_core_plugins::PluginsManager; +use codex_extension_api::ExtensionRegistry; +use codex_extension_api::McpServerContribution; use codex_login::CodexAuth; +use codex_mcp::CODEX_APPS_MCP_SERVER_NAME; use codex_mcp::EffectiveMcpServer; +use codex_mcp::McpConfig; use codex_mcp::ToolPluginProvenance; use codex_mcp::configured_mcp_servers; use codex_mcp::effective_mcp_servers; @@ -14,29 +18,91 @@ use codex_mcp::tool_plugin_provenance as collect_tool_plugin_provenance; #[derive(Clone)] pub struct McpManager { plugins_manager: Arc, + extensions: Arc>, } impl McpManager { pub fn new(plugins_manager: Arc) -> Self { - Self { plugins_manager } + Self { + plugins_manager, + extensions: codex_extension_api::empty_extension_registry(), + } } + /// Creates a manager that resolves host-installed MCP contributions. + pub fn new_with_extensions( + plugins_manager: Arc, + extensions: Arc>, + ) -> Self { + Self { + plugins_manager, + extensions, + } + } + + /// Returns the MCP config after applying runtime-only extension overlays. + pub async fn runtime_config(&self, config: &Config) -> McpConfig { + let mut mcp_config = config.to_mcp_config(self.plugins_manager.as_ref()).await; + let contributions = self.contributions(config).await; + if contributions + .iter() + .any(|contribution| contribution.name() == CODEX_APPS_MCP_SERVER_NAME) + { + mcp_config.legacy_apps_mcp_loader_enabled = false; + } + Self::apply_to_configured_servers(&contributions, &mut mcp_config.configured_mcp_servers); + mcp_config + } + + /// Returns config- and plugin-backed servers without runtime contributions. pub async fn configured_servers(&self, config: &Config) -> HashMap { let mcp_config = config.to_mcp_config(self.plugins_manager.as_ref()).await; configured_mcp_servers(&mcp_config) } + /// Returns configured and host-contributed servers before auth gating. + pub async fn runtime_servers(&self, config: &Config) -> HashMap { + let mcp_config = self.runtime_config(config).await; + configured_mcp_servers(&mcp_config) + } + + /// Returns runtime servers after auth gating and compatibility built-ins. pub async fn effective_servers( &self, config: &Config, auth: Option<&CodexAuth>, ) -> HashMap { - let mcp_config = config.to_mcp_config(self.plugins_manager.as_ref()).await; + let mcp_config = self.runtime_config(config).await; effective_mcp_servers(&mcp_config, auth) } + /// Returns provenance for plugin-owned servers in the configured view. pub async fn tool_plugin_provenance(&self, config: &Config) -> ToolPluginProvenance { let mcp_config = config.to_mcp_config(self.plugins_manager.as_ref()).await; collect_tool_plugin_provenance(&mcp_config) } + + async fn contributions(&self, config: &Config) -> Vec { + let mut contributions = Vec::new(); + for contributor in self.extensions.mcp_server_contributors() { + contributions.extend(contributor.contribute(config).await); + } + contributions + } + + fn apply_to_configured_servers( + contributions: &[McpServerContribution], + servers: &mut HashMap, + ) { + for contribution in contributions { + match contribution { + McpServerContribution::Set { name, config } => { + servers.insert(name.clone(), config.as_ref().clone()); + } + McpServerContribution::Remove { name } => { + servers.remove(name); + } + } + } + } } diff --git a/codex-rs/core/src/mcp_skill_dependencies.rs b/codex-rs/core/src/mcp_skill_dependencies.rs index 936f1b5fb5..304deb500d 100644 --- a/codex-rs/core/src/mcp_skill_dependencies.rs +++ b/codex-rs/core/src/mcp_skill_dependencies.rs @@ -56,7 +56,7 @@ pub(crate) async fn maybe_prompt_and_install_mcp_dependencies( let installed = sess .services .mcp_manager - .configured_servers(config.as_ref()) + .runtime_servers(config.as_ref()) .await; let missing = collect_missing_mcp_dependencies(mentioned_skills, &installed); if missing.is_empty() { @@ -98,7 +98,7 @@ pub(crate) async fn maybe_install_mcp_dependencies( } let codex_home = config.codex_home.clone(); - let installed = sess.services.mcp_manager.configured_servers(config).await; + let installed = sess.services.mcp_manager.runtime_servers(config).await; let missing = collect_missing_mcp_dependencies(mentioned_skills, &installed); if missing.is_empty() { return; @@ -190,16 +190,22 @@ pub(crate) async fn maybe_install_mcp_dependencies( } } - // Refresh from the config-backed merged MCP map (global + repo + managed) - // and overlay the updated global servers so we don't drop repo-scoped - // servers. Runtime additions such as built-ins are rebuilt by the refresh - // path from the current config. - let mut refresh_servers = sess.services.mcp_manager.configured_servers(config).await; + let mut refresh_config = config.clone(); + let mut configured_servers = config.mcp_servers.get().clone(); for (name, server_config) in &servers { - refresh_servers + configured_servers .entry(name.clone()) .or_insert_with(|| server_config.clone()); } + if let Err(err) = refresh_config.mcp_servers.set(configured_servers) { + warn!("failed to refresh MCP dependencies for mentioned skills: {err}"); + return; + } + let refresh_servers = sess + .services + .mcp_manager + .runtime_servers(&refresh_config) + .await; sess.refresh_mcp_servers_now( turn_context, refresh_servers, diff --git a/codex-rs/core/src/session/mcp.rs b/codex-rs/core/src/session/mcp.rs index cc018343eb..f2643d0e5c 100644 --- a/codex-rs/core/src/session/mcp.rs +++ b/codex-rs/core/src/session/mcp.rs @@ -311,8 +311,10 @@ impl Session { ) { let auth = self.services.auth_manager.auth().await; let config = self.get_config().await; - let mcp_config = config - .to_mcp_config(self.services.plugins_manager.as_ref()) + let mcp_config = self + .services + .mcp_manager + .runtime_config(config.as_ref()) .await; let tool_plugin_provenance = self .services diff --git a/codex-rs/core/src/thread_manager.rs b/codex-rs/core/src/thread_manager.rs index a0c1d63d71..d5f54d555e 100644 --- a/codex-rs/core/src/thread_manager.rs +++ b/codex-rs/core/src/thread_manager.rs @@ -272,7 +272,10 @@ impl ThreadManager { codex_home.to_path_buf(), restriction_product, )); - let mcp_manager = Arc::new(McpManager::new(Arc::clone(&plugins_manager))); + let mcp_manager = Arc::new(McpManager::new_with_extensions( + Arc::clone(&plugins_manager), + Arc::clone(&extensions), + )); let skills_manager = Arc::new(SkillsManager::new_with_restriction_product( codex_home, config.bundled_skills_enabled(), diff --git a/codex-rs/ext/extension-api/Cargo.toml b/codex-rs/ext/extension-api/Cargo.toml index 5497dc5439..2e155b430e 100644 --- a/codex-rs/ext/extension-api/Cargo.toml +++ b/codex-rs/ext/extension-api/Cargo.toml @@ -15,6 +15,7 @@ workspace = true [dependencies] async-trait = { workspace = true } +codex-config = { workspace = true } codex-context-fragments = { workspace = true } codex-protocol = { workspace = true } codex-tools = { workspace = true } diff --git a/codex-rs/ext/extension-api/src/contributors.rs b/codex-rs/ext/extension-api/src/contributors.rs index 8706e8ee7a..0403294371 100644 --- a/codex-rs/ext/extension-api/src/contributors.rs +++ b/codex-rs/ext/extension-api/src/contributors.rs @@ -10,12 +10,14 @@ use codex_tools::ToolExecutor; use crate::ExtensionData; +mod mcp; mod prompt; mod thread_lifecycle; mod tool_lifecycle; mod turn_input; mod turn_lifecycle; +pub use mcp::McpServerContribution; pub use prompt::PromptFragment; pub use prompt::PromptSlot; pub use thread_lifecycle::ThreadIdleInput; @@ -34,6 +36,18 @@ pub use turn_lifecycle::TurnErrorInput; pub use turn_lifecycle::TurnStartInput; pub use turn_lifecycle::TurnStopInput; +/// Extension contribution that resolves runtime MCP servers from host config. +/// +/// Contributors run in registration order. Later contributions for the same +/// name replace earlier ones. Implementations must contribute only names they +/// own and must apply any source-specific policy before returning a server. +/// Plugin-owned servers and their provenance continue to be resolved by the +/// plugin manager until that ownership moves into an extension explicitly. +#[async_trait::async_trait] +pub trait McpServerContributor: Send + Sync { + async fn contribute(&self, config: &C) -> Vec; +} + /// Extension contribution that adds prompt fragments during prompt assembly. pub trait ContextContributor: Send + Sync { fn contribute<'a>( diff --git a/codex-rs/ext/extension-api/src/contributors/mcp.rs b/codex-rs/ext/extension-api/src/contributors/mcp.rs new file mode 100644 index 0000000000..a55715692b --- /dev/null +++ b/codex-rs/ext/extension-api/src/contributors/mcp.rs @@ -0,0 +1,22 @@ +use codex_config::McpServerConfig; + +/// One extension-owned overlay for the runtime MCP server configuration. +#[derive(Clone, Debug)] +pub enum McpServerContribution { + /// Adds or replaces a named MCP server. + Set { + name: String, + config: Box, + }, + /// Removes a named MCP server. + Remove { name: String }, +} + +impl McpServerContribution { + /// Returns the stable server name owned by this contribution. + pub fn name(&self) -> &str { + match self { + Self::Set { name, .. } | Self::Remove { name } => name, + } + } +} diff --git a/codex-rs/ext/extension-api/src/lib.rs b/codex-rs/ext/extension-api/src/lib.rs index 7d8d95d139..8b269f31d1 100644 --- a/codex-rs/ext/extension-api/src/lib.rs +++ b/codex-rs/ext/extension-api/src/lib.rs @@ -31,6 +31,8 @@ pub use codex_tools::parse_tool_input_schema_without_compaction; pub use contributors::ApprovalReviewContributor; pub use contributors::ConfigContributor; pub use contributors::ContextContributor; +pub use contributors::McpServerContribution; +pub use contributors::McpServerContributor; pub use contributors::PromptFragment; pub use contributors::PromptSlot; pub use contributors::ThreadIdleInput; diff --git a/codex-rs/ext/extension-api/src/registry.rs b/codex-rs/ext/extension-api/src/registry.rs index 0849387110..e8cc989337 100644 --- a/codex-rs/ext/extension-api/src/registry.rs +++ b/codex-rs/ext/extension-api/src/registry.rs @@ -7,6 +7,7 @@ use crate::ConfigContributor; use crate::ContextContributor; use crate::ExtensionData; use crate::ExtensionEventSink; +use crate::McpServerContributor; use crate::NoopExtensionEventSink; use crate::ThreadLifecycleContributor; use crate::TokenUsageContributor; @@ -24,6 +25,7 @@ pub struct ExtensionRegistryBuilder { config_contributors: Vec>>, token_usage_contributors: Vec>, context_contributors: Vec>, + mcp_server_contributors: Vec>>, turn_input_contributors: Vec>, tool_contributors: Vec>, tool_lifecycle_contributors: Vec>, @@ -41,6 +43,7 @@ impl Default for ExtensionRegistryBuilder { token_usage_contributors: Vec::new(), approval_review_contributors: Vec::new(), context_contributors: Vec::new(), + mcp_server_contributors: Vec::new(), turn_input_contributors: Vec::new(), tool_contributors: Vec::new(), tool_lifecycle_contributors: Vec::new(), @@ -101,6 +104,11 @@ impl ExtensionRegistryBuilder { self.context_contributors.push(contributor); } + /// Registers one runtime MCP server contributor. + pub fn mcp_server_contributor(&mut self, contributor: Arc>) { + self.mcp_server_contributors.push(contributor); + } + /// Registers one turn-input contributor. pub fn turn_input_contributor(&mut self, contributor: Arc) { self.turn_input_contributors.push(contributor); @@ -131,6 +139,7 @@ impl ExtensionRegistryBuilder { token_usage_contributors: self.token_usage_contributors, approval_review_contributors: self.approval_review_contributors, context_contributors: self.context_contributors, + mcp_server_contributors: self.mcp_server_contributors, turn_input_contributors: self.turn_input_contributors, tool_contributors: self.tool_contributors, tool_lifecycle_contributors: self.tool_lifecycle_contributors, @@ -147,6 +156,7 @@ pub struct ExtensionRegistry { config_contributors: Vec>>, token_usage_contributors: Vec>, context_contributors: Vec>, + mcp_server_contributors: Vec>>, turn_input_contributors: Vec>, tool_contributors: Vec>, tool_lifecycle_contributors: Vec>, @@ -205,6 +215,11 @@ impl ExtensionRegistry { &self.context_contributors } + /// Returns the registered runtime MCP server contributors. + pub fn mcp_server_contributors(&self) -> &[Arc>] { + &self.mcp_server_contributors + } + /// Returns the registered turn-input contributors. pub fn turn_input_contributors(&self) -> &[Arc] { &self.turn_input_contributors diff --git a/codex-rs/ext/mcp/BUILD.bazel b/codex-rs/ext/mcp/BUILD.bazel new file mode 100644 index 0000000000..30aefba692 --- /dev/null +++ b/codex-rs/ext/mcp/BUILD.bazel @@ -0,0 +1,6 @@ +load("//:defs.bzl", "codex_rust_crate") + +codex_rust_crate( + name = "mcp", + crate_name = "codex_mcp_extension", +) diff --git a/codex-rs/ext/mcp/Cargo.toml b/codex-rs/ext/mcp/Cargo.toml new file mode 100644 index 0000000000..66c9ad1727 --- /dev/null +++ b/codex-rs/ext/mcp/Cargo.toml @@ -0,0 +1,29 @@ +[package] +edition.workspace = true +license.workspace = true +name = "codex-mcp-extension" +version.workspace = true + +[lib] +name = "codex_mcp_extension" +path = "src/lib.rs" +test = false +doctest = false + +[lints] +workspace = true + +[dependencies] +async-trait = { workspace = true } +codex-core = { workspace = true } +codex-extension-api = { workspace = true } +codex-features = { workspace = true } +codex-mcp = { workspace = true } + +[dev-dependencies] +codex-config = { workspace = true } +codex-core-plugins = { workspace = true } +codex-login = { workspace = true } +pretty_assertions = { workspace = true } +tempfile = { workspace = true } +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/codex-rs/ext/mcp/src/lib.rs b/codex-rs/ext/mcp/src/lib.rs new file mode 100644 index 0000000000..4a7838ccce --- /dev/null +++ b/codex-rs/ext/mcp/src/lib.rs @@ -0,0 +1,31 @@ +use codex_core::config::Config; +use codex_extension_api::ExtensionRegistryBuilder; +use codex_extension_api::McpServerContribution; +use codex_extension_api::McpServerContributor; +use codex_mcp::CODEX_APPS_MCP_SERVER_NAME; +use codex_mcp::codex_apps_mcp_server_config; + +struct HostedAppsMcpExtension; + +#[async_trait::async_trait] +impl McpServerContributor for HostedAppsMcpExtension { + async fn contribute(&self, config: &Config) -> Vec { + let name = CODEX_APPS_MCP_SERVER_NAME.to_string(); + if !config.features.enabled(codex_features::Feature::Apps) { + return vec![McpServerContribution::Remove { name }]; + } + + vec![McpServerContribution::Set { + name, + config: Box::new(codex_apps_mcp_server_config( + &config.chatgpt_base_url, + config.apps_mcp_path_override.as_deref(), + config.apps_mcp_product_sku.as_deref(), + )), + }] + } +} + +pub fn install(builder: &mut ExtensionRegistryBuilder) { + builder.mcp_server_contributor(std::sync::Arc::new(HostedAppsMcpExtension)); +} diff --git a/codex-rs/ext/mcp/tests/hosted_apps_mcp.rs b/codex-rs/ext/mcp/tests/hosted_apps_mcp.rs new file mode 100644 index 0000000000..0d025df954 --- /dev/null +++ b/codex-rs/ext/mcp/tests/hosted_apps_mcp.rs @@ -0,0 +1,93 @@ +use std::sync::Arc; + +use codex_config::McpServerTransportConfig; +use codex_core::McpManager; +use codex_core::config::Config; +use codex_core::config::ConfigBuilder; +use codex_core_plugins::PluginsManager; +use codex_extension_api::ExtensionRegistryBuilder; +use codex_login::CodexAuth; +use codex_mcp::CODEX_APPS_MCP_SERVER_NAME; +use pretty_assertions::assert_eq; + +#[tokio::test] +async fn contributes_hosted_apps_mcp_without_an_executor() -> Result<(), Box> +{ + let codex_home = tempfile::tempdir()?; + let config = ConfigBuilder::default() + .codex_home(codex_home.path().to_path_buf()) + .fallback_cwd(Some(codex_home.path().to_path_buf())) + .cli_overrides(vec![ + ("features.apps".to_string(), true.into()), + ("features.apps_mcp_path_override".to_string(), true.into()), + ("chatgpt_base_url".to_string(), "https://chatgpt.com".into()), + ]) + .build() + .await?; + let auth = CodexAuth::create_dummy_chatgpt_auth_for_testing(); + let manager = installed_manager(&config); + + let runtime_config = manager.runtime_config(&config).await; + assert!(!runtime_config.legacy_apps_mcp_loader_enabled); + let servers = manager.effective_servers(&config, Some(&auth)).await; + let server = servers + .get(CODEX_APPS_MCP_SERVER_NAME) + .and_then(|server| server.configured_config()) + .ok_or("Apps MCP should be contributed as a configured server")?; + let McpServerTransportConfig::StreamableHttp { url, .. } = &server.transport else { + panic!("Apps MCP should use streamable HTTP"); + }; + assert_eq!(url, "https://chatgpt.com/backend-api/ps/mcp"); + + Ok(()) +} + +#[tokio::test] +async fn hosted_apps_mcp_requires_chatgpt_auth() -> Result<(), Box> { + let codex_home = tempfile::tempdir()?; + let config = ConfigBuilder::default() + .codex_home(codex_home.path().to_path_buf()) + .fallback_cwd(Some(codex_home.path().to_path_buf())) + .cli_overrides(vec![("features.apps".to_string(), true.into())]) + .build() + .await?; + let auth = CodexAuth::from_api_key("test"); + let manager = installed_manager(&config); + + let servers = manager.effective_servers(&config, Some(&auth)).await; + assert!(!servers.contains_key(CODEX_APPS_MCP_SERVER_NAME)); + + Ok(()) +} + +#[tokio::test] +async fn disabled_apps_remove_reserved_server_config() -> Result<(), Box> { + let codex_home = tempfile::tempdir()?; + let config = ConfigBuilder::default() + .codex_home(codex_home.path().to_path_buf()) + .fallback_cwd(Some(codex_home.path().to_path_buf())) + .cli_overrides(vec![ + ("features.apps".to_string(), false.into()), + ( + "mcp_servers.codex_apps.url".to_string(), + "https://example.com/mcp".into(), + ), + ]) + .build() + .await?; + let manager = installed_manager(&config); + + let servers = manager.runtime_servers(&config).await; + + assert!(!servers.contains_key(CODEX_APPS_MCP_SERVER_NAME)); + Ok(()) +} + +fn installed_manager(config: &Config) -> McpManager { + let mut builder = ExtensionRegistryBuilder::new(); + codex_mcp_extension::install(&mut builder); + McpManager::new_with_extensions( + Arc::new(PluginsManager::new(config.codex_home.to_path_buf())), + Arc::new(builder.build()), + ) +} From 9e3081be9672c65f8a0cd958719065f49f47d839 Mon Sep 17 00:00:00 2001 From: "Adam Perry @ OpenAI" Date: Tue, 9 Jun 2026 13:51:57 -0700 Subject: [PATCH 12/47] [codex] Characterize global instruction lifecycle (#26830) ## Why Global instruction behavior spans thread creation, resume, forks, subagents, and compaction. Characterization coverage is needed before changing those semantics so preserved history can be distinguished from newly loaded configuration. ## What changed - Extends the existing `agents_md` suite with fresh-thread, warning, resume, fork, and subagent lifecycle coverage. - Extends the existing `compact` suite with manual, mid-turn, and remote-v2 compaction coverage. - Asserts rendered instruction fragments, reported source paths, and structured request history before and after instruction-file mutations. --- codex-rs/core/tests/suite/agents_md.rs | 602 ++++++++++++++++++++++++- codex-rs/core/tests/suite/compact.rs | 384 ++++++++++++++++ 2 files changed, 985 insertions(+), 1 deletion(-) diff --git a/codex-rs/core/tests/suite/agents_md.rs b/codex-rs/core/tests/suite/agents_md.rs index 3fa2136ecb..88f014f757 100644 --- a/codex-rs/core/tests/suite/agents_md.rs +++ b/codex-rs/core/tests/suite/agents_md.rs @@ -1,18 +1,46 @@ use anyhow::Result; +use anyhow::anyhow; +use codex_core::ForkSnapshot; use codex_exec_server::CreateDirectoryOptions; +use codex_features::Feature; +use codex_protocol::protocol::EventMsg; +use codex_protocol::protocol::Op; +use codex_protocol::user_input::UserInput; use codex_utils_absolute_path::AbsolutePathBuf; +use core_test_support::PathBufExt; use core_test_support::create_directory_symlink; +use core_test_support::load_default_config_for_test; +use core_test_support::responses; use core_test_support::responses::ev_completed; use core_test_support::responses::ev_response_created; use core_test_support::responses::mount_sse_once; use core_test_support::responses::sse; use core_test_support::responses::start_mock_server; +use core_test_support::skip_if_no_network; use core_test_support::test_codex::TestCodexBuilder; use core_test_support::test_codex::test_codex; +use core_test_support::wait_for_event; +use core_test_support::wait_for_event_match; use pretty_assertions::assert_eq; +use serde_json::json; use std::sync::Arc; +use std::time::Duration; use tempfile::TempDir; +const GLOBAL_AGENTS_FILENAME: &str = "AGENTS.md"; +const GLOBAL_AGENTS_OVERRIDE_FILENAME: &str = "AGENTS.override.md"; +const GLOBAL_INSTRUCTIONS: &str = "global instructions"; +const NEW_GLOBAL_INSTRUCTIONS: &str = "new global instructions"; +const NEW_PROJECT_INSTRUCTIONS: &str = "new project instructions"; +const OLD_GLOBAL_INSTRUCTIONS: &str = "old global instructions"; +const PROJECT_INSTRUCTIONS: &str = "project instructions"; +const PROJECT_SEPARATOR: &str = "--- project-doc ---"; +const SPAWN_CALL_ID: &str = "spawn-global-instructions-child"; +const SPAWN_CHILD_PROMPT: &str = "inspect inherited global instructions"; +const SPAWN_FRESH_PARENT_PROMPT: &str = "spawn a child with fresh context"; +const SPAWN_PARENT_PROMPT: &str = "spawn a child with the parent context"; +const SPAWN_SEED_PROMPT: &str = "seed parent history"; + async fn agents_instructions(mut builder: TestCodexBuilder) -> Result { let server = start_mock_server().await; let resp_mock = mount_sse_once( @@ -32,6 +60,52 @@ async fn agents_instructions(mut builder: TestCodexBuilder) -> Result { .ok_or_else(|| anyhow::anyhow!("instructions message not found")) } +fn write_global_file( + home: &TempDir, + filename: &str, + contents: impl AsRef<[u8]>, +) -> Result { + let path = home.path().join(filename); + std::fs::write(&path, contents)?; + Ok(path.abs()) +} + +fn instruction_fragments(request: &responses::ResponsesRequest) -> Vec { + request + .message_input_texts("user") + .into_iter() + .filter(|text| text.starts_with("# AGENTS.md instructions for ")) + .collect() +} + +fn expected_instruction_fragment(cwd: &AbsolutePathBuf, contents: &str) -> String { + let cwd = cwd.as_path().display(); + format!("# AGENTS.md instructions for {cwd}\n\n\n{contents}\n") +} + +fn assert_single_instruction_fragment(request: &responses::ResponsesRequest, expected: &str) { + assert_eq!(instruction_fragments(request), vec![expected.to_string()]); +} + +fn request_body_contains(request: &wiremock::Request, text: &str) -> bool { + let is_zstd = request + .headers + .get("content-encoding") + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| { + value + .split(',') + .any(|entry| entry.trim().eq_ignore_ascii_case("zstd")) + }); + let body = if is_zstd { + zstd::stream::decode_all(std::io::Cursor::new(&request.body)).ok() + } else { + Some(request.body.clone()) + }; + body.and_then(|body| String::from_utf8(body).ok()) + .is_some_and(|body| body.contains(text)) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn agents_override_is_preferred_over_agents_md() -> Result<()> { let instructions = @@ -250,7 +324,7 @@ async fn selected_environment_sources_match_model_visible_instructions() -> Resu }); let test = builder.build_with_remote_env(&server).await?; let project_agents = test.config.cwd.join("AGENTS.md"); - let global_agents = AbsolutePathBuf::try_from(global_agents).expect("absolute path"); + let global_agents = global_agents.abs(); assert_eq!( test.codex.instruction_sources().await, @@ -268,3 +342,529 @@ async fn selected_environment_sources_match_model_visible_instructions() -> Resu Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn fresh_thread_composes_global_before_project_and_reports_sources() -> Result<()> { + // Set up one global source, one project source, and two ordinary model turns. + let server = responses::start_mock_server().await; + let response_mock = responses::mount_sse_sequence( + &server, + vec![ + responses::sse(vec![ + responses::ev_response_created("response-1"), + responses::ev_completed("response-1"), + ]), + responses::sse(vec![ + responses::ev_response_created("response-2"), + responses::ev_completed("response-2"), + ]), + ], + ) + .await; + let home = Arc::new(TempDir::new()?); + let global_source = + write_global_file(home.as_ref(), GLOBAL_AGENTS_FILENAME, GLOBAL_INSTRUCTIONS)?; + + let mut builder = test_codex() + .with_home(Arc::clone(&home)) + .with_workspace_setup(|cwd, fs| async move { + fs.write_file( + &cwd.join("AGENTS.md"), + PROJECT_INSTRUCTIONS.as_bytes().to_vec(), + /*sandbox*/ None, + ) + .await?; + Ok(()) + }); + let test = builder.build_with_remote_env(&server).await?; + let project_source = test.config.cwd.join(GLOBAL_AGENTS_FILENAME); + let creation_sources = vec![global_source.clone(), project_source.clone()]; + + // Confirm the thread records both creation-time sources in composition order. + assert_eq!(test.codex.instruction_sources().await, creation_sources); + + // Materialize the initial snapshot, then rewrite both selected files in place before another + // ordinary turn. + test.submit_turn("first turn").await?; + let rewritten_global_source = write_global_file( + home.as_ref(), + GLOBAL_AGENTS_FILENAME, + NEW_GLOBAL_INSTRUCTIONS, + )?; + test.fs() + .write_file( + &project_source, + NEW_PROJECT_INSTRUCTIONS.as_bytes().to_vec(), + /*sandbox*/ None, + ) + .await?; + assert_eq!( + rewritten_global_source, global_source, + "same-path mutation should retain the selected global source path" + ); + test.submit_turn("second turn").await?; + + // Assert the running thread keeps its original rendering and structured prefix even though + // both files at the reported source paths now contain different text. + let requests = response_mock.requests(); + assert_eq!(requests.len(), 2); + let expected_contents = + format!("{GLOBAL_INSTRUCTIONS}\n\n{PROJECT_SEPARATOR}\n\n{PROJECT_INSTRUCTIONS}"); + let expected_fragment = expected_instruction_fragment(&test.config.cwd, &expected_contents); + let fragments = instruction_fragments(&requests[0]); + assert_eq!(fragments, vec![expected_fragment.clone()]); + assert_single_instruction_fragment(&requests[1], &expected_fragment); + let rendered = fragments + .into_iter() + .next() + .ok_or_else(|| anyhow!("expected one rendered instruction fragment"))?; + let global_position = rendered.find(GLOBAL_INSTRUCTIONS).ok_or_else(|| { + anyhow!( + "expected rendered instructions to contain {GLOBAL_INSTRUCTIONS:?}; observed: {rendered}" + ) + })?; + let project_position = rendered.find(PROJECT_INSTRUCTIONS).ok_or_else(|| { + anyhow!( + "expected rendered instructions to contain {PROJECT_INSTRUCTIONS:?}; observed: {rendered}" + ) + })?; + assert!( + global_position < project_position, + "global instructions should precede project instructions: {rendered}" + ); + assert!( + rendered.contains(PROJECT_SEPARATOR), + "expected rendered instructions to contain {PROJECT_SEPARATOR:?}; observed: {rendered}" + ); + assert_eq!( + test.codex.instruction_sources().await, + creation_sources, + "ordinary turns retain the creation-time source list" + ); + let first_input = requests[0].input(); + let second_input = requests[1].input(); + assert_eq!( + second_input.get(..first_input.len()), + Some(first_input.as_slice()), + "the ordinary second turn should retain the cached prefix" + ); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn global_loading_warning_surfaces_during_thread_creation() -> Result<()> { + // Set up a malformed global instruction file and one model response. + let server = responses::start_mock_server().await; + let response_mock = responses::mount_sse_once( + &server, + responses::sse(vec![ + responses::ev_response_created("warning-response"), + responses::ev_completed("warning-response"), + ]), + ) + .await; + let home = Arc::new(TempDir::new()?); + let source = write_global_file( + home.as_ref(), + GLOBAL_AGENTS_FILENAME, + b"global\xFFinstructions", + )?; + + // Create the thread, capture its load warning, and submit one turn for rendered output. + let mut builder = test_codex().with_home(home); + let test = builder.build(&server).await?; + let warning = wait_for_event_match(&test.codex, |event| match event { + EventMsg::Warning(warning) + if warning + .message + .contains(source.as_path().display().to_string().as_str()) => + { + Some(warning.message.clone()) + } + _ => None, + }) + .await; + test.submit_turn("inspect lossy global instructions") + .await?; + + // Assert the source is reported, the warning is specific, and rendering is lossily decoded. + assert_eq!(test.codex.instruction_sources().await, vec![source.clone()]); + assert!( + warning.contains("invalid UTF-8"), + "expected warning to contain \"invalid UTF-8\"; observed: {warning}" + ); + let expected_fragment = + expected_instruction_fragment(&test.config.cwd, "global\u{FFFD}instructions"); + assert_single_instruction_fragment(&response_mock.single_request(), &expected_fragment); + + Ok(()) +} + +// TODO(anp): Align cold-resume instruction sources with the historical instructions replayed to +// the model so the API source list and model-visible context describe the same files. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn cold_resume_replays_rendered_instructions_but_reports_current_config_sources() -> Result<()> +{ + // Set up an initial turn and a later cold-resumed turn against the same rollout. + let server = responses::start_mock_server().await; + let response_mock = responses::mount_sse_sequence( + &server, + vec![ + responses::sse(vec![ + responses::ev_response_created("initial-response"), + responses::ev_completed("initial-response"), + ]), + responses::sse(vec![ + responses::ev_response_created("resumed-response"), + responses::ev_completed("resumed-response"), + ]), + ], + ) + .await; + let home = Arc::new(TempDir::new()?); + let old_source = write_global_file( + home.as_ref(), + GLOBAL_AGENTS_FILENAME, + OLD_GLOBAL_INSTRUCTIONS, + )?; + + // Create the initial thread and persist its creation-time instruction snapshot. + let mut initial_builder = test_codex().with_home(Arc::clone(&home)); + let initial = initial_builder.build(&server).await?; + + // Assert the pre-resume thread reports the source used to create its snapshot. + assert_eq!( + initial.codex.instruction_sources().await, + vec![old_source.clone()], + "initial thread reports the creation-time global source" + ); + initial.submit_turn("persist instructions").await?; + let rollout_path = initial + .session_configured + .rollout_path + .clone() + .expect("rollout path"); + initial.codex.submit(Op::Shutdown).await?; + wait_for_event(&initial.codex, |event| { + matches!(event, EventMsg::ShutdownComplete) + }) + .await; + + // Add a preferred override source, then cold-resume with freshly loaded configuration. + let new_source = write_global_file( + home.as_ref(), + GLOBAL_AGENTS_OVERRIDE_FILENAME, + NEW_GLOBAL_INSTRUCTIONS, + )?; + assert_ne!(old_source, new_source); + let mut resume_builder = test_codex().with_home(Arc::clone(&home)); + let resumed = resume_builder + .resume(&server, Arc::clone(&home), rollout_path) + .await?; + + // Assert the API reports the new source while model history replays the old structured prefix. + assert_eq!( + resumed.codex.instruction_sources().await, + vec![new_source], + "resume reports sources from the newly loaded config" + ); + + resumed.submit_turn("continue resumed thread").await?; + + let requests = response_mock.requests(); + assert_eq!(requests.len(), 2); + let initial_input = requests[0].input(); + let resumed_input = requests[1].input(); + assert_eq!( + resumed_input.get(..initial_input.len()), + Some(initial_input.as_slice()), + "cold resume should replay the original structured input prefix" + ); + let expected_fragment = + expected_instruction_fragment(&initial.config.cwd, OLD_GLOBAL_INSTRUCTIONS); + assert_single_instruction_fragment(&requests[0], &expected_fragment); + assert_single_instruction_fragment(&requests[1], &expected_fragment); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn fork_replays_rendered_instructions_from_shared_history() -> Result<()> { + // Set up a parent turn and a later fork turn against the parent's rollout. + let server = responses::start_mock_server().await; + let response_mock = responses::mount_sse_sequence( + &server, + vec![ + responses::sse(vec![ + responses::ev_response_created("parent-response"), + responses::ev_completed("parent-response"), + ]), + responses::sse(vec![ + responses::ev_response_created("fork-response"), + responses::ev_completed("fork-response"), + ]), + ], + ) + .await; + let home = Arc::new(TempDir::new()?); + let source = write_global_file( + home.as_ref(), + GLOBAL_AGENTS_FILENAME, + OLD_GLOBAL_INSTRUCTIONS, + )?; + + // Create the parent and persist its creation-time instruction snapshot. + let mut builder = test_codex().with_home(Arc::clone(&home)); + let parent = builder.build(&server).await?; + + // Assert the parent reports the source used to create its snapshot. + assert_eq!( + parent.codex.instruction_sources().await, + vec![source.clone()], + "parent reports the creation-time global source" + ); + parent.submit_turn("persist instructions").await?; + parent.codex.ensure_rollout_materialized().await; + parent.codex.flush_rollout().await?; + let rollout_path = parent.codex.rollout_path().expect("rollout path"); + + // Add a preferred override source, then fork with freshly loaded configuration. + let new_source = write_global_file( + home.as_ref(), + GLOBAL_AGENTS_OVERRIDE_FILENAME, + NEW_GLOBAL_INSTRUCTIONS, + )?; + assert_ne!(source, new_source); + let mut fork_config = load_default_config_for_test(home.as_ref()).await; + fork_config.cwd = parent.config.cwd.clone(); + fork_config.model = parent.config.model.clone(); + fork_config.model_provider = parent.config.model_provider.clone(); + fork_config.model_catalog = parent.config.model_catalog.clone(); + fork_config.codex_self_exe = parent.config.codex_self_exe.clone(); + let forked = parent + .thread_manager + .fork_thread( + ForkSnapshot::Interrupted, + fork_config, + rollout_path, + /*thread_source*/ None, + /*parent_trace*/ None, + ) + .await?; + + // Assert the fork reports the new source before issuing its first turn. + assert_eq!( + forked.thread.instruction_sources().await, + vec![new_source], + "fork config should reflect the newly loaded global source" + ); + + forked + .thread + .submit(Op::UserInput { + items: vec![UserInput::Text { + text: "continue fork".to_string(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + responsesapi_client_metadata: None, + additional_context: Default::default(), + thread_settings: Default::default(), + }) + .await?; + wait_for_event(&forked.thread, |event| { + matches!(event, EventMsg::TurnComplete(_)) + }) + .await; + + // Assert the forked model request replays the parent's exact structured history. + let requests = response_mock.requests(); + assert_eq!(requests.len(), 2); + let parent_input = requests[0].input(); + let fork_input = requests[1].input(); + assert_eq!( + fork_input.get(..parent_input.len()), + Some(parent_input.as_slice()), + "fork should replay the parent's original structured input prefix" + ); + let expected_fragment = + expected_instruction_fragment(&parent.config.cwd, OLD_GLOBAL_INSTRUCTIONS); + assert_single_instruction_fragment(&requests[0], &expected_fragment); + assert_single_instruction_fragment(&requests[1], &expected_fragment); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn forked_subagent_replays_one_creation_time_global_instruction_fragment() -> Result<()> { + skip_if_no_network!(Ok(())); + run_subagent_global_instruction_case(/*fork_context*/ true).await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn fresh_subagent_uses_creation_time_instructions_without_parent_history() -> Result<()> { + skip_if_no_network!(Ok(())); + run_subagent_global_instruction_case(/*fork_context*/ false).await +} + +async fn run_subagent_global_instruction_case(fork_context: bool) -> Result<()> { + // Set up matched responses for the parent seed, spawn call, child turn, and parent follow-up. + let server = responses::start_mock_server().await; + let parent_prompt = if fork_context { + SPAWN_PARENT_PROMPT + } else { + SPAWN_FRESH_PARENT_PROMPT + }; + let seed_mock = responses::mount_sse_once_match( + &server, + |request: &wiremock::Request| request_body_contains(request, SPAWN_SEED_PROMPT), + responses::sse(vec![ + responses::ev_response_created("seed-response"), + responses::ev_assistant_message("seed-message", "seeded"), + responses::ev_completed("seed-response"), + ]), + ) + .await; + let spawn_args = serde_json::to_string(&json!({ + "message": SPAWN_CHILD_PROMPT, + "fork_context": fork_context, + }))?; + let spawn_mock = responses::mount_sse_once_match( + &server, + move |request: &wiremock::Request| request_body_contains(request, parent_prompt), + responses::sse(vec![ + responses::ev_response_created("spawn-response"), + responses::ev_function_call_with_namespace( + SPAWN_CALL_ID, + "multi_agent_v1", + "spawn_agent", + &spawn_args, + ), + responses::ev_completed("spawn-response"), + ]), + ) + .await; + let child_mock = responses::mount_sse_once_match( + &server, + |request: &wiremock::Request| { + request_body_contains(request, SPAWN_CHILD_PROMPT) + && !request_body_contains(request, SPAWN_CALL_ID) + }, + responses::sse(vec![ + responses::ev_response_created("child-response"), + responses::ev_assistant_message("child-message", "done"), + responses::ev_completed("child-response"), + ]), + ) + .await; + responses::mount_sse_once_match( + &server, + |request: &wiremock::Request| request_body_contains(request, SPAWN_CALL_ID), + responses::sse(vec![ + responses::ev_response_created("spawn-follow-up-response"), + responses::ev_assistant_message("spawn-follow-up-message", "child started"), + responses::ev_completed("spawn-follow-up-response"), + ]), + ) + .await; + + // Create the parent thread, record its source, and seed the history inherited by the child. + let home = Arc::new(TempDir::new()?); + let source = write_global_file( + home.as_ref(), + GLOBAL_AGENTS_FILENAME, + OLD_GLOBAL_INSTRUCTIONS, + )?; + let mut builder = test_codex() + .with_home(Arc::clone(&home)) + .with_config(|config| { + let _ = config.features.enable(Feature::Collab); + let _ = config.features.disable(Feature::EnableRequestCompression); + }); + let test = builder.build(&server).await?; + + // Assert the parent reports the creation-time source before spawning. + assert_eq!( + test.codex.instruction_sources().await, + vec![source.clone()], + "parent reports the creation-time global source before spawning" + ); + test.submit_turn(SPAWN_SEED_PROMPT).await?; + let seed_request = seed_mock.single_request(); + + // Add a preferred override, then spawn a full-history child while observing its thread ID. + let new_source = write_global_file( + home.as_ref(), + GLOBAL_AGENTS_OVERRIDE_FILENAME, + NEW_GLOBAL_INSTRUCTIONS, + )?; + assert_ne!(source, new_source); + let mut created_threads = test.thread_manager.subscribe_thread_created(); + test.submit_turn(parent_prompt).await?; + let child_thread_id = tokio::time::timeout(Duration::from_secs(10), created_threads.recv()) + .await + .map_err(|_| anyhow!("timed out waiting for the subagent thread"))??; + let child_thread = test.thread_manager.get_thread(child_thread_id).await?; + let spawn_request = spawn_mock.single_request(); + let child_request = tokio::time::timeout(Duration::from_secs(10), async { + loop { + if let Some(request) = child_mock.requests().into_iter().find(|request| { + request + .message_input_texts("user") + .iter() + .any(|text| text == SPAWN_CHILD_PROMPT) + }) { + break request; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .map_err(|_| anyhow!("timed out waiting for the subagent request"))?; + + // Assert parent and child report and render the parent's creation-time snapshot exactly once. + let expected_fragment = + expected_instruction_fragment(&test.config.cwd, OLD_GLOBAL_INSTRUCTIONS); + assert_single_instruction_fragment(&seed_request, &expected_fragment); + assert_single_instruction_fragment(&spawn_request, &expected_fragment); + assert_single_instruction_fragment(&child_request, &expected_fragment); + assert_eq!( + test.codex.instruction_sources().await, + vec![source.clone()], + "running parent retains the creation-time global source after spawning" + ); + assert_eq!( + child_thread.instruction_sources().await, + vec![source], + "subagent reports the parent's creation-time source" + ); + if fork_context { + let seed_input = seed_request.input(); + let child_input = child_request.input(); + assert_eq!( + child_input.get(..seed_input.len()), + Some(seed_input.as_slice()), + "forked subagent should replay the parent's original structured input prefix" + ); + } else { + let child_user_texts = child_request.message_input_texts("user"); + assert_eq!( + child_user_texts + .iter() + .filter(|text| text.as_str() == SPAWN_SEED_PROMPT) + .count(), + 0, + "fresh-context subagent should omit parent user history; observed: {child_user_texts:?}" + ); + assert_eq!( + child_user_texts + .iter() + .filter(|text| text.as_str() == SPAWN_CHILD_PROMPT) + .count(), + 1, + "fresh-context subagent should contain its own prompt exactly once; observed: {child_user_texts:?}" + ); + } + + Ok(()) +} diff --git a/codex-rs/core/tests/suite/compact.rs b/codex-rs/core/tests/suite/compact.rs index ac497aff3b..339cadda4e 100644 --- a/codex-rs/core/tests/suite/compact.rs +++ b/codex-rs/core/tests/suite/compact.rs @@ -1,4 +1,6 @@ #![allow(clippy::expect_used)] +use anyhow::Result; +use anyhow::anyhow; use codex_core::compact::SUMMARIZATION_PROMPT; use codex_core::compact::SUMMARY_PREFIX; use codex_core::config::Config; @@ -23,11 +25,13 @@ use codex_protocol::protocol::RolloutItem; use codex_protocol::protocol::RolloutLine; use codex_protocol::protocol::WarningEvent; use codex_protocol::user_input::UserInput; +use codex_utils_absolute_path::AbsolutePathBuf; use core_test_support::PathBufExt; use core_test_support::context_snapshot; use core_test_support::context_snapshot::ContextSnapshotOptions; use core_test_support::context_snapshot::ContextSnapshotRenderMode; use core_test_support::hooks::trust_discovered_hooks; +use core_test_support::responses; use core_test_support::responses::ev_reasoning_item; use core_test_support::responses::mount_models_once; use core_test_support::skip_if_no_network; @@ -57,6 +61,8 @@ use serde_json::Value; use serde_json::json; use std::fs; use std::path::Path; +use std::sync::Arc; +use tempfile::TempDir; use wiremock::MockServer; // --- Test helpers ----------------------------------------------------------- @@ -78,6 +84,11 @@ const DUMMY_CALL_ID: &str = "call-multi-auto"; const FUNCTION_CALL_LIMIT_MSG: &str = "function call limit push"; const POST_AUTO_USER_MSG: &str = "post auto follow-up"; const PRETURN_CONTEXT_DIFF_CWD: &str = "/tmp/PRETURN_CONTEXT_DIFF_CWD"; +const GLOBAL_AGENTS_FILENAME: &str = "AGENTS.md"; +const GLOBAL_AGENTS_OVERRIDE_FILENAME: &str = "AGENTS.override.md"; +const NEW_GLOBAL_INSTRUCTIONS: &str = "new global instructions"; +const OLD_GLOBAL_INSTRUCTIONS: &str = "old global instructions"; +const REMOTE_V2_SUMMARY: &str = "global-instructions-remote-v2-summary"; pub(super) const COMPACT_WARNING_MESSAGE: &str = "Heads up: Long threads and multiple compactions can cause the model to be less accurate. Start a new thread when possible to keep threads small and targeted."; @@ -267,6 +278,92 @@ fn non_openai_model_provider(server: &MockServer) -> ModelProviderInfo { provider } +fn write_global_file( + home: &TempDir, + filename: &str, + contents: impl AsRef<[u8]>, +) -> Result { + let path = home.path().join(filename); + std::fs::write(&path, contents)?; + Ok(path.abs()) +} + +fn instruction_fragments(request: &responses::ResponsesRequest) -> Vec { + request + .message_input_texts("user") + .into_iter() + .filter(|text| text.starts_with("# AGENTS.md instructions for ")) + .collect() +} + +fn instruction_fragments_in_items(items: &[Value]) -> Vec { + items + .iter() + .filter(|item| { + item.get("type").and_then(Value::as_str) == Some("message") + && item.get("role").and_then(Value::as_str) == Some("user") + }) + .filter_map(|item| item.get("content").and_then(Value::as_array)) + .flatten() + .filter_map(|span| span.get("text").and_then(Value::as_str)) + .filter(|text| text.starts_with("# AGENTS.md instructions for ")) + .map(str::to_string) + .collect() +} + +fn expected_instruction_fragment(cwd: &AbsolutePathBuf, contents: &str) -> String { + let cwd = cwd.as_path().display(); + format!("# AGENTS.md instructions for {cwd}\n\n\n{contents}\n") +} + +fn assert_single_instruction_fragment(request: &responses::ResponsesRequest, expected: &str) { + assert_eq!(instruction_fragments(request), vec![expected.to_string()]); +} + +fn replacement_history_from_rollout(path: &Path) -> Result> { + let rollout_text = fs::read_to_string(path)?; + let mut replacement_history = None; + for line in rollout_text + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + { + let entry: RolloutLine = serde_json::from_str(line)?; + if let RolloutItem::Compacted(compacted) = entry.item + && let Some(items) = compacted.replacement_history + { + replacement_history = Some( + items + .into_iter() + .map(serde_json::to_value) + .collect::, _>>()?, + ); + } + } + replacement_history.ok_or_else(|| anyhow!("expected rollout replacement history")) +} + +fn remote_v2_compaction_response() -> String { + responses::sse(vec![ + json!({ + "type": "response.output_item.done", + "item": { + "type": "compaction", + "encrypted_content": REMOTE_V2_SUMMARY, + } + }), + responses::ev_completed("remote-v2-compact-response"), + ]) +} + +fn local_compaction_provider(server: &wiremock::MockServer) -> ModelProviderInfo { + let mut provider = built_in_model_providers(/*openai_base_url*/ None)["openai"].clone(); + provider.name = "OpenAI-compatible test provider".to_string(); + provider.base_url = Some(format!("{}/v1", server.uri())); + provider.supports_websockets = false; + provider +} + fn model_info_with_context_window(slug: &str, context_window: i64) -> ModelInfo { let models_response = bundled_models_response() .unwrap_or_else(|err| panic!("bundled models.json should parse: {err}")); @@ -3998,3 +4095,290 @@ async fn snapshot_request_shape_manual_compact_without_previous_user_messages() ) ); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn manual_compaction_keeps_the_creation_time_global_instructions() -> Result<()> { + // Set up an initial turn, a manual compaction response, and a post-compaction turn. + let server = responses::start_mock_server().await; + let response_mock = responses::mount_sse_sequence( + &server, + vec![ + responses::sse(vec![ + responses::ev_response_created("first-response"), + responses::ev_completed("first-response"), + ]), + responses::sse(vec![ + responses::ev_response_created("compact-response"), + responses::ev_assistant_message("compact-message", "summary"), + responses::ev_completed("compact-response"), + ]), + responses::sse(vec![ + responses::ev_response_created("follow-up-response"), + responses::ev_completed("follow-up-response"), + ]), + ], + ) + .await; + let home = Arc::new(TempDir::new()?); + let source = write_global_file( + home.as_ref(), + GLOBAL_AGENTS_FILENAME, + OLD_GLOBAL_INSTRUCTIONS, + )?; + let provider = local_compaction_provider(&server); + + // Create the thread with the old global source loaded into its instruction snapshot. + let mut builder = test_codex() + .with_home(Arc::clone(&home)) + .with_config(move |config| { + config.model_provider = provider; + }); + let test = builder.build(&server).await?; + + // Assert the pre-compaction source list points at the creation-time file. + assert_eq!( + test.codex.instruction_sources().await, + vec![source.clone()], + "thread reports the creation-time global source before compaction" + ); + + // Materialize the old snapshot, rewrite the selected file in place, and manually compact. + test.submit_turn("first turn").await?; + let rewritten_source = write_global_file( + home.as_ref(), + GLOBAL_AGENTS_FILENAME, + NEW_GLOBAL_INSTRUCTIONS, + )?; + assert_eq!(source, rewritten_source); + + test.codex.submit(Op::Compact).await?; + wait_for_event(&test.codex, |event| { + matches!(event, EventMsg::TurnComplete(_)) + }) + .await; + test.submit_turn("after compact").await?; + + // Assert ordinary and compact turns keep the old rendering even though the reported source + // path now contains new text. + let requests = response_mock.requests(); + assert_eq!(requests.len(), 3); + let expected_fragment = + expected_instruction_fragment(&test.config.cwd, OLD_GLOBAL_INSTRUCTIONS); + assert_single_instruction_fragment(&requests[0], &expected_fragment); + assert_single_instruction_fragment(&requests[1], &expected_fragment); + assert_single_instruction_fragment(&requests[2], &expected_fragment); + assert_eq!( + test.codex.instruction_sources().await, + vec![source], + "thread retains the creation-time global source after compaction" + ); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn mid_turn_compaction_keeps_the_creation_time_global_instructions() -> Result<()> { + // Set up a turn that crosses the auto-compaction limit and a post-compaction response. + let server = responses::start_mock_server().await; + let response_mock = responses::mount_sse_sequence( + &server, + vec![ + responses::sse(vec![ + responses::ev_function_call("call-1", "unsupported_tool", "{}"), + responses::ev_completed_with_tokens("first-response", /*total_tokens*/ 96), + ]), + responses::sse(vec![ + responses::ev_assistant_message("compact-message", "summary"), + responses::ev_completed_with_tokens("compact-response", /*total_tokens*/ 10), + ]), + responses::sse(vec![ + responses::ev_assistant_message("final-message", "done"), + responses::ev_completed_with_tokens("follow-up-response", /*total_tokens*/ 10), + ]), + ], + ) + .await; + let home = Arc::new(TempDir::new()?); + let source = write_global_file( + home.as_ref(), + GLOBAL_AGENTS_FILENAME, + OLD_GLOBAL_INSTRUCTIONS, + )?; + let provider = local_compaction_provider(&server); + + // Create the thread with the old global source loaded into its instruction snapshot. + let mut builder = test_codex() + .with_home(Arc::clone(&home)) + .with_config(move |config| { + config.model_provider = provider; + config.model_context_window = Some(100); + config.model_auto_compact_token_limit = Some(90); + }); + let test = builder.build(&server).await?; + + // Assert the pre-compaction source list points at the creation-time file. + assert_eq!( + test.codex.instruction_sources().await, + vec![source.clone()], + "thread reports the creation-time global source before mid-turn compaction" + ); + + // Add a preferred override before the turn triggers automatic mid-turn compaction. + let new_source = write_global_file( + home.as_ref(), + GLOBAL_AGENTS_OVERRIDE_FILENAME, + NEW_GLOBAL_INSTRUCTIONS, + )?; + assert_ne!(source, new_source); + test.submit_turn("trigger mid-turn compaction").await?; + + // Assert the initial, compact, and resumed requests all keep the old snapshot and source. + let requests = response_mock.requests(); + assert_eq!(requests.len(), 3); + let expected_fragment = + expected_instruction_fragment(&test.config.cwd, OLD_GLOBAL_INSTRUCTIONS); + assert_single_instruction_fragment(&requests[0], &expected_fragment); + assert_single_instruction_fragment(&requests[1], &expected_fragment); + assert_single_instruction_fragment(&requests[2], &expected_fragment); + assert_eq!( + test.codex.instruction_sources().await, + vec![source], + "thread retains the creation-time global source after mid-turn compaction" + ); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn remote_v2_compaction_keeps_creation_time_instructions_after_same_path_mutation() +-> Result<()> { + skip_if_no_network!(Ok(())); + + // Set up an ordinary turn, a remote-v2 compact response, and a post-compaction turn. + let server = responses::start_mock_server().await; + let response_mock = responses::mount_sse_sequence( + &server, + vec![ + responses::sse(vec![ + responses::ev_response_created("remote-v2-initial-response"), + responses::ev_completed("remote-v2-initial-response"), + ]), + remote_v2_compaction_response(), + responses::sse(vec![ + responses::ev_response_created("remote-v2-follow-up-response"), + responses::ev_completed("remote-v2-follow-up-response"), + ]), + responses::sse(vec![ + responses::ev_response_created("remote-v2-resumed-response"), + responses::ev_completed("remote-v2-resumed-response"), + ]), + ], + ) + .await; + let home = Arc::new(TempDir::new()?); + let source = write_global_file( + home.as_ref(), + GLOBAL_AGENTS_FILENAME, + OLD_GLOBAL_INSTRUCTIONS, + )?; + let mut builder = test_codex() + .with_home(Arc::clone(&home)) + .with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing()) + .with_config(|config| { + let _ = config.features.enable(Feature::RemoteCompactionV2); + }); + let test = builder.build(&server).await?; + + // Materialize the old snapshot, rewrite the selected file in place, and compact remotely. + test.submit_turn("before remote v2 compaction").await?; + let rewritten_source = write_global_file( + home.as_ref(), + GLOBAL_AGENTS_FILENAME, + NEW_GLOBAL_INSTRUCTIONS, + )?; + assert_eq!(source, rewritten_source); + test.codex.submit(Op::Compact).await?; + wait_for_event(&test.codex, |event| { + matches!(event, EventMsg::TurnComplete(_)) + }) + .await; + test.submit_turn("after remote v2 compaction").await?; + test.codex.flush_rollout().await?; + + // Assert the compact request, installed replacement history, and follow-up all keep the + // creation-time item despite the file-backed source now containing new text. + let requests = response_mock.requests(); + assert_eq!(requests.len(), 3); + let old_fragment = expected_instruction_fragment(&test.config.cwd, OLD_GLOBAL_INSTRUCTIONS); + assert_single_instruction_fragment(&requests[0], &old_fragment); + assert_single_instruction_fragment(&requests[1], &old_fragment); + assert_single_instruction_fragment(&requests[2], &old_fragment); + assert_eq!( + requests[1].input().last(), + Some(&json!({"type": "compaction_trigger"})), + "remote-v2 compact request should append exactly one compaction trigger" + ); + let rollout_path = test.codex.rollout_path().expect("rollout path"); + let replacement_history = replacement_history_from_rollout(&rollout_path)?; + assert_eq!( + instruction_fragments_in_items(&replacement_history), + Vec::::new(), + "remote-v2 replacement history currently omits the global-instruction fragment" + ); + assert_eq!( + test.codex.instruction_sources().await, + vec![source.clone()], + "running thread retains the selected same-path source" + ); + assert_eq!( + fs::read_to_string(source.as_path())?, + NEW_GLOBAL_INSTRUCTIONS, + "the selected source path should contain the rewritten text" + ); + + // Cold-resume the persisted replacement history with freshly loaded same-path configuration. + test.codex.submit(Op::Shutdown).await?; + wait_for_event(&test.codex, |event| { + matches!(event, EventMsg::ShutdownComplete) + }) + .await; + let resumed_cwd = test.config.cwd.clone(); + let mut resume_builder = test_codex() + .with_home(Arc::clone(&home)) + .with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing()) + .with_config(move |config| { + config.cwd = resumed_cwd; + let _ = config.features.enable(Feature::RemoteCompactionV2); + }); + let resumed = resume_builder + .resume(&server, Arc::clone(&home), rollout_path) + .await?; + resumed + .submit_turn("after remote v2 compaction cold resume") + .await?; + + // Modern replacement-history resume replays the persisted checkpoint and its later old-context + // suffix even though the same source path now contains new text. + let requests = response_mock.requests(); + assert_eq!(requests.len(), 4); + assert_single_instruction_fragment(&requests[3], &old_fragment); + let resumed_input = requests[3].input(); + assert_eq!( + resumed_input.get(..replacement_history.len()), + Some(replacement_history.as_slice()), + "remote-v2 cold resume should replay persisted replacement history verbatim" + ); + let post_compact_input = requests[2].input(); + assert_eq!( + resumed_input.get(..post_compact_input.len()), + Some(post_compact_input.as_slice()), + "remote-v2 cold resume should replay the complete post-compaction structured prefix" + ); + assert_eq!( + resumed.codex.instruction_sources().await, + vec![source], + "cold-resumed thread reports the same rewritten source path" + ); + + Ok(()) +} From f574946960a4d21e884453ac9963a869153b847d Mon Sep 17 00:00:00 2001 From: "Adam Perry @ OpenAI" Date: Tue, 9 Jun 2026 14:18:24 -0700 Subject: [PATCH 13/47] [codex] Report unusable MCP OAuth credentials as logged out (#26713) ## Why Persisted MCP OAuth credentials were reported as authenticated whenever a credential record existed. An expired token without a usable refresh token could therefore appear as `OAuth` even though startup could not authenticate with it, leaving users with a misleading status instead of a login prompt. ## What changed - Classify stored OAuth credentials as missing, usable, or requiring authorization. - Reuse the existing refresh window so near-expiry credentials without a refresh path are also treated as logged out. - Validate required credential fields before reporting OAuth authentication. - Add unit coverage for credential usability and integration coverage for expired, unexpired, and refreshable persisted credentials. ## Validation - `just test -p codex-rmcp-client` --- codex-rs/rmcp-client/src/auth_status.rs | 11 +- codex-rs/rmcp-client/src/oauth.rs | 112 +++++++++++++++++- .../tests/streamable_http_oauth_startup.rs | 103 ++++++++++++++++ 3 files changed, 220 insertions(+), 6 deletions(-) diff --git a/codex-rs/rmcp-client/src/auth_status.rs b/codex-rs/rmcp-client/src/auth_status.rs index 0b3b3bf6a7..bef9edc6c9 100644 --- a/codex-rs/rmcp-client/src/auth_status.rs +++ b/codex-rs/rmcp-client/src/auth_status.rs @@ -12,7 +12,8 @@ use reqwest::header::HeaderMap; use serde::Deserialize; use tracing::debug; -use crate::oauth::has_oauth_tokens; +use crate::oauth::StoredOAuthTokenStatus; +use crate::oauth::oauth_token_status; use crate::utils::apply_default_headers; use crate::utils::build_default_headers; use codex_config::types::OAuthCredentialsStoreMode; @@ -44,8 +45,12 @@ pub async fn determine_streamable_http_auth_status( return Ok(McpAuthStatus::BearerToken); } - if has_oauth_tokens(server_name, url, store_mode)? { - return Ok(McpAuthStatus::OAuth); + match oauth_token_status(server_name, url, store_mode)? { + StoredOAuthTokenStatus::Usable => return Ok(McpAuthStatus::OAuth), + StoredOAuthTokenStatus::AuthorizationRequired => { + return Ok(McpAuthStatus::NotLoggedIn); + } + StoredOAuthTokenStatus::Missing => {} } match discover_streamable_http_oauth_with_headers(url, &default_headers).await { diff --git a/codex-rs/rmcp-client/src/oauth.rs b/codex-rs/rmcp-client/src/oauth.rs index c348460795..5742660277 100644 --- a/codex-rs/rmcp-client/src/oauth.rs +++ b/codex-rs/rmcp-client/src/oauth.rs @@ -76,6 +76,13 @@ impl PartialEq for WrappedOAuthTokenResponse { } } +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum StoredOAuthTokenStatus { + Missing, + Usable, + AuthorizationRequired, +} + pub(crate) fn load_oauth_tokens( server_name: &str, url: &str, @@ -94,12 +101,33 @@ pub(crate) fn load_oauth_tokens( } } -pub(crate) fn has_oauth_tokens( +pub(crate) fn oauth_token_status( server_name: &str, url: &str, store_mode: OAuthCredentialsStoreMode, -) -> Result { - Ok(load_oauth_tokens(server_name, url, store_mode)?.is_some()) +) -> Result { + Ok( + match load_oauth_tokens(server_name, url, store_mode)?.as_ref() { + None => StoredOAuthTokenStatus::Missing, + Some(tokens) if oauth_tokens_are_usable(tokens) => StoredOAuthTokenStatus::Usable, + Some(_) => StoredOAuthTokenStatus::AuthorizationRequired, + }, + ) +} + +fn oauth_tokens_are_usable(tokens: &StoredOAuthTokens) -> bool { + if tokens.client_id.trim().is_empty() { + return false; + } + + let token_response = &tokens.token_response.0; + if token_needs_refresh(tokens.expires_at) { + return token_response + .refresh_token() + .is_some_and(|token| !token.secret().trim().is_empty()); + } + + !token_response.access_token().secret().trim().is_empty() } fn refresh_expires_in_from_timestamp(tokens: &mut StoredOAuthTokens) { @@ -852,6 +880,84 @@ mod tests { assert_eq!(tokens.token_response.0.expires_in(), Some(Duration::ZERO)); } + #[test] + fn oauth_tokens_are_usable_when_expiry_is_unknown() { + let mut tokens = sample_tokens(); + tokens.expires_at = None; + tokens.token_response.0.set_refresh_token(None); + + assert!(super::oauth_tokens_are_usable(&tokens)); + } + + #[test] + fn oauth_tokens_are_usable_when_unexpired_without_refresh_token() { + let mut tokens = sample_tokens(); + tokens.token_response.0.set_refresh_token(None); + + assert!(super::oauth_tokens_are_usable(&tokens)); + } + + #[test] + fn oauth_tokens_are_usable_when_expired_but_refreshable() { + let mut tokens = sample_tokens(); + tokens.expires_at = Some(0); + + assert!(super::oauth_tokens_are_usable(&tokens)); + } + + #[test] + fn oauth_tokens_are_not_usable_when_expired_and_unrefreshable() { + let mut tokens = sample_tokens(); + tokens.expires_at = Some(0); + tokens.token_response.0.set_refresh_token(None); + + assert!(!super::oauth_tokens_are_usable(&tokens)); + } + + #[test] + fn oauth_tokens_are_not_usable_when_near_expiry_and_unrefreshable() { + let mut tokens = sample_tokens(); + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_else(|_| Duration::from_secs(0)) + .as_millis() as u64; + tokens.expires_at = Some(now.saturating_add(REFRESH_SKEW_MILLIS - 1)); + tokens.token_response.0.set_refresh_token(None); + + assert!(!super::oauth_tokens_are_usable(&tokens)); + } + + #[test] + fn oauth_tokens_are_not_usable_when_client_id_is_blank() { + let mut tokens = sample_tokens(); + tokens.client_id = " ".to_string(); + + assert!(!super::oauth_tokens_are_usable(&tokens)); + } + + #[test] + fn oauth_tokens_are_not_usable_when_access_token_is_blank() { + let mut tokens = sample_tokens(); + tokens + .token_response + .0 + .set_access_token(AccessToken::new(" ".to_string())); + + assert!(!super::oauth_tokens_are_usable(&tokens)); + } + + #[test] + fn oauth_tokens_are_not_usable_when_required_refresh_token_is_blank() { + let mut tokens = sample_tokens(); + tokens.expires_at = Some(0); + tokens + .token_response + .0 + .set_refresh_token(Some(RefreshToken::new(" ".to_string()))); + + assert!(!super::oauth_tokens_are_usable(&tokens)); + } + fn assert_tokens_match_without_expiry( actual: &StoredOAuthTokens, expected: &StoredOAuthTokens, diff --git a/codex-rs/rmcp-client/tests/streamable_http_oauth_startup.rs b/codex-rs/rmcp-client/tests/streamable_http_oauth_startup.rs index 1c18f2c98b..d7d8cf4232 100644 --- a/codex-rs/rmcp-client/tests/streamable_http_oauth_startup.rs +++ b/codex-rs/rmcp-client/tests/streamable_http_oauth_startup.rs @@ -1,16 +1,21 @@ mod streamable_http_test_support; use std::time::Duration; +use std::time::SystemTime; +use std::time::UNIX_EPOCH; use codex_config::types::OAuthCredentialsStoreMode; use codex_exec_server::Environment; +use codex_rmcp_client::McpAuthStatus; use codex_rmcp_client::RmcpClient; use codex_rmcp_client::StoredOAuthTokens; use codex_rmcp_client::WrappedOAuthTokenResponse; +use codex_rmcp_client::determine_streamable_http_auth_status; use codex_rmcp_client::save_oauth_tokens; use oauth2::AccessToken; use oauth2::RefreshToken; use oauth2::basic::BasicTokenType; +use pretty_assertions::assert_eq; use rmcp::transport::auth::OAuthTokenResponse; use rmcp::transport::auth::VendorExtraTokenFields; use serde_json::Value; @@ -33,6 +38,9 @@ const EXPIRED_ACCESS_TOKEN: &str = "expired-access-token"; const REFRESH_TOKEN: &str = "valid-refresh-token"; const REFRESHED_ACCESS_TOKEN: &str = "refreshed-access-token"; const CHILD_SERVER_URL_ENV: &str = "MCP_TEST_OAUTH_STARTUP_SERVER_URL"; +const UNREFRESHABLE_SERVER_URL: &str = "https://unrefreshable.example/mcp"; +const UNEXPIRED_SERVER_URL: &str = "https://unexpired.example/mcp"; +const REFRESHABLE_SERVER_URL: &str = "https://refreshable.example/mcp"; #[tokio::test(flavor = "multi_thread", worker_threads = 1)] async fn refreshes_expired_persisted_token_before_initialize() -> anyhow::Result<()> { @@ -112,6 +120,101 @@ async fn refreshes_expired_persisted_token_before_initialize() -> anyhow::Result Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn reports_auth_status_for_persisted_credentials() -> anyhow::Result<()> { + let codex_home = TempDir::new()?; + + let status = Command::new(std::env::current_exe()?) + .args([ + "persisted_credentials_auth_status_child", + "--exact", + "--ignored", + "--nocapture", + ]) + .env("CODEX_HOME", codex_home.path()) + .status() + .await?; + + assert!( + status.success(), + "persisted credentials auth status child failed: {status}" + ); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +#[ignore = "spawned by reports_auth_status_for_persisted_credentials"] +async fn persisted_credentials_auth_status_child() -> anyhow::Result<()> { + let response = OAuthTokenResponse::new( + AccessToken::new(EXPIRED_ACCESS_TOKEN.to_string()), + BasicTokenType::Bearer, + VendorExtraTokenFields::default(), + ); + let tokens = StoredOAuthTokens { + server_name: SERVER_NAME.to_string(), + url: UNREFRESHABLE_SERVER_URL.to_string(), + client_id: "test-client-id".to_string(), + token_response: WrappedOAuthTokenResponse(response), + expires_at: Some(0), + }; + save_oauth_tokens(SERVER_NAME, &tokens, OAuthCredentialsStoreMode::File)?; + + let status = auth_status(UNREFRESHABLE_SERVER_URL).await?; + assert_eq!(status, McpAuthStatus::NotLoggedIn); + + let response = OAuthTokenResponse::new( + AccessToken::new("unexpired-access-token".to_string()), + BasicTokenType::Bearer, + VendorExtraTokenFields::default(), + ); + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_else(|_| Duration::from_secs(0)) + .as_millis() as u64; + let tokens = StoredOAuthTokens { + server_name: SERVER_NAME.to_string(), + url: UNEXPIRED_SERVER_URL.to_string(), + client_id: "test-client-id".to_string(), + token_response: WrappedOAuthTokenResponse(response), + expires_at: Some(now.saturating_add(/*rhs*/ 60_000)), + }; + save_oauth_tokens(SERVER_NAME, &tokens, OAuthCredentialsStoreMode::File)?; + + let status = auth_status(UNEXPIRED_SERVER_URL).await?; + assert_eq!(status, McpAuthStatus::OAuth); + + let mut response = OAuthTokenResponse::new( + AccessToken::new(EXPIRED_ACCESS_TOKEN.to_string()), + BasicTokenType::Bearer, + VendorExtraTokenFields::default(), + ); + response.set_refresh_token(Some(RefreshToken::new(REFRESH_TOKEN.to_string()))); + let tokens = StoredOAuthTokens { + server_name: SERVER_NAME.to_string(), + url: REFRESHABLE_SERVER_URL.to_string(), + client_id: "test-client-id".to_string(), + token_response: WrappedOAuthTokenResponse(response), + expires_at: Some(0), + }; + save_oauth_tokens(SERVER_NAME, &tokens, OAuthCredentialsStoreMode::File)?; + + let status = auth_status(REFRESHABLE_SERVER_URL).await?; + assert_eq!(status, McpAuthStatus::OAuth); + Ok(()) +} + +async fn auth_status(server_url: &str) -> anyhow::Result { + determine_streamable_http_auth_status( + SERVER_NAME, + server_url, + /*bearer_token_env_var*/ None, + /*http_headers*/ None, + /*env_http_headers*/ None, + OAuthCredentialsStoreMode::File, + ) + .await +} + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] #[ignore = "spawned by refreshes_expired_persisted_token_before_initialize"] async fn oauth_startup_child() -> anyhow::Result<()> { From f2969f36e8b6aa2aefcd625d2a9fc8425bb2a519 Mon Sep 17 00:00:00 2001 From: pakrym-oai Date: Tue, 9 Jun 2026 15:10:17 -0700 Subject: [PATCH 14/47] [codex] Handle Ctrl-C for non-TTY unified exec (#26734) ## Why A long-running unified exec process started with `tty: false` could not be interrupted via `write_stdin`: ordinary non-TTY stdin writes are rejected once stdin is closed, but an exact U+0003 payload should still map to a process interrupt. The interrupt should flow through the same process lifecycle path as a real signal so Codex preserves process-reported output and exit metadata instead of fabricating a Ctrl-C exit code or tearing down the session early. ## What Changed - Add `process/signal` to exec-server with `ProcessSignal::Interrupt` and an empty response. - Add a non-consuming `ProcessHandle::signal` path for spawned processes; on Unix it sends SIGINT to the process group and leaves terminate/hard-kill unchanged. - Route non-TTY U+0003 `write_stdin` through `process.signal(...)` instead of `terminate`, then let the normal post-write collection path drain output and observe exit. - Add exec-server coverage where a shell `trap INT` handler prints the signal and exits with its own code. - Add unified exec coverage where a `tty: false` process traps SIGINT, emits output, and exits with its own code. ## Validation - `just test -p codex-exec-server exec_process_signal_interrupts_process` - `just test -p codex-exec-server` - `just test -p codex-core write_stdin_ctrl_c_interrupts_non_tty_session` --- codex-rs/core/src/unified_exec/process.rs | 15 +- .../core/src/unified_exec/process_manager.rs | 40 +-- .../core/src/unified_exec/process_tests.rs | 5 + codex-rs/core/tests/suite/unified_exec.rs | 243 ++++++++++++++++++ codex-rs/exec-server/src/client.rs | 25 ++ codex-rs/exec-server/src/lib.rs | 3 + codex-rs/exec-server/src/local_process.rs | 54 +++- codex-rs/exec-server/src/process.rs | 3 + codex-rs/exec-server/src/protocol.rs | 18 ++ codex-rs/exec-server/src/remote_process.rs | 6 + codex-rs/exec-server/src/server/handler.rs | 10 + .../exec-server/src/server/process_handler.rs | 9 + codex-rs/exec-server/src/server/registry.rs | 8 + codex-rs/exec-server/tests/exec_process.rs | 125 ++++++++- codex-rs/utils/pty/src/lib.rs | 2 + codex-rs/utils/pty/src/pipe.rs | 20 +- codex-rs/utils/pty/src/process.rs | 48 ++++ codex-rs/utils/pty/src/process_group.rs | 43 ++-- codex-rs/utils/pty/src/pty.rs | 26 +- 19 files changed, 659 insertions(+), 44 deletions(-) diff --git a/codex-rs/core/src/unified_exec/process.rs b/codex-rs/core/src/unified_exec/process.rs index 9671429d01..229e5b52f8 100644 --- a/codex-rs/core/src/unified_exec/process.rs +++ b/codex-rs/core/src/unified_exec/process.rs @@ -14,6 +14,7 @@ use tokio_util::sync::CancellationToken; use crate::exec::is_likely_sandbox_denied; use codex_exec_server::ExecProcess; +use codex_exec_server::ProcessSignal as ExecServerProcessSignal; use codex_exec_server::ReadResponse as ExecReadResponse; use codex_exec_server::StartedExecProcess; use codex_exec_server::WriteStatus; @@ -23,6 +24,7 @@ use codex_protocol::protocol::TruncationPolicy; use codex_sandboxing::SandboxType; use codex_utils_output_truncation::formatted_truncate_text; use codex_utils_pty::ExecCommandSession; +use codex_utils_pty::ProcessSignal as PtyProcessSignal; use codex_utils_pty::SpawnedPty; use super::UNIFIED_EXEC_OUTPUT_MAX_TOKENS; @@ -31,7 +33,6 @@ use super::head_tail_buffer::HeadTailBuffer; use super::process_state::ProcessState; const EARLY_EXIT_GRACE_PERIOD: Duration = Duration::from_millis(150); - pub(crate) trait SpawnLifecycle: std::fmt::Debug + Send + Sync { /// Returns file descriptors that must stay open across the child `exec()`. /// @@ -212,6 +213,18 @@ impl UnifiedExecProcess { } } + pub(super) async fn interrupt(&self) -> Result<(), UnifiedExecError> { + match &self.process_handle { + ProcessHandle::Local(process_handle) => process_handle + .signal(PtyProcessSignal::Interrupt) + .map_err(|err| UnifiedExecError::process_failed(err.to_string())), + ProcessHandle::ExecServer(process_handle) => process_handle + .signal(ExecServerProcessSignal::Interrupt) + .await + .map_err(|err| UnifiedExecError::process_failed(err.to_string())), + } + } + pub(super) fn fail_and_terminate(&self, message: String) { let state = self.state_rx.borrow().clone(); if state.failure_message.is_none() { diff --git a/codex-rs/core/src/unified_exec/process_manager.rs b/codex-rs/core/src/unified_exec/process_manager.rs index cb455b113b..c9e6f3fdca 100644 --- a/codex-rs/core/src/unified_exec/process_manager.rs +++ b/codex-rs/core/src/unified_exec/process_manager.rs @@ -72,6 +72,7 @@ const UNIFIED_EXEC_ENV: [(&str, &str); 10] = [ const NETWORK_ACCESS_DENIED_MESSAGE: &str = "Network access was denied by the Codex sandbox network proxy."; const LATE_NETWORK_DENIAL_GRACE_PERIOD: Duration = Duration::from_millis(100); +const INTERRUPT: &str = "\u{3}"; /// Test-only override for deterministic unified exec process IDs. /// @@ -617,24 +618,29 @@ impl UnifiedExecProcessManager { if !request.input.is_empty() { if !tty { - return Err(UnifiedExecError::StdinClosed); - } - match process.write(request.input.as_bytes()).await { - Ok(()) => { - // Give the remote process a brief window to react so that we are - // more likely to capture its output in the poll below. - tokio::time::sleep(Duration::from_millis(100)).await; + if request.input == INTERRUPT { + process.interrupt().await?; + } else { + return Err(UnifiedExecError::StdinClosed); } - Err(err) => { - let status = self.refresh_process_state(process_id).await; - if matches!(status, ProcessStatus::Exited { .. }) { - status_after_write = Some(status); - } else if matches!(err, UnifiedExecError::ProcessFailed { .. }) { - process.terminate(); - self.release_process_id(process_id).await; - return Err(err); - } else { - return Err(err); + } else { + match process.write(request.input.as_bytes()).await { + Ok(()) => { + // Give the remote process a brief window to react so that we are + // more likely to capture its output in the poll below. + tokio::time::sleep(Duration::from_millis(100)).await; + } + Err(err) => { + let status = self.refresh_process_state(process_id).await; + if matches!(status, ProcessStatus::Exited { .. }) { + status_after_write = Some(status); + } else if matches!(err, UnifiedExecError::ProcessFailed { .. }) { + process.terminate(); + self.release_process_id(process_id).await; + return Err(err); + } else { + return Err(err); + } } } } diff --git a/codex-rs/core/src/unified_exec/process_tests.rs b/codex-rs/core/src/unified_exec/process_tests.rs index ee0ea1cba3..42db18ff78 100644 --- a/codex-rs/core/src/unified_exec/process_tests.rs +++ b/codex-rs/core/src/unified_exec/process_tests.rs @@ -5,6 +5,7 @@ use codex_exec_server::ExecProcess; use codex_exec_server::ExecProcessEventReceiver; use codex_exec_server::ExecServerError; use codex_exec_server::ProcessId; +use codex_exec_server::ProcessSignal; use codex_exec_server::ReadResponse; use codex_exec_server::StartedExecProcess; use codex_exec_server::WriteResponse; @@ -63,6 +64,10 @@ impl ExecProcess for MockExecProcess { Ok(self.write_response.clone()) } + async fn signal(&self, _signal: ProcessSignal) -> Result<(), ExecServerError> { + Ok(()) + } + async fn terminate(&self) -> Result<(), ExecServerError> { Ok(()) } diff --git a/codex-rs/core/tests/suite/unified_exec.rs b/codex-rs/core/tests/suite/unified_exec.rs index caea5ecf55..eb8bf29551 100644 --- a/codex-rs/core/tests/suite/unified_exec.rs +++ b/codex-rs/core/tests/suite/unified_exec.rs @@ -1995,6 +1995,249 @@ async fn write_stdin_returns_exit_metadata_and_clears_session() -> Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn write_stdin_ctrl_c_interrupts_non_tty_session() -> Result<()> { + assert_write_stdin_ctrl_c_interrupts_non_tty_session( + "trap", + "trap 'echo INT-TRAP; exit 42' INT; echo READY; while true; do sleep 30; done", + /*expected_exit_code*/ 42, + Some("INT-TRAP"), + ) + .await +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn write_stdin_ctrl_c_default_interrupt_reports_130_for_non_tty_session() -> Result<()> { + assert_write_stdin_ctrl_c_interrupts_non_tty_session( + "default", + "echo READY; exec sleep 30", + /*expected_exit_code*/ 130, + /*expected_interrupt_output*/ None, + ) + .await +} + +async fn assert_write_stdin_ctrl_c_interrupts_non_tty_session( + test_name: &str, + command: &str, + expected_exit_code: i32, + expected_interrupt_output: Option<&str>, +) -> Result<()> { + skip_if_no_network!(Ok(())); + skip_if_sandbox!(Ok(())); + skip_if_windows!(Ok(())); + + let server = start_mock_server().await; + + let mut builder = test_codex().with_config(|config| { + if let Err(err) = config.features.enable(Feature::UnifiedExec) { + panic!("test config should allow feature update: {err}"); + } + }); + let test = builder.build_with_remote_env(&server).await?; + + let start_call_id = format!("uexec-non-tty-interrupt-{test_name}-start"); + let interrupt_call_id = format!("uexec-non-tty-interrupt-{test_name}"); + + let start_args = serde_json::json!({ + "cmd": command, + "yield_time_ms": 250, + "tty": false, + }); + let interrupt_args = serde_json::json!({ + "chars": "\u{3}", + "session_id": 1000, + "yield_time_ms": 1000, + }); + + let responses = vec![ + sse(vec![ + ev_response_created("resp-1"), + ev_function_call( + &start_call_id, + "exec_command", + &serde_json::to_string(&start_args)?, + ), + ev_completed("resp-1"), + ]), + sse(vec![ + ev_response_created("resp-2"), + ev_function_call( + &interrupt_call_id, + "write_stdin", + &serde_json::to_string(&interrupt_args)?, + ), + ev_completed("resp-2"), + ]), + sse(vec![ + ev_assistant_message("msg-1", "done"), + ev_completed("resp-3"), + ]), + ]; + let request_log = mount_sse_sequence(&server, responses).await; + + submit_unified_exec_turn( + &test, + "interrupt non-tty unified exec", + PermissionProfile::Disabled, + ) + .await?; + + wait_for_event(&test.codex, |event| { + matches!(event, EventMsg::TurnComplete(_)) + }) + .await; + + let requests = request_log.requests(); + assert!(!requests.is_empty(), "expected at least one POST request"); + let bodies = requests + .into_iter() + .map(|request| request.body_json()) + .collect::>(); + + let outputs = collect_tool_outputs(&bodies)?; + + let start_output = outputs + .get(&start_call_id) + .with_context(|| format!("missing start output for exec_command {start_call_id}"))?; + assert_eq!( + start_output.process_id.as_deref(), + Some("1000"), + "exec_command should leave a running non-TTY session" + ); + assert!( + start_output.exit_code.is_none(), + "initial exec_command should not include exit_code while session is running" + ); + assert!( + start_output.output.contains("READY"), + "start output should include command readiness marker, got {:?}", + start_output.output + ); + + let interrupt_output = outputs + .get(&interrupt_call_id) + .with_context(|| format!("missing interrupt output for write_stdin {interrupt_call_id}"))?; + assert!( + interrupt_output.process_id.is_none(), + "interrupted process should be cleared from the session map" + ); + assert_eq!( + interrupt_output.exit_code, + Some(expected_exit_code), + "interrupt should preserve the process-reported exit code" + ); + if let Some(expected_interrupt_output) = expected_interrupt_output { + assert!( + interrupt_output.output.contains(expected_interrupt_output), + "interrupt should drain output from the signal handler, got {:?}", + interrupt_output.output + ); + } + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[cfg_attr(not(windows), ignore = "Windows-only unified exec interrupt test")] +async fn write_stdin_ctrl_c_reports_unsupported_interrupt_to_model_on_windows() -> Result<()> { + skip_if_no_network!(Ok(())); + skip_if_sandbox!(Ok(())); + + let server = start_mock_server().await; + + let mut builder = test_codex().with_config(|config| { + config + .features + .enable(Feature::UnifiedExec) + .expect("test config should allow feature update"); + }); + let test = builder.build_with_remote_env(&server).await?; + + let start_call_id = "uexec-windows-interrupt-start"; + let interrupt_call_id = "uexec-windows-interrupt"; + + let start_args = serde_json::json!({ + "shell": "cmd", + "cmd": "echo READY && ping -n 30 127.0.0.1 >NUL", + "yield_time_ms": 250, + "tty": false, + }); + let interrupt_args = serde_json::json!({ + "chars": "\u{3}", + "session_id": 1000, + "yield_time_ms": 1000, + }); + + let responses = vec![ + sse(vec![ + ev_response_created("resp-1"), + ev_function_call( + start_call_id, + "exec_command", + &serde_json::to_string(&start_args)?, + ), + ev_completed("resp-1"), + ]), + sse(vec![ + ev_response_created("resp-2"), + ev_function_call( + interrupt_call_id, + "write_stdin", + &serde_json::to_string(&interrupt_args)?, + ), + ev_completed("resp-2"), + ]), + sse(vec![ + ev_response_created("resp-3"), + ev_assistant_message("msg-1", "done"), + ev_completed("resp-3"), + ]), + ]; + let request_log = mount_sse_sequence(&server, responses).await; + + submit_unified_exec_turn( + &test, + "interrupt non-tty unified exec on Windows", + PermissionProfile::Disabled, + ) + .await?; + + wait_for_event(&test.codex, |event| { + matches!(event, EventMsg::TurnComplete(_)) + }) + .await; + + let start_output = request_log + .function_call_output_text(start_call_id) + .expect("missing start output for exec_command"); + let start_output = parse_unified_exec_output(&start_output)?; + assert_eq!( + start_output.process_id.as_deref(), + Some("1000"), + "exec_command should leave a running non-TTY session" + ); + assert!( + start_output.output.contains("READY"), + "start output should include command readiness marker, got {:?}", + start_output.output + ); + + let interrupt_output = request_log + .function_call_output_text(interrupt_call_id) + .expect("missing interrupt output for write_stdin"); + assert!( + interrupt_output.contains("write_stdin failed"), + "model-visible write_stdin output should report failure, got {interrupt_output:?}" + ); + assert!( + interrupt_output.contains("process interrupt is not supported by this process backend"), + "model-visible write_stdin output should explain unsupported interrupt, got {interrupt_output:?}" + ); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn unified_exec_emits_end_event_when_session_dies_via_stdin() -> Result<()> { skip_if_no_network!(Ok(())); diff --git a/codex-rs/exec-server/src/client.rs b/codex-rs/exec-server/src/client.rs index 8cf18e7d1f..26b3e46e88 100644 --- a/codex-rs/exec-server/src/client.rs +++ b/codex-rs/exec-server/src/client.rs @@ -35,6 +35,7 @@ use crate::protocol::EXEC_EXITED_METHOD; use crate::protocol::EXEC_METHOD; use crate::protocol::EXEC_OUTPUT_DELTA_METHOD; use crate::protocol::EXEC_READ_METHOD; +use crate::protocol::EXEC_SIGNAL_METHOD; use crate::protocol::EXEC_TERMINATE_METHOD; use crate::protocol::EXEC_WRITE_METHOD; use crate::protocol::EnvironmentInfo; @@ -80,8 +81,11 @@ use crate::protocol::INITIALIZED_METHOD; use crate::protocol::InitializeParams; use crate::protocol::InitializeResponse; use crate::protocol::ProcessOutputChunk; +use crate::protocol::ProcessSignal; use crate::protocol::ReadParams; use crate::protocol::ReadResponse; +use crate::protocol::SignalParams; +use crate::protocol::SignalResponse; use crate::protocol::TerminateParams; use crate::protocol::TerminateResponse; use crate::protocol::WriteParams; @@ -394,6 +398,23 @@ impl ExecServerClient { .await } + pub async fn signal( + &self, + process_id: &ProcessId, + signal: ProcessSignal, + ) -> Result<(), ExecServerError> { + let _response: SignalResponse = self + .call( + EXEC_SIGNAL_METHOD, + &SignalParams { + process_id: process_id.clone(), + signal, + }, + ) + .await?; + Ok(()) + } + pub async fn terminate( &self, process_id: &ProcessId, @@ -763,6 +784,10 @@ impl Session { self.client.write(&self.process_id, chunk).await } + pub(crate) async fn signal(&self, signal: ProcessSignal) -> Result<(), ExecServerError> { + self.client.signal(&self.process_id, signal).await + } + pub(crate) async fn terminate(&self) -> Result<(), ExecServerError> { self.client.terminate(&self.process_id).await?; Ok(()) diff --git a/codex-rs/exec-server/src/lib.rs b/codex-rs/exec-server/src/lib.rs index e306ec395e..378d12e632 100644 --- a/codex-rs/exec-server/src/lib.rs +++ b/codex-rs/exec-server/src/lib.rs @@ -93,9 +93,12 @@ pub use protocol::HttpRequestResponse; pub use protocol::InitializeParams; pub use protocol::InitializeResponse; pub use protocol::ProcessOutputChunk; +pub use protocol::ProcessSignal; pub use protocol::ReadParams; pub use protocol::ReadResponse; pub use protocol::ShellInfo; +pub use protocol::SignalParams; +pub use protocol::SignalResponse; pub use protocol::TerminateParams; pub use protocol::TerminateResponse; pub use protocol::WriteParams; diff --git a/codex-rs/exec-server/src/local_process.rs b/codex-rs/exec-server/src/local_process.rs index bc69ec6105..691f807032 100644 --- a/codex-rs/exec-server/src/local_process.rs +++ b/codex-rs/exec-server/src/local_process.rs @@ -10,6 +10,7 @@ use codex_protocol::config_types::EnvironmentVariablePattern; use codex_protocol::config_types::ShellEnvironmentPolicy; use codex_protocol::shell_environment; use codex_utils_pty::ExecCommandSession; +use codex_utils_pty::ProcessSignal as PtyProcessSignal; use codex_utils_pty::TerminalSize; use tokio::sync::Mutex; use tokio::sync::Notify; @@ -33,8 +34,11 @@ use crate::protocol::ExecOutputStream; use crate::protocol::ExecParams; use crate::protocol::ExecResponse; use crate::protocol::ProcessOutputChunk; +use crate::protocol::ProcessSignal; use crate::protocol::ReadParams; use crate::protocol::ReadResponse; +use crate::protocol::SignalParams; +use crate::protocol::SignalResponse; use crate::protocol::TerminateParams; use crate::protocol::TerminateResponse; use crate::protocol::WriteParams; @@ -272,7 +276,6 @@ impl LocalProcess { &self, params: ReadParams, ) -> Result { - let _process_id = params.process_id.clone(); let after_seq = params.after_seq.unwrap_or(0); let max_bytes = params.max_bytes.unwrap_or(usize::MAX); let wait = Duration::from_millis(params.wait_ms.unwrap_or(0)); @@ -351,7 +354,6 @@ impl LocalProcess { &self, params: WriteParams, ) -> Result { - let _process_id = params.process_id.clone(); let _input_bytes = params.chunk.0.len(); let writer_tx = { let process_map = self.inner.processes.lock().await; @@ -383,11 +385,33 @@ impl LocalProcess { }) } + pub(crate) async fn signal_process( + &self, + params: SignalParams, + ) -> Result { + { + let process_map = self.inner.processes.lock().await; + match process_map.get(¶ms.process_id) { + Some(ProcessEntry::Running(process)) => { + if process.exit_code.is_some() { + return Ok(SignalResponse {}); + } + process + .session + .signal(pty_process_signal(params.signal)) + .map_err(|err| internal_error(format!("failed to signal process: {err}")))? + } + Some(ProcessEntry::Starting) | None => {} + } + } + + Ok(SignalResponse {}) + } + pub(crate) async fn terminate_process( &self, params: TerminateParams, ) -> Result { - let _process_id = params.process_id.clone(); let running = { let process_map = self.inner.processes.lock().await; match process_map.get(¶ms.process_id) { @@ -483,6 +507,10 @@ impl ExecProcess for LocalExecProcess { self.backend.write(&self.process_id, chunk).await } + async fn signal(&self, signal: ProcessSignal) -> Result<(), ExecServerError> { + self.backend.signal(&self.process_id, signal).await + } + async fn terminate(&self) -> Result<(), ExecServerError> { self.backend.terminate(&self.process_id).await } @@ -519,6 +547,20 @@ impl LocalProcess { .map_err(map_handler_error) } + async fn signal( + &self, + process_id: &ProcessId, + signal: ProcessSignal, + ) -> Result<(), ExecServerError> { + self.signal_process(SignalParams { + process_id: process_id.clone(), + signal, + }) + .await + .map_err(map_handler_error)?; + Ok(()) + } + async fn terminate(&self, process_id: &ProcessId) -> Result<(), ExecServerError> { self.terminate_process(TerminateParams { process_id: process_id.clone(), @@ -529,6 +571,12 @@ impl LocalProcess { } } +fn pty_process_signal(signal: ProcessSignal) -> PtyProcessSignal { + match signal { + ProcessSignal::Interrupt => PtyProcessSignal::Interrupt, + } +} + fn map_handler_error(error: JSONRPCErrorError) -> ExecServerError { ExecServerError::Server { code: error.code, diff --git a/codex-rs/exec-server/src/process.rs b/codex-rs/exec-server/src/process.rs index cb6c832138..97f53aaefb 100644 --- a/codex-rs/exec-server/src/process.rs +++ b/codex-rs/exec-server/src/process.rs @@ -10,6 +10,7 @@ use crate::ExecServerError; use crate::ProcessId; use crate::protocol::ExecParams; use crate::protocol::ProcessOutputChunk; +use crate::protocol::ProcessSignal; use crate::protocol::ReadResponse; use crate::protocol::WriteResponse; @@ -178,6 +179,8 @@ pub trait ExecProcess: Send + Sync { async fn write(&self, chunk: Vec) -> Result; + async fn signal(&self, signal: ProcessSignal) -> Result<(), ExecServerError>; + async fn terminate(&self) -> Result<(), ExecServerError>; } diff --git a/codex-rs/exec-server/src/protocol.rs b/codex-rs/exec-server/src/protocol.rs index 6eeeeefdd7..b85bff3577 100644 --- a/codex-rs/exec-server/src/protocol.rs +++ b/codex-rs/exec-server/src/protocol.rs @@ -15,6 +15,7 @@ pub const INITIALIZED_METHOD: &str = "initialized"; pub const EXEC_METHOD: &str = "process/start"; pub const EXEC_READ_METHOD: &str = "process/read"; pub const EXEC_WRITE_METHOD: &str = "process/write"; +pub const EXEC_SIGNAL_METHOD: &str = "process/signal"; pub const EXEC_TERMINATE_METHOD: &str = "process/terminate"; pub const EXEC_OUTPUT_DELTA_METHOD: &str = "process/output"; pub const EXEC_EXITED_METHOD: &str = "process/exited"; @@ -166,6 +167,23 @@ pub struct WriteResponse { pub status: WriteStatus, } +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ProcessSignal { + Interrupt, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SignalParams { + pub process_id: ProcessId, + pub signal: ProcessSignal, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SignalResponse {} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct TerminateParams { diff --git a/codex-rs/exec-server/src/remote_process.rs b/codex-rs/exec-server/src/remote_process.rs index d8d06735cd..c7d16d4d41 100644 --- a/codex-rs/exec-server/src/remote_process.rs +++ b/codex-rs/exec-server/src/remote_process.rs @@ -12,6 +12,7 @@ use crate::StartedExecProcess; use crate::client::LazyRemoteExecServerClient; use crate::client::Session; use crate::protocol::ExecParams; +use crate::protocol::ProcessSignal; use crate::protocol::ReadResponse; use crate::protocol::WriteResponse; @@ -76,6 +77,11 @@ impl ExecProcess for RemoteExecProcess { self.session.write(chunk).await } + async fn signal(&self, signal: ProcessSignal) -> Result<(), ExecServerError> { + trace!("exec process signal"); + self.session.signal(signal).await + } + async fn terminate(&self) -> Result<(), ExecServerError> { trace!("exec process terminate"); self.session.terminate().await diff --git a/codex-rs/exec-server/src/server/handler.rs b/codex-rs/exec-server/src/server/handler.rs index b8934705d2..395ce67730 100644 --- a/codex-rs/exec-server/src/server/handler.rs +++ b/codex-rs/exec-server/src/server/handler.rs @@ -42,6 +42,8 @@ use crate::protocol::InitializeParams; use crate::protocol::InitializeResponse; use crate::protocol::ReadParams; use crate::protocol::ReadResponse; +use crate::protocol::SignalParams; +use crate::protocol::SignalResponse; use crate::protocol::TerminateParams; use crate::protocol::TerminateResponse; use crate::protocol::WriteParams; @@ -171,6 +173,14 @@ impl ExecServerHandler { session.process().exec_write(params).await } + pub(crate) async fn signal( + &self, + params: SignalParams, + ) -> Result { + let session = self.require_initialized_for("exec")?; + session.process().signal(params).await + } + pub(crate) async fn terminate( &self, params: TerminateParams, diff --git a/codex-rs/exec-server/src/server/process_handler.rs b/codex-rs/exec-server/src/server/process_handler.rs index 38fbace1cd..9fced9c166 100644 --- a/codex-rs/exec-server/src/server/process_handler.rs +++ b/codex-rs/exec-server/src/server/process_handler.rs @@ -5,6 +5,8 @@ use crate::protocol::ExecParams; use crate::protocol::ExecResponse; use crate::protocol::ReadParams; use crate::protocol::ReadResponse; +use crate::protocol::SignalParams; +use crate::protocol::SignalResponse; use crate::protocol::TerminateParams; use crate::protocol::TerminateResponse; use crate::protocol::WriteParams; @@ -49,6 +51,13 @@ impl ProcessHandler { self.process.exec_write(params).await } + pub(crate) async fn signal( + &self, + params: SignalParams, + ) -> Result { + self.process.signal_process(params).await + } + pub(crate) async fn terminate( &self, params: TerminateParams, diff --git a/codex-rs/exec-server/src/server/registry.rs b/codex-rs/exec-server/src/server/registry.rs index 26d2876c63..1f652c1c4c 100644 --- a/codex-rs/exec-server/src/server/registry.rs +++ b/codex-rs/exec-server/src/server/registry.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use crate::protocol::ENVIRONMENT_INFO_METHOD; use crate::protocol::EXEC_METHOD; use crate::protocol::EXEC_READ_METHOD; +use crate::protocol::EXEC_SIGNAL_METHOD; use crate::protocol::EXEC_TERMINATE_METHOD; use crate::protocol::EXEC_WRITE_METHOD; use crate::protocol::ExecParams; @@ -32,6 +33,7 @@ use crate::protocol::INITIALIZE_METHOD; use crate::protocol::INITIALIZED_METHOD; use crate::protocol::InitializeParams; use crate::protocol::ReadParams; +use crate::protocol::SignalParams; use crate::protocol::TerminateParams; use crate::protocol::WriteParams; use crate::rpc::RpcRouter; @@ -77,6 +79,12 @@ pub(crate) fn build_router() -> RpcRouter { handler.exec_write(params).await }, ); + router.request( + EXEC_SIGNAL_METHOD, + |handler: Arc, params: SignalParams| async move { + handler.signal(params).await + }, + ); router.request( EXEC_TERMINATE_METHOD, |handler: Arc, params: TerminateParams| async move { diff --git a/codex-rs/exec-server/tests/exec_process.rs b/codex-rs/exec-server/tests/exec_process.rs index e1f330fc4e..af8e722c66 100644 --- a/codex-rs/exec-server/tests/exec_process.rs +++ b/codex-rs/exec-server/tests/exec_process.rs @@ -1,5 +1,3 @@ -#![cfg(unix)] - mod common; use std::sync::Arc; @@ -13,6 +11,7 @@ use codex_exec_server::ExecParams; use codex_exec_server::ExecProcess; use codex_exec_server::ExecProcessEvent; use codex_exec_server::ProcessId; +use codex_exec_server::ProcessSignal; use codex_exec_server::ReadResponse; use codex_exec_server::StartedExecProcess; use codex_exec_server::WriteStatus; @@ -505,6 +504,98 @@ async fn assert_exec_process_rejects_write_without_pipe_stdin(use_remote: bool) Ok(()) } +async fn assert_exec_process_signal_interrupts_process(use_remote: bool) -> Result<()> { + let context = create_process_context(use_remote).await?; + let process_id = "proc-signal".to_string(); + let session = context + .backend + .start(ExecParams { + process_id: process_id.clone().into(), + argv: vec![ + "/bin/sh".to_string(), + "-c".to_string(), + "trap 'printf \"signal:2\\n\"; exit 7' INT; printf 'ready\\n'; while :; do :; done".to_string(), + ], + cwd: std::env::current_dir()?, + env_policy: /*env_policy*/ None, + env: Default::default(), + tty: false, + pipe_stdin: false, + arg0: None, + }) + .await?; + assert_eq!(session.process.process_id().as_str(), process_id); + + let StartedExecProcess { process } = session; + let mut wake_rx = process.subscribe_wake(); + let mut ready_output = String::new(); + let mut after_seq = None; + loop { + let response = + read_process_until_change(Arc::clone(&process), &mut wake_rx, after_seq).await?; + for chunk in response.chunks { + ready_output.push_str(&String::from_utf8_lossy(&chunk.chunk.into_inner())); + after_seq = Some(chunk.seq); + } + if ready_output.contains("ready\n") { + break; + } + if response.closed { + anyhow::bail!("process closed before readiness marker: {ready_output:?}"); + } + after_seq = response.next_seq.checked_sub(1).or(after_seq); + } + + process.signal(ProcessSignal::Interrupt).await?; + let (output, exit_code, closed) = collect_process_output_from_reads(process, wake_rx).await?; + + assert!( + output.contains("signal:2"), + "expected signal handler output, got {output:?}" + ); + assert_eq!(exit_code, Some(7)); + assert!(closed); + Ok(()) +} + +async fn assert_exec_process_signal_reports_unsupported_on_windows(use_remote: bool) -> Result<()> { + let context = create_process_context(use_remote).await?; + let session = context + .backend + .start(ExecParams { + process_id: ProcessId::from("proc-windows-signal"), + argv: vec![ + "cmd".to_string(), + "/C".to_string(), + "echo ready && ping -n 30 127.0.0.1 >NUL".to_string(), + ], + cwd: std::env::current_dir()?, + env_policy: /*env_policy*/ None, + env: Default::default(), + tty: false, + pipe_stdin: false, + arg0: None, + }) + .await?; + + let err = match session.process.signal(ProcessSignal::Interrupt).await { + Ok(()) => anyhow::bail!("Windows non-TTY signal should report unsupported"), + Err(err) => err, + }; + let message = err.to_string(); + assert!( + message.contains("failed to signal process"), + "unexpected signal error: {message}" + ); + assert!( + message.contains("process interrupt is not supported by this process backend"), + "unexpected signal error: {message}" + ); + + session.process.terminate().await?; + Ok(()) +} + async fn assert_exec_process_preserves_queued_events_before_subscribe( use_remote: bool, ) -> Result<()> { @@ -539,6 +630,7 @@ async fn assert_exec_process_preserves_queued_events_before_subscribe( } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[cfg_attr(not(unix), ignore = "Unix-only exec-server process test")] // Serialize tests that launch a real exec-server process through the full CLI. #[serial_test::serial(remote_exec_server)] async fn remote_exec_process_reports_transport_disconnect() -> Result<()> { @@ -630,6 +722,7 @@ async fn remote_exec_process_reports_transport_disconnect() -> Result<()> { #[test_case(false ; "local")] #[test_case(true ; "remote")] +#[cfg_attr(not(unix), ignore = "Unix-only exec-server process test")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] // Serialize tests that launch a real exec-server process through the full CLI. #[serial_test::serial(remote_exec_server)] @@ -639,6 +732,7 @@ async fn exec_process_starts_and_exits(use_remote: bool) -> Result<()> { #[test_case(false ; "local")] #[test_case(true ; "remote")] +#[cfg_attr(not(unix), ignore = "Unix-only exec-server process test")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] // Serialize tests that launch a real exec-server process through the full CLI. #[serial_test::serial(remote_exec_server)] @@ -648,6 +742,7 @@ async fn exec_process_streams_output(use_remote: bool) -> Result<()> { #[test_case(false ; "local")] #[test_case(true ; "remote")] +#[cfg_attr(not(unix), ignore = "Unix-only exec-server process test")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] // Serialize tests that launch a real exec-server process through the full CLI. #[serial_test::serial(remote_exec_server)] @@ -657,6 +752,7 @@ async fn exec_process_pushes_events(use_remote: bool) -> Result<()> { #[test_case(false ; "local")] #[test_case(true ; "remote")] +#[cfg_attr(not(unix), ignore = "Unix-only exec-server process test")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] // Serialize tests that launch a real exec-server process through the full CLI. #[serial_test::serial(remote_exec_server)] @@ -666,6 +762,7 @@ async fn exec_process_replays_events_after_close(use_remote: bool) -> Result<()> #[test_case(false ; "local")] #[test_case(true ; "remote")] +#[cfg_attr(not(unix), ignore = "Unix-only exec-server process test")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] // Serialize tests that launch a real exec-server process through the full CLI. #[serial_test::serial(remote_exec_server)] @@ -677,6 +774,7 @@ async fn exec_process_retains_output_after_exit_until_streams_close( #[test_case(false ; "local")] #[test_case(true ; "remote")] +#[cfg_attr(not(unix), ignore = "Unix-only exec-server process test")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] // Serialize tests that launch a real exec-server process through the full CLI. #[serial_test::serial(remote_exec_server)] @@ -686,6 +784,7 @@ async fn exec_process_write_then_read(use_remote: bool) -> Result<()> { #[test_case(false ; "local")] #[test_case(true ; "remote")] +#[cfg_attr(not(unix), ignore = "Unix-only exec-server process test")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] // Serialize tests that launch a real exec-server process through the full CLI. #[serial_test::serial(remote_exec_server)] @@ -695,6 +794,7 @@ async fn exec_process_write_then_read_without_tty(use_remote: bool) -> Result<() #[test_case(false ; "local")] #[test_case(true ; "remote")] +#[cfg_attr(not(unix), ignore = "Unix-only exec-server process test")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] // Serialize tests that launch a real exec-server process through the full CLI. #[serial_test::serial(remote_exec_server)] @@ -704,6 +804,27 @@ async fn exec_process_rejects_write_without_pipe_stdin(use_remote: bool) -> Resu #[test_case(false ; "local")] #[test_case(true ; "remote")] +#[cfg_attr(not(unix), ignore = "Unix-only exec-server process test")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +// Serialize tests that launch a real exec-server process through the full CLI. +#[serial_test::serial(remote_exec_server)] +async fn exec_process_signal_interrupts_process(use_remote: bool) -> Result<()> { + assert_exec_process_signal_interrupts_process(use_remote).await +} + +#[test_case(false ; "local")] +#[test_case(true ; "remote")] +#[cfg_attr(not(windows), ignore = "Windows-only exec-server process test")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +// Serialize tests that launch a real exec-server process through the full CLI. +#[serial_test::serial(remote_exec_server)] +async fn exec_process_signal_reports_unsupported_on_windows(use_remote: bool) -> Result<()> { + assert_exec_process_signal_reports_unsupported_on_windows(use_remote).await +} + +#[test_case(false ; "local")] +#[test_case(true ; "remote")] +#[cfg_attr(not(unix), ignore = "Unix-only exec-server process test")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] // Serialize tests that launch a real exec-server process through the full CLI. #[serial_test::serial(remote_exec_server)] diff --git a/codex-rs/utils/pty/src/lib.rs b/codex-rs/utils/pty/src/lib.rs index 39fc9b5522..9f0cb442a2 100644 --- a/codex-rs/utils/pty/src/lib.rs +++ b/codex-rs/utils/pty/src/lib.rs @@ -17,6 +17,8 @@ pub use pipe::spawn_process_no_stdin as spawn_pipe_process_no_stdin; pub use process::ProcessDriver; /// Handle for interacting with a spawned process (PTY or pipe). pub use process::ProcessHandle; +/// Process signal supported by spawned-process handles. +pub use process::ProcessSignal; /// Bundle of process handles plus split output and exit receivers returned by spawn helpers. pub use process::SpawnedProcess; /// Terminal size in character cells used for PTY spawn and resize operations. diff --git a/codex-rs/utils/pty/src/pipe.rs b/codex-rs/utils/pty/src/pipe.rs index 541a2ecf2f..3a9b62d9b7 100644 --- a/codex-rs/utils/pty/src/pipe.rs +++ b/codex-rs/utils/pty/src/pipe.rs @@ -19,7 +19,9 @@ use tokio::task::JoinHandle; use crate::process::ChildTerminator; use crate::process::ProcessHandle; +use crate::process::ProcessSignal; use crate::process::SpawnedProcess; +use crate::process::exit_code_from_status; #[cfg(target_os = "linux")] use libc; @@ -32,6 +34,22 @@ struct PipeChildTerminator { } impl ChildTerminator for PipeChildTerminator { + fn signal(&mut self, signal: ProcessSignal) -> io::Result<()> { + match signal { + ProcessSignal::Interrupt => { + #[cfg(unix)] + { + crate::process_group::interrupt_process_group(self.process_group_id) + } + + #[cfg(not(unix))] + { + Err(crate::process::unsupported_signal(signal)) + } + } + } + } + fn kill(&mut self) -> io::Result<()> { #[cfg(unix)] { @@ -209,7 +227,7 @@ async fn spawn_process_with_stdin_mode( let wait_exit_code = Arc::clone(&exit_code); let wait_handle: JoinHandle<()> = tokio::spawn(async move { let code = match child.wait().await { - Ok(status) => status.code().unwrap_or(-1), + Ok(status) => exit_code_from_status(status), Err(_) => -1, }; wait_exit_status.store(true, std::sync::atomic::Ordering::SeqCst); diff --git a/codex-rs/utils/pty/src/process.rs b/codex-rs/utils/pty/src/process.rs index 898a3d90ff..fdd693f43d 100644 --- a/codex-rs/utils/pty/src/process.rs +++ b/codex-rs/utils/pty/src/process.rs @@ -2,6 +2,7 @@ use core::fmt; use std::io; #[cfg(unix)] use std::os::fd::RawFd; +use std::process::ExitStatus; use std::sync::Arc; use std::sync::Mutex as StdMutex; use std::sync::atomic::AtomicBool; @@ -17,7 +18,39 @@ use tokio::sync::watch; use tokio::task::AbortHandle; use tokio::task::JoinHandle; +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ProcessSignal { + Interrupt, +} + +pub(crate) fn unsupported_signal(signal: ProcessSignal) -> io::Error { + match signal { + ProcessSignal::Interrupt => io::Error::new( + io::ErrorKind::Unsupported, + "process interrupt is not supported by this process backend", + ), + } +} + +pub(crate) fn exit_code_from_status(status: ExitStatus) -> i32 { + if let Some(code) = status.code() { + return code; + } + + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt; + if let Some(signal) = status.signal() { + return 128 + signal; + } + } + + -1 +} + pub(crate) trait ChildTerminator: Send + Sync { + fn signal(&mut self, signal: ProcessSignal) -> io::Result<()>; + fn kill(&mut self) -> io::Result<()>; } @@ -193,6 +226,17 @@ impl ProcessHandle { } } + pub fn signal(&self, signal: ProcessSignal) -> io::Result<()> { + let Ok(mut killer_opt) = self.killer.lock() else { + return Ok(()); + }; + let Some(killer) = killer_opt.as_mut() else { + return Ok(()); + }; + + killer.signal(signal) + } + /// Attempts to kill the child and abort helper tasks. pub fn terminate(&self) { self.request_terminate(); @@ -232,6 +276,10 @@ struct ClosureTerminator { } impl ChildTerminator for ClosureTerminator { + fn signal(&mut self, signal: ProcessSignal) -> io::Result<()> { + Err(unsupported_signal(signal)) + } + fn kill(&mut self) -> io::Result<()> { if let Some(inner) = self.inner.as_mut() { (inner)(); diff --git a/codex-rs/utils/pty/src/process_group.rs b/codex-rs/utils/pty/src/process_group.rs index 22a934fab7..9d10d7c368 100644 --- a/codex-rs/utils/pty/src/process_group.rs +++ b/codex-rs/utils/pty/src/process_group.rs @@ -118,15 +118,10 @@ pub fn kill_process_group_by_pid(_pid: u32) -> io::Result<()> { } #[cfg(unix)] -/// Send SIGTERM to a specific process group ID (best-effort). -/// -/// Returns `Ok(true)` when SIGTERM was delivered to an existing group and -/// `Ok(false)` when the group no longer exists. -pub fn terminate_process_group(process_group_id: u32) -> io::Result { +fn signal_process_group_id(pgid: libc::pid_t, signal: libc::c_int) -> io::Result { use std::io::ErrorKind; - let pgid = process_group_id as libc::pid_t; - let result = unsafe { libc::killpg(pgid, libc::SIGTERM) }; + let result = unsafe { libc::killpg(pgid, signal) }; if result == -1 { let err = io::Error::last_os_error(); if err.kind() == ErrorKind::NotFound || err.raw_os_error() == Some(libc::ESRCH) { @@ -138,27 +133,37 @@ pub fn terminate_process_group(process_group_id: u32) -> io::Result { Ok(true) } +#[cfg(unix)] +/// Send SIGTERM to a specific process group ID (best-effort). +/// +/// Returns `Ok(true)` when SIGTERM was delivered to an existing group and +/// `Ok(false)` when the group no longer exists. +pub fn terminate_process_group(process_group_id: u32) -> io::Result { + signal_process_group_id(process_group_id as libc::pid_t, libc::SIGTERM) +} + #[cfg(not(unix))] /// No-op on non-Unix platforms. pub fn terminate_process_group(_process_group_id: u32) -> io::Result { Ok(false) } +#[cfg(unix)] +/// Send SIGINT to a specific process group ID (best-effort). +pub fn interrupt_process_group(process_group_id: u32) -> io::Result<()> { + signal_process_group_id(process_group_id as libc::pid_t, libc::SIGINT).map(|_| ()) +} + +#[cfg(not(unix))] +/// No-op on non-Unix platforms. +pub fn interrupt_process_group(_process_group_id: u32) -> io::Result<()> { + Ok(()) +} + #[cfg(unix)] /// Kill a specific process group ID (best-effort). pub fn kill_process_group(process_group_id: u32) -> io::Result<()> { - use std::io::ErrorKind; - - let pgid = process_group_id as libc::pid_t; - let result = unsafe { libc::killpg(pgid, libc::SIGKILL) }; - if result == -1 { - let err = io::Error::last_os_error(); - if err.kind() != ErrorKind::NotFound && err.raw_os_error() != Some(libc::ESRCH) { - return Err(err); - } - } - - Ok(()) + signal_process_group_id(process_group_id as libc::pid_t, libc::SIGKILL).map(|_| ()) } #[cfg(not(unix))] diff --git a/codex-rs/utils/pty/src/pty.rs b/codex-rs/utils/pty/src/pty.rs index 45c587b328..951818a80d 100644 --- a/codex-rs/utils/pty/src/pty.rs +++ b/codex-rs/utils/pty/src/pty.rs @@ -30,10 +30,13 @@ use tokio::task::JoinHandle; use crate::process::ChildTerminator; use crate::process::ProcessHandle; +use crate::process::ProcessSignal; use crate::process::PtyHandles; use crate::process::PtyMasterHandle; use crate::process::SpawnedProcess; use crate::process::TerminalSize; +#[cfg(unix)] +use crate::process::exit_code_from_status; /// Returns true when ConPTY support is available (Windows only). #[cfg(windows)] @@ -54,6 +57,19 @@ struct PtyChildTerminator { } impl ChildTerminator for PtyChildTerminator { + fn signal(&mut self, signal: ProcessSignal) -> std::io::Result<()> { + match signal { + ProcessSignal::Interrupt => { + #[cfg(unix)] + if let Some(process_group_id) = self.process_group_id { + return crate::process_group::interrupt_process_group(process_group_id); + } + + Err(crate::process::unsupported_signal(signal)) + } + } + } + fn kill(&mut self) -> std::io::Result<()> { #[cfg(unix)] if let Some(process_group_id) = self.process_group_id { @@ -81,6 +97,14 @@ struct RawPidTerminator { #[cfg(unix)] impl ChildTerminator for RawPidTerminator { + fn signal(&mut self, signal: ProcessSignal) -> std::io::Result<()> { + match signal { + ProcessSignal::Interrupt => { + crate::process_group::interrupt_process_group(self.process_group_id) + } + } + } + fn kill(&mut self) -> std::io::Result<()> { crate::process_group::kill_process_group(self.process_group_id) } @@ -368,7 +392,7 @@ async fn spawn_process_preserving_fds( let wait_exit_code = Arc::clone(&exit_code); let wait_handle: JoinHandle<()> = tokio::task::spawn_blocking(move || { let code = match child.wait() { - Ok(status) => status.code().unwrap_or(-1), + Ok(status) => exit_code_from_status(status), Err(_) => -1, }; wait_exit_status.store(true, std::sync::atomic::Ordering::SeqCst); From cc8325f181401c481f4da8ca16c073867e78d46e Mon Sep 17 00:00:00 2001 From: guinness-oai Date: Tue, 9 Jun 2026 15:20:01 -0700 Subject: [PATCH 15/47] Stop mirroring Codex user input into realtime (#27116) ## Why The realtime frontend model and the backing Codex thread should present one coherent assistant. Raw typed messages, steers, and worker reports belong to the orchestrator; the frontend model should receive the orchestrator's user-facing result rather than a second copy of those inputs. Today normal `turn/start` input is automatically inserted into the realtime conversation, while `turn/steer` is not. Besides creating inconsistent context, this can make the frontend model react independently before Codex has produced the response it should speak. ## What changed - Remove automatic accepted-user-input mirroring into realtime - Remove the mirror-only echo-suppression flag and dead V2 prefix helper - Preserve explicit app-to-realtime text injection and FEM-to-Codex delegation - Replace the positive mirror tests and obsolete snapshots with a negative routing regression test ## Test plan - `cargo test -p codex-core conversation_user_text_turn_is_not_sent_to_realtime` - `cargo test -p codex-core conversation_startup_context_is_truncated_and_sent_once_per_start` - `cargo test -p codex-core inbound_handoff_request_starts_turn` --- codex-rs/core/src/realtime_conversation.rs | 4 - codex-rs/core/src/session/handlers.rs | 47 +---- codex-rs/core/src/session/mod.rs | 1 - .../core/tests/suite/realtime_conversation.rs | 191 +----------------- ...n_is_capped_when_mirrored_to_realtime.snap | 12 -- ..._turn_is_sent_to_realtime_when_active.snap | 10 - 6 files changed, 13 insertions(+), 252 deletions(-) delete mode 100644 codex-rs/core/tests/suite/snapshots/all__suite__realtime_conversation__conversation_user_text_turn_is_capped_when_mirrored_to_realtime.snap delete mode 100644 codex-rs/core/tests/suite/snapshots/all__suite__realtime_conversation__conversation_user_text_turn_is_sent_to_realtime_when_active.snap diff --git a/codex-rs/core/src/realtime_conversation.rs b/codex-rs/core/src/realtime_conversation.rs index 9f35fb2fa4..47bdc8c0c1 100644 --- a/codex-rs/core/src/realtime_conversation.rs +++ b/codex-rs/core/src/realtime_conversation.rs @@ -739,10 +739,6 @@ fn prefix_realtime_text(text: String, prefix: &str, session_kind: RealtimeSessio format!("{prefix}{text}") } -pub(crate) fn prefix_realtime_v2_text(text: String, prefix: &str) -> String { - prefix_realtime_text(text, prefix, RealtimeSessionKind::V2) -} - fn validate_realtime_voice(version: RealtimeWsVersion, voice: RealtimeVoice) -> CodexResult<()> { let voices = RealtimeVoicesList::builtin(); let allowed = match version { diff --git a/codex-rs/core/src/session/handlers.rs b/codex-rs/core/src/session/handlers.rs index c0f62ec664..10b0b27cc1 100644 --- a/codex-rs/core/src/session/handlers.rs +++ b/codex-rs/core/src/session/handlers.rs @@ -15,10 +15,6 @@ use crate::session::session::Session; use crate::session::session::SessionSettingsUpdate; use crate::config::Config; -use crate::realtime_context::REALTIME_TURN_TOKEN_BUDGET; -use crate::realtime_context::truncate_realtime_text_to_token_budget; -use crate::realtime_conversation::REALTIME_USER_TEXT_PREFIX; -use crate::realtime_conversation::prefix_realtime_v2_text; use crate::review_prompts::resolve_review_request; use crate::session::spawn_review_thread; use crate::tasks::CompactTask; @@ -54,9 +50,7 @@ use codex_protocol::request_user_input::RequestUserInputResponse; use crate::context_manager::is_user_turn_boundary; use codex_protocol::dynamic_tools::DynamicToolResponse; -use codex_protocol::items::UserMessageItem; use codex_protocol::mcp::RequestId as ProtocolRequestId; -use codex_protocol::user_input::UserInput; use codex_rmcp_client::ElicitationAction; use codex_rmcp_client::ElicitationResponse; use serde_json::Value; @@ -91,14 +85,7 @@ pub async fn user_input_or_turn( op: Op, client_user_message_id: Option, ) { - user_input_or_turn_inner( - sess, - sub_id, - op, - /*mirror_user_text_to_realtime*/ Some(()), - client_user_message_id, - ) - .await; + user_input_or_turn_inner(sess, sub_id, op, client_user_message_id).await; } pub async fn update_thread_settings( @@ -196,7 +183,6 @@ pub(super) async fn user_input_or_turn_inner( sess: &Arc, sub_id: String, op: Op, - mirror_user_text_to_realtime: Option<()>, client_user_message_id: Option, ) { let Op::UserInput { @@ -230,7 +216,7 @@ pub(super) async fn user_input_or_turn_inner( } sess.maybe_emit_unknown_model_warning_for_turn(current_context.as_ref()) .await; - let accepted_items = match sess + match sess .steer_input( items.clone(), additional_context.clone(), @@ -242,7 +228,6 @@ pub(super) async fn user_input_or_turn_inner( { Ok(_) => { current_context.session_telemetry.user_prompt(&items); - Some(items) } Err(SteerInputError::NoActiveTurn(items)) => { if let Some(responsesapi_client_metadata) = responsesapi_client_metadata { @@ -256,7 +241,6 @@ pub(super) async fn user_input_or_turn_inner( Some(sess.mcp_elicitation_reviewer()), ) .await; - let accepted_items = items.clone(); let additional_context_input = { let mut state = sess.state.lock().await; state.additional_context.merge(additional_context) @@ -278,7 +262,6 @@ pub(super) async fn user_input_or_turn_inner( crate::tasks::RegularTask::new(), ) .await; - Some(accepted_items) } Err(err) => { sess.send_event_raw(Event { @@ -286,33 +269,7 @@ pub(super) async fn user_input_or_turn_inner( msg: EventMsg::Error(err.to_error_event()), }) .await; - None } - }; - if let (Some(items), Some(())) = (accepted_items, mirror_user_text_to_realtime) { - self::mirror_user_text_to_realtime(sess, &items).await; - } -} - -async fn mirror_user_text_to_realtime(sess: &Arc, items: &[UserInput]) { - let text = UserMessageItem::new(items).message(); - if text.is_empty() { - return; - } - let text = if sess.conversation.is_running_v2().await { - prefix_realtime_v2_text(text, REALTIME_USER_TEXT_PREFIX) - } else { - text - }; - let text = truncate_realtime_text_to_token_budget(&text, REALTIME_TURN_TOKEN_BUDGET); - if text.is_empty() { - return; - } - if sess.conversation.running_state().await.is_none() { - return; - } - if let Err(err) = sess.conversation.text_in(text).await { - debug!("failed to mirror user text to realtime conversation: {err}"); } } diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs index 23dbe1f45e..3e71afee8a 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -1132,7 +1132,6 @@ impl Session { additional_context: Default::default(), thread_settings: Default::default(), }, - /*mirror_user_text_to_realtime*/ None, /*client_user_message_id*/ None, ) .await; diff --git a/codex-rs/core/tests/suite/realtime_conversation.rs b/codex-rs/core/tests/suite/realtime_conversation.rs index 1c6763aeba..095b3f59d4 100644 --- a/codex-rs/core/tests/suite/realtime_conversation.rs +++ b/codex-rs/core/tests/suite/realtime_conversation.rs @@ -27,7 +27,6 @@ use codex_protocol::protocol::RealtimeVoice; use codex_protocol::protocol::RolloutItem; use codex_protocol::protocol::SessionSource; use codex_protocol::user_input::UserInput; -use codex_utils_output_truncation::approx_token_count; use core_test_support::responses; use core_test_support::responses::WebSocketConnectionConfig; use core_test_support::responses::start_mock_server; @@ -2099,7 +2098,7 @@ async fn conversation_startup_context_is_truncated_and_sent_once_per_start() -> } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn conversation_user_text_turn_is_sent_to_realtime_when_active() -> Result<()> { +async fn conversation_user_text_turn_is_not_sent_to_realtime() -> Result<()> { skip_if_no_network!(Ok(())); let api_server = start_mock_server().await; @@ -2107,7 +2106,6 @@ async fn conversation_user_text_turn_is_sent_to_realtime_when_active() -> Result &api_server, responses::sse(vec![ responses::ev_response_created("resp_user_text"), - responses::ev_assistant_message("msg_user_text", "ack"), responses::ev_completed("resp_user_text"), ]), ) @@ -2155,7 +2153,6 @@ async fn conversation_user_text_turn_is_sent_to_realtime_when_active() -> Result assert_eq!(session_updated, "sess_user_text"); let user_text = "typed follow-up for realtime"; - let prefixed_user_text = format!("[USER] {user_text}"); test.codex .submit(Op::UserInput { items: vec![UserInput::Text { @@ -2169,181 +2166,22 @@ async fn conversation_user_text_turn_is_sent_to_realtime_when_active() -> Result }) .await?; - wait_for_event(&test.codex, |event| { - matches!(event, EventMsg::TurnComplete(_)) - }) - .await; - - let realtime_text_request = wait_for_matching_websocket_request( - &realtime_server, - "normal user turn text mirrored to realtime", - |request| websocket_request_text(request).as_deref() == Some(prefixed_user_text.as_str()), - ) - .await; - let model_user_texts = response_mock.single_request().message_input_texts("user"); - assert_eq!( - ( - model_user_texts.iter().any(|text| text == user_text), - websocket_request_text(&realtime_text_request), - ), - (true, Some(prefixed_user_text)), - ); - let realtime_response_create = timeout(Duration::from_millis(200), async { - wait_for_matching_websocket_request( - &realtime_server, - "unexpected realtime response request for mirrored user text", - |request| request.body_json()["type"].as_str() == Some("response.create"), - ) - .await - }) - .await; - assert!( - realtime_response_create.is_err(), - "mirrored user text should not request a realtime response" - ); - - let realtime_request_body = realtime_text_request.body_json(); - let content = &realtime_request_body["item"]["content"][0]; - let snapshot = format!( - "type: {}\nitem.type: {}\nitem.role: {}\ncontent[0].type: {}\ncontent[0].text: {}\nresponse.create: {}", - realtime_request_body["type"].as_str().unwrap_or_default(), - realtime_request_body["item"]["type"] - .as_str() - .unwrap_or_default(), - realtime_request_body["item"]["role"] - .as_str() - .unwrap_or_default(), - content["type"].as_str().unwrap_or_default(), - content["text"].as_str().unwrap_or_default(), - realtime_response_create.is_ok(), - ); - insta::assert_snapshot!( - "conversation_user_text_turn_is_sent_to_realtime_when_active", - snapshot - ); - - realtime_server.shutdown().await; - Ok(()) -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn conversation_user_text_turn_is_capped_when_mirrored_to_realtime() -> Result<()> { - skip_if_no_network!(Ok(())); - - let api_server = start_mock_server().await; - let response_mock = responses::mount_sse_once( - &api_server, - responses::sse(vec![ - responses::ev_response_created("resp_long_user_text"), - responses::ev_assistant_message("msg_long_user_text", "ack"), - responses::ev_completed("resp_long_user_text"), - ]), - ) - .await; - - let realtime_server = start_websocket_server(vec![vec![ - vec![json!({ - "type": "session.updated", - "session": { "id": "sess_long_user_text", "instructions": "backend prompt" } - })], - vec![], - ]]) - .await; - - let mut builder = test_codex().with_config({ - let realtime_base_url = realtime_server.uri().to_string(); - move |config| { - config.experimental_realtime_ws_base_url = Some(realtime_base_url); - config.experimental_realtime_ws_startup_context = Some(String::new()); - } - }); - let test = builder.build(&api_server).await?; - - // Phase 1: start realtime so the next normal user turn mirrors over the - // active WebSocket session. - test.codex - .submit(Op::RealtimeConversationStart(ConversationStartParams { - output_modality: RealtimeOutputModality::Audio, - prompt: Some(Some("backend prompt".to_string())), - realtime_session_id: None, - transport: None, - voice: None, - })) - .await?; - - let session_updated = wait_for_event_match(&test.codex, |msg| match msg { - EventMsg::RealtimeConversationRealtime(RealtimeConversationRealtimeEvent { - payload: - RealtimeEvent::SessionUpdated { - realtime_session_id: session_id, - .. - }, - }) => Some(session_id.clone()), + let turn_complete = wait_for_event_match(&test.codex, |event| match event { + EventMsg::TurnComplete(turn_complete) => Some(turn_complete.clone()), _ => None, }) .await; - assert_eq!(session_updated, "sess_long_user_text"); + assert_eq!(turn_complete.last_agent_message, None); - // Phase 2: submit one oversized text turn. The model request should keep - // the exact user text, while the realtime mirror should get the capped copy. - let user_text = format!( - "mirror-head {} mirror-middle {} mirror-tail", - "alpha ".repeat(900), - "omega ".repeat(900), - ); - test.codex - .submit(Op::UserInput { - items: vec![UserInput::Text { - text: user_text.clone(), - text_elements: Vec::new(), - }], - final_output_json_schema: None, - responsesapi_client_metadata: None, - additional_context: Default::default(), - thread_settings: Default::default(), - }) - .await?; - - wait_for_event(&test.codex, |event| { - matches!(event, EventMsg::TurnComplete(_)) - }) - .await; - - // Phase 3: capture the mirrored WebSocket item; the snapshot below records - // the capped payload shape. - let realtime_text_request = wait_for_matching_websocket_request( - &realtime_server, - "capped normal user turn text mirrored to realtime", - |request| websocket_request_text(request).is_some_and(|text| text.contains("mirror-head")), - ) - .await; - let realtime_text = - websocket_request_text(&realtime_text_request).expect("realtime request text"); let model_user_texts = response_mock.single_request().message_input_texts("user"); + assert!(model_user_texts.iter().any(|text| text == user_text)); - let realtime_request_body = realtime_text_request.body_json(); - let content = &realtime_request_body["item"]["content"][0]; - - // Snapshot the request envelope and capped text together so reviewers can - // see the preserved head/tail and truncation marker in one place. - let snapshot = format!( - "type: {}\nitem.type: {}\nitem.role: {}\ncontent[0].type: {}\nmodel_has_full_user_text: {}\nrealtime_text_equal_full_user_text: {}\nrealtime_text_approx_tokens: {}\ncontent[0].text: {}", - realtime_request_body["type"].as_str().unwrap_or_default(), - realtime_request_body["item"]["type"] - .as_str() - .unwrap_or_default(), - realtime_request_body["item"]["role"] - .as_str() - .unwrap_or_default(), - content["type"].as_str().unwrap_or_default(), - model_user_texts.iter().any(|text| text == &user_text), - realtime_text == user_text, - approx_token_count(&realtime_text), - realtime_text, - ); - insta::assert_snapshot!( - "conversation_user_text_turn_is_capped_when_mirrored_to_realtime", - snapshot + let realtime_connections = realtime_server.connections(); + assert_eq!(realtime_connections.len(), 1); + assert_eq!(realtime_connections[0].len(), 1); + assert_eq!( + realtime_connections[0][0].body_json()["type"].as_str(), + Some("session.update") ); realtime_server.shutdown().await; @@ -3441,7 +3279,6 @@ async fn inbound_handoff_request_steers_active_turn() -> Result<()> { "type": "session.updated", "session": { "id": "sess_steer", "instructions": "backend prompt" } })], - vec![], vec![ json!({ "type": "conversation.input_transcript.delta", @@ -3504,12 +3341,6 @@ async fn inbound_handoff_request_steers_active_turn() -> Result<()> { matches!(event, EventMsg::AgentMessageContentDelta(_)) }) .await; - let _ = wait_for_matching_websocket_request( - &realtime_server, - "first prompt mirrored to realtime", - |request| websocket_request_text(request).as_deref() == Some("first prompt"), - ) - .await; test.codex .submit(Op::RealtimeConversationAudio(ConversationAudioParams { diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__realtime_conversation__conversation_user_text_turn_is_capped_when_mirrored_to_realtime.snap b/codex-rs/core/tests/suite/snapshots/all__suite__realtime_conversation__conversation_user_text_turn_is_capped_when_mirrored_to_realtime.snap deleted file mode 100644 index 3daae5d378..0000000000 --- a/codex-rs/core/tests/suite/snapshots/all__suite__realtime_conversation__conversation_user_text_turn_is_capped_when_mirrored_to_realtime.snap +++ /dev/null @@ -1,12 +0,0 @@ ---- -source: core/tests/suite/realtime_conversation.rs -expression: snapshot ---- -type: conversation.item.create -item.type: message -item.role: user -content[0].type: input_text -model_has_full_user_text: true -realtime_text_equal_full_user_text: false -realtime_text_approx_tokens: 300 -content[0].text: [USER] mirror-head alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alpha alp…2419 tokens truncated…ega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega omega mirror-tail diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__realtime_conversation__conversation_user_text_turn_is_sent_to_realtime_when_active.snap b/codex-rs/core/tests/suite/snapshots/all__suite__realtime_conversation__conversation_user_text_turn_is_sent_to_realtime_when_active.snap deleted file mode 100644 index 3ca8233029..0000000000 --- a/codex-rs/core/tests/suite/snapshots/all__suite__realtime_conversation__conversation_user_text_turn_is_sent_to_realtime_when_active.snap +++ /dev/null @@ -1,10 +0,0 @@ ---- -source: core/tests/suite/realtime_conversation.rs -expression: snapshot ---- -type: conversation.item.create -item.type: message -item.role: user -content[0].type: input_text -content[0].text: [USER] typed follow-up for realtime -response.create: false From 9316acf9b238da7559d7fba67c612712acc3b419 Mon Sep 17 00:00:00 2001 From: Ahmed Ibrahim Date: Tue, 9 Jun 2026 15:32:15 -0700 Subject: [PATCH 16/47] [2/4] Add private Python goal operations (#27111) ## Why The Python SDK must treat the runtime's initial goal turn and its continuations as one logical operation. That requires a private lifecycle engine before the public API can return the existing turn handle and result types. ## What - start goals by composing the existing clear/set goal RPCs - enforce persisted, idle threads and a bounded startup handshake - coalesce continuation notifications under a stable logical turn ID - aggregate items, usage, timing, and terminal status - support rollover-aware steering, interruption, cancellation, and cleanup - provide equivalent sync and async internals This is the second PR in the stack and intentionally adds no public API. ## Test plan - online CI, including the Python SDK suite - behavioral coverage is added in the following two stack PRs --- sdk/python/src/openai_codex/_goal.py | 304 +++++++++++++++++- .../src/openai_codex/_message_router.py | 29 +- sdk/python/src/openai_codex/async_client.py | 54 ++++ sdk/python/src/openai_codex/client.py | 143 +++++++- .../tests/test_app_server_goal_operations.py | 90 ++++++ 5 files changed, 597 insertions(+), 23 deletions(-) create mode 100644 sdk/python/tests/test_app_server_goal_operations.py diff --git a/sdk/python/src/openai_codex/_goal.py b/sdk/python/src/openai_codex/_goal.py index dc1fe358c2..37a63fc28f 100644 --- a/sdk/python/src/openai_codex/_goal.py +++ b/sdk/python/src/openai_codex/_goal.py @@ -1,8 +1,12 @@ +import asyncio import queue import threading import time +from collections import deque from dataclasses import dataclass, field +from typing import AsyncIterator, Awaitable, Callable, Iterator +from .generated.notification_registry import notification_turn_id from .generated.v2_all import ( ThreadGoalClearedNotification, ThreadGoalStatus, @@ -10,8 +14,9 @@ from .generated.v2_all import ( Turn, TurnCompletedNotification, TurnStartedNotification, + TurnStatus, ) -from .models import Notification +from .models import Notification, UnknownNotification class _GoalStreamClosed(Exception): @@ -45,10 +50,16 @@ class _GoalOperationState: _notifications: queue.Queue[Notification | BaseException] = field(default_factory=queue.Queue) _failure: BaseException | None = None _finished: bool = False + _turn_routing_active: bool = False - def observe(self, notification: Notification) -> None: + def observe(self, notification: Notification) -> bool: payload = notification.payload with self._condition: + if not self._turn_routing_active and not isinstance( + payload, + ThreadGoalClearedNotification | ThreadGoalUpdatedNotification, + ): + return False if isinstance(payload, TurnStartedNotification): if self.logical_turn_id is None: self.logical_turn_id = payload.turn.id @@ -73,6 +84,12 @@ class _GoalOperationState: self._finished = True self._condition.notify_all() self._notifications.put(notification) + return True + + def activate_turn_routing(self) -> None: + """Accept physical turns after the previous stored goal is cleared.""" + with self._condition: + self._turn_routing_active = True def wait_for_start(self, timeout: float) -> str | None: """Wait for the runtime-generated first turn without consuming its event.""" @@ -121,16 +138,18 @@ class _GoalOperationState: with self._condition: self.interrupted = True self.interrupt_requested = False + self._condition.notify_all() def cancel_interrupt(self) -> None: with self._condition: self.interrupt_requested = False + self._condition.notify_all() - def explicit_interrupt(self, status: ThreadGoalStatus | None) -> bool: + def explicit_interrupt(self) -> bool: with self._condition: - return self.interrupted or ( - self.interrupt_requested and status == ThreadGoalStatus.paused - ) + while self.interrupt_requested: + self._condition.wait() + return self.interrupted def active_turn(self, *, after: str | None = None) -> str | None: """Wait for the current turn, or return None once the goal has ended.""" @@ -151,6 +170,279 @@ class _GoalOperationState: with self._condition: return self.current_turn_id + def resolve_active_turn(self, expected: str, active: str) -> None: + """Adopt a server-reported active id when routed state is still stale.""" + with self._condition: + if self.current_turn_id in {None, expected}: + self.current_turn_id = active + self._condition.notify_all() + + def turn_for_interrupt(self) -> str | None: + """Return an active or stale turn id that can resolve rollover races.""" + with self._condition: + if self.current_turn_id is not None: + return self.current_turn_id + if self.completed_turn is not None: + return self.completed_turn.id + if self.started_turn is not None: + return self.started_turn.id + return None + def wake_notification_reader(self) -> None: """Release a reader blocked after its stream has been closed.""" self._notifications.put(_GoalStreamClosed()) + + +def _logical_notification(notification: Notification, logical_turn_id: str) -> Notification: + """Return a copy whose turn metadata uses the logical operation id.""" + payload = notification.payload + if isinstance(payload, UnknownNotification): + params = dict(payload.params) + if isinstance(params.get("turnId"), str): + params["turnId"] = logical_turn_id + turn = params.get("turn") + if isinstance(turn, dict) and isinstance(turn.get("id"), str): + params["turn"] = {**turn, "id": logical_turn_id} + return Notification(notification.method, UnknownNotification(params)) + + turn_id = notification_turn_id(payload) + if turn_id is None: + return notification + if hasattr(payload, "turn_id"): + return Notification( + notification.method, + payload.model_copy(update={"turn_id": logical_turn_id}), + ) + if hasattr(payload, "turn"): + logical_turn = payload.turn.model_copy(update={"id": logical_turn_id}) + return Notification( + notification.method, + payload.model_copy(update={"turn": logical_turn}), + ) + return notification + + +def _logical_completion( + completed: TurnCompletedNotification, + *, + logical_turn_id: str, + started: Turn | None, + interrupted: bool, +) -> TurnCompletedNotification: + """Coalesce the final physical completion into one logical completion.""" + final_turn = completed.turn + started_at = started.started_at if started is not None else final_turn.started_at + duration_ms = final_turn.duration_ms + if started_at is not None and final_turn.completed_at is not None: + duration_ms = max(0, final_turn.completed_at - started_at) * 1000 + updates: dict[str, object] = { + "id": logical_turn_id, + "started_at": started_at, + "duration_ms": duration_ms, + } + if interrupted: + updates["status"] = TurnStatus.interrupted + return completed.model_copy(update={"turn": final_turn.model_copy(update=updates)}) + + +@dataclass(slots=True) +class _GoalStreamCursor: + """Consume physical goal events as one ordered logical turn stream.""" + + state: _GoalOperationState + started: Turn | None = None + last_completed: TurnCompletedNotification | None = None + failed_completion: TurnCompletedNotification | None = None + status: ThreadGoalStatus | None = None + active: bool = False + cleared: bool = False + + def process(self, notification: Notification) -> tuple[list[Notification], bool]: + logical_turn_id = self.state.logical_turn_id + if logical_turn_id is None: + raise RuntimeError("goal operation has not been bound to a logical turn id") + + payload = notification.payload + if isinstance(payload, TurnStartedNotification): + self.active = True + if self.started is not None: + return [], False + self.started = payload.turn + return [_logical_notification(notification, logical_turn_id)], False + + if isinstance(payload, TurnCompletedNotification): + self.active = False + self.last_completed = payload + if payload.turn.status == TurnStatus.interrupted: + return [ + self._completion( + notification.method, + self.failed_completion or payload, + ) + ], True + if payload.turn.status == TurnStatus.failed: + self.failed_completion = payload + if self.cleared or _terminal_goal_status(self.status): + self.state.finish() + return [self._completion(notification.method, payload)], True + return [], False + if self.status is None and not self.cleared: + raise RuntimeError( + "the connected Codex runtime did not activate goal mode for this turn" + ) + if self.cleared or _terminal_goal_status(self.status): + self.state.finish() + return [ + self._completion( + notification.method, + self.failed_completion or payload, + ) + ], True + return [], False + + events = [_logical_notification(notification, logical_turn_id)] + if isinstance(payload, ThreadGoalUpdatedNotification): + self.status = payload.goal.status + if self.status == ThreadGoalStatus.active: + self.cleared = False + events = [] + elif isinstance(payload, ThreadGoalClearedNotification): + self.cleared = True + events = [] + + if ( + not self.active + and self.last_completed is not None + and (self.cleared or _terminal_goal_status(self.status)) + ): + self.state.finish() + events.append( + self._completion( + "turn/completed", + self.failed_completion or self.last_completed, + ) + ) + return events, True + return events, False + + def _completion( + self, + method: str, + payload: TurnCompletedNotification, + ) -> Notification: + logical_turn_id = self.state.logical_turn_id + if logical_turn_id is None: + raise RuntimeError("goal operation has not been bound to a logical turn id") + return Notification( + method, + _logical_completion( + payload, + logical_turn_id=logical_turn_id, + started=self.started, + interrupted=self.state.explicit_interrupt(), + ), + ) + + +@dataclass(slots=True) +class _GoalNotificationStream(Iterator[Notification]): + """Closeable synchronous view of one logical goal operation.""" + + state: _GoalOperationState + next_notification: Callable[[], Notification] + unregister: Callable[[], None] + cancel_goal: Callable[[], None] + _cursor: _GoalStreamCursor = field(init=False) + _pending: deque[Notification] = field(default_factory=deque) + _closed: bool = False + + def __post_init__(self) -> None: + self._cursor = _GoalStreamCursor(self.state) + + def __iter__(self) -> "_GoalNotificationStream": + return self + + def __next__(self) -> Notification: + if self._closed: + raise StopIteration + try: + while not self._pending: + notification = self.next_notification() + events, completed = self._cursor.process(notification) + self._pending.extend(events) + if completed: + self._finish() + return self._pending.popleft() + except _GoalStreamClosed: + self.close() + raise StopIteration from None + except KeyboardInterrupt: + self.cancel_goal() + self.close() + raise + except BaseException: + self.close() + raise + + def _finish(self) -> None: + if self._closed: + return + self.state.finish() + self.state.wake_notification_reader() + self.unregister() + self._closed = True + + def close(self) -> None: + self._finish() + + +@dataclass(slots=True) +class _AsyncGoalNotificationStream(AsyncIterator[Notification]): + """Closeable asynchronous view of one logical goal operation.""" + + state: _GoalOperationState + next_notification: Callable[[], Awaitable[Notification]] + unregister: Callable[[], None] + cancel_goal: Callable[[], Awaitable[None]] + _cursor: _GoalStreamCursor = field(init=False) + _pending: deque[Notification] = field(default_factory=deque) + _closed: bool = False + + def __post_init__(self) -> None: + self._cursor = _GoalStreamCursor(self.state) + + def __aiter__(self) -> "_AsyncGoalNotificationStream": + return self + + async def __anext__(self) -> Notification: + if self._closed: + raise StopAsyncIteration + try: + while not self._pending: + notification = await self.next_notification() + events, completed = self._cursor.process(notification) + self._pending.extend(events) + if completed: + self._finish() + return self._pending.popleft() + except _GoalStreamClosed: + await self.aclose() + raise StopAsyncIteration from None + except asyncio.CancelledError: + await self.cancel_goal() + await self.aclose() + raise + except BaseException: + await self.aclose() + raise + + def _finish(self) -> None: + if self._closed: + return + self.state.finish() + self.state.wake_notification_reader() + self.unregister() + self._closed = True + + async def aclose(self) -> None: + self._finish() diff --git a/sdk/python/src/openai_codex/_message_router.py b/sdk/python/src/openai_codex/_message_router.py index e71544a0ef..c979c8c8dd 100644 --- a/sdk/python/src/openai_codex/_message_router.py +++ b/sdk/python/src/openai_codex/_message_router.py @@ -121,10 +121,20 @@ class MessageRouter: def register_goal(self, thread_id: str) -> _GoalOperationState: """Register one thread-scoped logical goal operation before it starts.""" state = _GoalOperationState(thread_id=thread_id) + state.activate_turn_routing() + return self._register_goal(state) + + def reserve_goal(self, thread_id: str) -> _GoalOperationState: + """Reserve a thread route without accepting physical turns yet.""" + return self._register_goal(_GoalOperationState(thread_id=thread_id)) + + def _register_goal(self, state: _GoalOperationState) -> _GoalOperationState: with self._lock: - if thread_id in self._goal_operations: - raise RuntimeError(f"thread {thread_id!r} already has an active goal operation") - self._goal_operations[thread_id] = state + if state.thread_id in self._goal_operations: + raise RuntimeError( + f"thread {state.thread_id!r} already has an active goal operation" + ) + self._goal_operations[state.thread_id] = state return state def unregister_goal(self, state: _GoalOperationState) -> None: @@ -133,6 +143,11 @@ class MessageRouter: if self._goal_operations.get(state.thread_id) is state: self._goal_operations.pop(state.thread_id) + def has_goal(self, thread_id: str) -> bool: + """Return whether a logical goal operation owns this thread route.""" + with self._lock: + return thread_id in self._goal_operations + def route_response(self, msg: dict[str, JsonValue]) -> None: """Deliver a JSON-RPC response or error to its request waiter.""" @@ -181,10 +196,10 @@ class MessageRouter: if goal_state is not None and ( turn_id is not None or notification.method.startswith("thread/goal/") ): - goal_state.observe(notification) - if goal_state.is_finished(): - self.unregister_goal(goal_state) - return + if goal_state.observe(notification): + if goal_state.is_finished(): + self.unregister_goal(goal_state) + return if turn_id is None: self._global_notifications.put(notification) return diff --git a/sdk/python/src/openai_codex/async_client.py b/sdk/python/src/openai_codex/async_client.py index ff12f16884..07e7d0d405 100644 --- a/sdk/python/src/openai_codex/async_client.py +++ b/sdk/python/src/openai_codex/async_client.py @@ -1,7 +1,9 @@ from __future__ import annotations import asyncio +import threading from collections.abc import Iterator +from concurrent.futures import Future from typing import AsyncIterator, Callable, ParamSpec, TypeVar from pydantic import BaseModel @@ -227,6 +229,58 @@ class AsyncCodexClient: """Pause the active goal through the wrapped sync client.""" return await self._call_sync(self._sync.pause_goal, thread_id) + async def cancel_goal_operation(self, state: _GoalOperationState) -> None: + """Stop continuation work after a logical goal operation is cancelled.""" + await self._call_sync(self._sync.cancel_goal_operation, state) + + async def start_goal_operation( + self, + thread_id: str, + objective: str, + ) -> tuple[_GoalOperationState, str]: + """Start a logical goal through the wrapped sync client.""" + operation: Future[tuple[_GoalOperationState, str]] = Future() + + def start_operation() -> None: + try: + operation.set_result(self._sync.start_goal_operation(thread_id, objective)) + except BaseException as exc: + operation.set_exception(exc) + + worker = threading.Thread( + target=start_operation, + name="codex-goal-start", + daemon=True, + ) + worker.start() + try: + return await asyncio.shield(asyncio.wrap_future(operation)) + except asyncio.CancelledError: + + def cleanup_cancelled_start( + completed: Future[tuple[_GoalOperationState, str]], + ) -> None: + try: + state, _ = completed.result() + except BaseException: + return + + def stop_cancelled_goal() -> None: + try: + self._sync.cancel_goal_operation(state) + finally: + state.finish() + self._sync.unregister_goal_operation(state) + + threading.Thread( + target=stop_cancelled_goal, + name="codex-goal-start-cleanup", + daemon=True, + ).start() + + operation.add_done_callback(cleanup_cancelled_start) + raise + async def turn_start( self, thread_id: str, diff --git a/sdk/python/src/openai_codex/client.py b/sdk/python/src/openai_codex/client.py index 951f4f95f0..ab5390ae52 100644 --- a/sdk/python/src/openai_codex/client.py +++ b/sdk/python/src/openai_codex/client.py @@ -1,10 +1,13 @@ import json import os +import re import subprocess import threading import uuid +from _thread import LockType from collections import deque -from dataclasses import dataclass +from contextlib import contextmanager +from dataclasses import dataclass, field from pathlib import Path from typing import Callable, Iterator, TypeVar @@ -13,7 +16,7 @@ from pydantic import BaseModel from ._goal import _GoalOperationState from ._message_router import MessageRouter from ._version import __version__ as SDK_VERSION -from .errors import CodexError, TransportClosedError +from .errors import CodexError, InvalidRequestError, TransportClosedError from .generated.notification_registry import NOTIFICATION_MODELS from .generated.v2_all import ( AccountLoginCompletedNotification, @@ -23,6 +26,7 @@ from .generated.v2_all import ( ChatgptLoginAccountResponse, GetAccountParams as V2GetAccountParams, GetAccountResponse, + IdleThreadStatus, LoginAccountParams as V2LoginAccountParams, LoginAccountResponse, LogoutAccountResponse, @@ -61,6 +65,18 @@ from .retry import retry_on_overload ModelT = TypeVar("ModelT", bound=BaseModel) ApprovalHandler = Callable[[str, JsonObject | None], JsonObject] RUNTIME_PKG_NAME = "openai-codex-cli-bin" +_GOAL_START_TIMEOUT_S = 30.0 + + +@dataclass(slots=True) +class _ThreadStartLock: + lock: LockType = field(default_factory=threading.Lock) + users: int = 0 + + +def _active_turn_id_from_error(exc: InvalidRequestError) -> str | None: + match = re.search(r" but found `?([^`]+)`?$", exc.message) + return match.group(1) if match is not None else None def _params_dict( @@ -205,6 +221,8 @@ class CodexClient: self._approval_handler = approval_handler or self._default_approval_handler self._proc: subprocess.Popen[str] | None = None self._lock = threading.Lock() + self._thread_start_locks_guard = threading.Lock() + self._thread_start_locks: dict[str, _ThreadStartLock] = {} self._router = MessageRouter() self._stderr_lines: deque[str] = deque(maxlen=400) self._stderr_thread: threading.Thread | None = None @@ -360,6 +378,10 @@ class CodexClient: """Register a private thread-scoped route for a logical goal turn.""" return self._router.register_goal(thread_id) + def reserve_goal_operation(self, thread_id: str) -> _GoalOperationState: + """Reserve a private thread route before replacing its stored goal.""" + return self._router.reserve_goal(thread_id) + def unregister_goal_operation(self, state: _GoalOperationState) -> None: """Release routing state for one logical goal turn.""" self._router.unregister_goal(state) @@ -499,6 +521,84 @@ class CodexClient: """Pause the active goal used by a logical goal turn.""" return self.thread_goal_set(thread_id, status=ThreadGoalStatus.paused) + def cancel_goal_operation(self, state: _GoalOperationState) -> None: + """Best-effort cleanup after a logical goal operation is cancelled.""" + try: + self.pause_goal(state.thread_id) + except Exception: + pass + self._interrupt_goal_operation(state) + + def _interrupt_goal_operation(self, state: _GoalOperationState) -> None: + turn_id = state.turn_for_interrupt() + if turn_id is None: + return + try: + self.turn_interrupt(state.thread_id, turn_id) + except InvalidRequestError as exc: + if not exc.message.startswith("expected active turn id"): + return + next_turn_id = _active_turn_id_from_error(exc) or state.current_turn() + if next_turn_id is None or next_turn_id == turn_id: + return + try: + self.turn_interrupt(state.thread_id, next_turn_id) + except Exception: + pass + except Exception: + pass + + def start_goal_operation( + self, + thread_id: str, + objective: str, + ) -> tuple[_GoalOperationState, str]: + """Start a logical goal and wait for its runtime-generated first turn.""" + with self._thread_start_lock(thread_id): + return self._start_goal_operation(thread_id, objective) + + def _start_goal_operation( + self, + thread_id: str, + objective: str, + ) -> tuple[_GoalOperationState, str]: + thread = self.thread_read(thread_id).thread + if not isinstance(thread.status.root, IdleThreadStatus): + raise InvalidRequestError( + -32600, + f"thread must be idle before starting a goal: {thread_id}", + ) + if thread.ephemeral or thread.path is None: + raise InvalidRequestError( + -32600, + f"thread must be persisted before starting a goal: {thread_id}", + ) + + state = self.reserve_goal_operation(thread_id) + activated = False + try: + self.thread_goal_clear(thread_id) + state.activate_turn_routing() + self.thread_goal_set( + thread_id, + objective=objective, + status=ThreadGoalStatus.active, + ) + activated = True + turn_id = state.wait_for_start(_GOAL_START_TIMEOUT_S) + if turn_id is None: + raise CodexError( + "timed out waiting for goal turn to start after " + f"{int(_GOAL_START_TIMEOUT_S)} seconds" + ) + return state, turn_id + except BaseException as exc: + if activated or not isinstance(exc, InvalidRequestError): + self.cancel_goal_operation(state) + state.finish() + self.unregister_goal_operation(state) + raise + def turn_start( self, thread_id: str, @@ -506,14 +606,37 @@ class CodexClient: params: V2TurnStartParams | JsonObject | None = None, ) -> TurnStartResponse: """Start a turn and register its notification queue as early as possible.""" - payload = { - **_params_dict(params), - "threadId": thread_id, - "input": self._normalize_input_items(input_items), - } - started = self.request("turn/start", payload, response_model=TurnStartResponse) - self.register_turn_notifications(started.turn.id) - return started + with self._thread_start_lock(thread_id): + if self._router.has_goal(thread_id): + raise InvalidRequestError( + -32600, + f"thread has an active goal operation: {thread_id}", + ) + payload = { + **_params_dict(params), + "threadId": thread_id, + "input": self._normalize_input_items(input_items), + } + started = self.request("turn/start", payload, response_model=TurnStartResponse) + self.register_turn_notifications(started.turn.id) + return started + + @contextmanager + def _thread_start_lock(self, thread_id: str) -> Iterator[None]: + with self._thread_start_locks_guard: + entry = self._thread_start_locks.get(thread_id) + if entry is None: + entry = _ThreadStartLock() + self._thread_start_locks[thread_id] = entry + entry.users += 1 + try: + with entry.lock: + yield + finally: + with self._thread_start_locks_guard: + entry.users -= 1 + if entry.users == 0: + self._thread_start_locks.pop(thread_id, None) def turn_interrupt(self, thread_id: str, turn_id: str) -> TurnInterruptResponse: return self.request( diff --git a/sdk/python/tests/test_app_server_goal_operations.py b/sdk/python/tests/test_app_server_goal_operations.py new file mode 100644 index 0000000000..c280c7522c --- /dev/null +++ b/sdk/python/tests/test_app_server_goal_operations.py @@ -0,0 +1,90 @@ +from app_server_harness import ( + AppServerHarness, + ev_assistant_message, + ev_completed, + ev_function_call, + ev_response_created, + sse, +) +from app_server_helpers import agent_message_texts + +from openai_codex import Codex +from openai_codex._goal import _GoalNotificationStream +from openai_codex._run import _collect_turn_result +from openai_codex.generated.notification_registry import notification_turn_id +from openai_codex.generated.v2_all import TurnStatus + + +def test_private_goal_operation_coalesces_runtime_continuations(tmp_path) -> None: + """The private engine should expose automatic continuations as one turn.""" + with AppServerHarness(tmp_path) as harness: + harness.responses.enqueue_assistant_message( + "Initial pass complete.", + response_id="goal-initial", + ) + harness.responses.enqueue_sse( + sse( + [ + ev_response_created("goal-complete-tool"), + ev_function_call( + "call-goal-complete", + "update_goal", + '{"status":"complete"}', + ), + ev_completed("goal-complete-tool"), + ] + ) + ) + harness.responses.enqueue_sse( + sse( + [ + ev_response_created("goal-final"), + ev_assistant_message("msg-goal-final", "Goal complete."), + ev_completed("goal-final"), + ] + ) + ) + + with Codex(config=harness.app_server_config()) as codex: + thread = codex.thread_start() + state, turn_id = codex._client.start_goal_operation( # noqa: SLF001 + thread.id, + "Improve benchmark coverage", + ) + stream = _GoalNotificationStream( + state, + state.next_notification, + lambda: codex._client.unregister_goal_operation(state), # noqa: SLF001 + lambda: codex._client.cancel_goal_operation(state), # noqa: SLF001 + ) + events = list(stream) + result = _collect_turn_result(iter(events), turn_id=turn_id) + routes = codex._client._router._goal_operations.copy() # noqa: SLF001 + requests = harness.responses.wait_for_requests(3) + + lifecycle = [event.method for event in events if event.method.startswith("turn/")] + routed_ids = [ + routed_id + for event in events + if (routed_id := notification_turn_id(event.payload)) is not None + ] + assert { + "lifecycle": lifecycle, + "routed_ids": routed_ids, + "result": (result.id, result.status, result.final_response), + "messages": agent_message_texts(events), + "request_count": len(requests), + "objective_reached_model": ( + "\nImprove benchmark coverage\n" + in "\n".join(requests[0].message_input_texts("user")) + ), + "routes_after_completion": routes, + } == { + "lifecycle": ["turn/started", "turn/completed"], + "routed_ids": [turn_id] * len(routed_ids), + "result": (turn_id, TurnStatus.completed, "Goal complete."), + "messages": ["Initial pass complete.", "Goal complete."], + "request_count": 3, + "objective_reached_model": True, + "routes_after_completion": {}, + } From 490340ffcf593518bcd843ee903d1ea2160d5363 Mon Sep 17 00:00:00 2001 From: ssetty-oai Date: Tue, 9 Jun 2026 15:49:48 -0700 Subject: [PATCH 17/47] [codex] Retry streamable HTTP initialize failures (#25147) ## Summary - Retry transient streamable HTTP failures during RMCP startup when the failure happens while sending the initialize request. - Retry transient streamable HTTP failures for tools/list, which is read-only and safe to replay. - Cover both retryable HTTP statuses and request-layer failures where no HTTP status is returned. - Surface retryable HTTP statuses from the streamable HTTP adapter as typed client errors. - Add integration coverage for initialize retry, tools/list retry, no-status request failure retry, and non-retryable initialize status. ## Root cause The observed codex_apps failures can happen before normal tool execution: RMCP startup fails while sending initialize, or the first read-only tools/list fails after startup. Retrying hosted_apps_bridge tools/call would not cover initialize and would risk replaying side-effecting tool calls. This change retries the streamable HTTP handshake itself, recreates the transport between initialize attempts, and retries only tools/list among post-initialize service operations. ## Validation - cargo fmt --package codex-rmcp-client - cargo test -p codex-rmcp-client --test streamable_http_recovery --- .../src/bin/test_streamable_http_server.rs | 119 ++++++-- .../rmcp-client/src/http_client_adapter.rs | 72 ++++- codex-rs/rmcp-client/src/rmcp_client.rs | 154 +++++++++-- .../rmcp-client/src/streamable_http_retry.rs | 239 +++++++++++++++++ .../src/streamable_http_retry_tests.rs | 74 +++++ .../tests/streamable_http_recovery.rs | 253 +++++++++++++++++- .../tests/streamable_http_test_support.rs | 115 +++++++- 7 files changed, 986 insertions(+), 40 deletions(-) create mode 100644 codex-rs/rmcp-client/src/streamable_http_retry.rs create mode 100644 codex-rs/rmcp-client/src/streamable_http_retry_tests.rs diff --git a/codex-rs/rmcp-client/src/bin/test_streamable_http_server.rs b/codex-rs/rmcp-client/src/bin/test_streamable_http_server.rs index 2384d39473..5232c04dd4 100644 --- a/codex-rs/rmcp-client/src/bin/test_streamable_http_server.rs +++ b/codex-rs/rmcp-client/src/bin/test_streamable_http_server.rs @@ -8,6 +8,7 @@ use std::time::Duration; use axum::Router; use axum::body::Body; +use axum::body::to_bytes; use axum::extract::Json; use axum::extract::State; use axum::http::HeaderMap; @@ -48,6 +49,7 @@ use rmcp::transport::StreamableHttpServerConfig; use rmcp::transport::StreamableHttpService; use rmcp::transport::streamable_http_server::session::local::LocalSessionManager; use serde::Deserialize; +use serde_json::Value; use serde_json::json; use tokio::sync::Mutex; use tokio::task; @@ -64,18 +66,32 @@ const MEMO_URI: &str = "memo://codex/example-note"; const MEMO_CONTENT: &str = "This is a sample MCP resource served by the rmcp test server."; const MCP_SESSION_ID_HEADER: &str = "mcp-session-id"; const SESSION_POST_FAILURE_CONTROL_PATH: &str = "/test/control/session-post-failure"; +const INITIALIZE_POST_FAILURE_CONTROL_PATH: &str = "/test/control/initialize-post-failure"; +const INITIALIZED_NOTIFICATION_POST_FAILURE_CONTROL_PATH: &str = + "/test/control/initialized-notification-post-failure"; +const MAX_MCP_POST_BODY_BYTES: usize = 1024 * 1024; #[derive(Clone, Default)] -struct SessionFailureState { +struct PostFailureState { armed_failure: Arc>>, } +#[derive(Clone, Copy, Debug)] +enum ArmedFailureTarget { + Initialize, + InitializedNotification, + Session, +} + #[derive(Clone, Debug)] struct ArmedFailure { + target: ArmedFailureTarget, status: StatusCode, remaining: usize, /// Raw `WWW-Authenticate` challenge header field values returned with the failure. www_authenticate_headers: Vec, + content_type: Option, + body: Option, } #[derive(Debug, Deserialize)] @@ -85,6 +101,8 @@ struct ArmSessionPostFailureRequest { /// Raw `WWW-Authenticate` challenge header field values to add to the failure. #[serde(default)] www_authenticate_headers: Vec, + content_type: Option, + body: Option, } #[derive(Deserialize)] @@ -97,7 +115,7 @@ struct EchoArgs { #[tokio::main] async fn main() -> Result<(), Box> { let bind_addr = parse_bind_addr()?; - let session_failure_state = SessionFailureState::default(); + let post_failure_state = PostFailureState::default(); const MAX_BIND_RETRIES: u32 = 20; const BIND_RETRY_DELAY: Duration = Duration::from_millis(50); @@ -129,6 +147,14 @@ async fn main() -> Result<(), Box> { SESSION_POST_FAILURE_CONTROL_PATH, post(arm_session_post_failure), ) + .route( + INITIALIZE_POST_FAILURE_CONTROL_PATH, + post(arm_initialize_post_failure), + ) + .route( + INITIALIZED_NOTIFICATION_POST_FAILURE_CONTROL_PATH, + post(arm_initialized_notification_post_failure), + ) .route( "/.well-known/oauth-authorization-server/mcp", get({ @@ -162,10 +188,10 @@ async fn main() -> Result<(), Box> { ), ) .layer(middleware::from_fn_with_state( - session_failure_state.clone(), - fail_session_post_when_armed, + post_failure_state.clone(), + fail_mcp_post_when_armed, )) - .with_state(session_failure_state); + .with_state(post_failure_state); let router = if let Ok(token) = std::env::var("MCP_EXPECT_BEARER") { let expected = Arc::new(format!("Bearer {token}")); @@ -404,8 +430,30 @@ async fn require_bearer( } async fn arm_session_post_failure( - State(state): State, + State(state): State, Json(request): Json, +) -> Result { + arm_post_failure(state, request, ArmedFailureTarget::Session).await +} + +async fn arm_initialize_post_failure( + State(state): State, + Json(request): Json, +) -> Result { + arm_post_failure(state, request, ArmedFailureTarget::Initialize).await +} + +async fn arm_initialized_notification_post_failure( + State(state): State, + Json(request): Json, +) -> Result { + arm_post_failure(state, request, ArmedFailureTarget::InitializedNotification).await +} + +async fn arm_post_failure( + state: PostFailureState, + request: ArmSessionPostFailureRequest, + target: ArmedFailureTarget, ) -> Result { let status = StatusCode::from_u16(request.status).map_err(|_| StatusCode::BAD_REQUEST)?; let www_authenticate_headers = request @@ -413,46 +461,76 @@ async fn arm_session_post_failure( .into_iter() .map(|value| HeaderValue::from_str(&value).map_err(|_| StatusCode::BAD_REQUEST)) .collect::, _>>()?; + let content_type = request + .content_type + .map(|value| HeaderValue::from_str(&value).map_err(|_| StatusCode::BAD_REQUEST)) + .transpose()?; let armed_failure = if request.remaining == 0 { None } else { Some(ArmedFailure { + target, status, remaining: request.remaining, www_authenticate_headers, + content_type, + body: request.body, }) }; *state.armed_failure.lock().await = armed_failure; Ok(StatusCode::NO_CONTENT) } -async fn fail_session_post_when_armed( - State(state): State, +async fn fail_mcp_post_when_armed( + State(state): State, request: Request, next: Next, ) -> Response { - if request.uri().path() != "/mcp" - || request.method() != Method::POST - || !request.headers().contains_key(MCP_SESSION_ID_HEADER) - { + if request.uri().path() != "/mcp" || request.method() != Method::POST { return next.run(request).await; } + let (parts, body) = request.into_parts(); + let body_bytes = match to_bytes(body, MAX_MCP_POST_BODY_BYTES).await { + Ok(body_bytes) => body_bytes, + Err(_) => { + let mut response = Response::new(Body::from("failed to read request body")); + *response.status_mut() = StatusCode::BAD_REQUEST; + return response; + } + }; + let has_session_id = parts.headers.contains_key(MCP_SESSION_ID_HEADER); + let mcp_method = request_mcp_method(&body_bytes); { let mut armed_failure = state.armed_failure.lock().await; if let Some(failure) = armed_failure.as_mut() && failure.remaining > 0 + && match failure.target { + ArmedFailureTarget::Initialize => !has_session_id, + ArmedFailureTarget::InitializedNotification => { + has_session_id && mcp_method.as_deref() == Some("notifications/initialized") + } + ArmedFailureTarget::Session => { + has_session_id && mcp_method.as_deref() != Some("notifications/initialized") + } + } { failure.remaining -= 1; let status = failure.status; let www_authenticate_headers = failure.www_authenticate_headers.clone(); + let content_type = failure.content_type.clone(); + let body = failure + .body + .clone() + .unwrap_or_else(|| format!("forced session failure with status {status}")); if failure.remaining == 0 { *armed_failure = None; } - let mut response = Response::new(Body::from(format!( - "forced session failure with status {status}" - ))); + let mut response = Response::new(Body::from(body)); *response.status_mut() = status; + if let Some(content_type) = content_type { + response.headers_mut().insert(CONTENT_TYPE, content_type); + } for www_authenticate_header in www_authenticate_headers { response .headers_mut() @@ -462,5 +540,14 @@ async fn fail_session_post_when_armed( } } - next.run(request).await + next.run(Request::from_parts(parts, Body::from(body_bytes))) + .await +} + +fn request_mcp_method(body: &[u8]) -> Option { + serde_json::from_slice::(body) + .ok()? + .get("method")? + .as_str() + .map(ToString::to_string) } diff --git a/codex-rs/rmcp-client/src/http_client_adapter.rs b/codex-rs/rmcp-client/src/http_client_adapter.rs index 6f98f78920..a9cdc7b88f 100644 --- a/codex-rs/rmcp-client/src/http_client_adapter.rs +++ b/codex-rs/rmcp-client/src/http_client_adapter.rs @@ -28,6 +28,8 @@ use reqwest::header::CONTENT_TYPE; use reqwest::header::HeaderMap; use reqwest::header::HeaderName; use rmcp::model::ClientJsonRpcMessage; +use rmcp::model::ClientNotification; +use rmcp::model::ConstString; use rmcp::model::JsonRpcMessage; use rmcp::model::ServerJsonRpcMessage; use rmcp::transport::streamable_http_client::AuthRequiredError; @@ -185,6 +187,25 @@ impl StreamableHttpClient for StreamableHttpClientAdapter { let content_type = response_header(&response.headers, CONTENT_TYPE); let session_id = response_header(&response.headers, HEADER_SESSION_ID); + if !status_is_success(response.status) { + let body = collect_body(&mut body_stream).await?; + if !retryable_post_response_status(mcp_method.as_deref(), response.status) + && content_type + .as_deref() + .is_some_and(|content_type| content_type.starts_with(JSON_MIME_TYPE)) + && let Some(message) = parse_json_rpc_error(&body) + { + return Ok(StreamableHttpPostResponse::Json(message, session_id)); + } + return Err(StreamableHttpError::UnexpectedServerResponse( + format!( + "HTTP {}: {}", + response.status, + body_preview(String::from_utf8_lossy(&body).to_string()) + ) + .into(), + )); + } match content_type.as_deref() { Some(content_type) if content_type.starts_with(EVENT_STREAM_MIME_TYPE) => { let event_stream = sse_stream_from_body(body_stream); @@ -380,7 +401,26 @@ fn client_jsonrpc_message_fields( Some(request.id.to_string()), ), JsonRpcMessage::Response(response) => (None, Some(response.id.to_string())), - JsonRpcMessage::Notification(_) => (None, None), + JsonRpcMessage::Notification(notification) => { + let method = match ¬ification.notification { + ClientNotification::CancelledNotification(notification) => { + notification.method.as_str() + } + ClientNotification::ProgressNotification(notification) => { + notification.method.as_str() + } + ClientNotification::InitializedNotification(notification) => { + notification.method.as_str() + } + ClientNotification::RootsListChangedNotification(notification) => { + notification.method.as_str() + } + ClientNotification::CustomNotification(notification) => { + notification.method.as_str() + } + }; + (Some(method.to_string()), None) + } JsonRpcMessage::Error(error) => (None, error.id.as_ref().map(ToString::to_string)), } } @@ -463,6 +503,36 @@ fn status_is_success(status: u16) -> bool { StatusCode::from_u16(status).is_ok_and(|status| status.is_success()) } +fn retryable_post_response_status(mcp_method: Option<&str>, status: u16) -> bool { + let Ok(status) = StatusCode::from_u16(status) else { + return false; + }; + is_retryable_http_status(status) + && matches!( + mcp_method, + Some("initialize" | "notifications/initialized" | "tools/list") + ) +} + +fn is_retryable_http_status(status: StatusCode) -> bool { + matches!( + status, + StatusCode::REQUEST_TIMEOUT + | StatusCode::TOO_MANY_REQUESTS + | StatusCode::INTERNAL_SERVER_ERROR + | StatusCode::BAD_GATEWAY + | StatusCode::SERVICE_UNAVAILABLE + | StatusCode::GATEWAY_TIMEOUT + ) +} + +fn parse_json_rpc_error(body: &[u8]) -> Option { + match serde_json::from_slice::(body) { + Ok(message @ JsonRpcMessage::Error(_)) => Some(message), + _ => None, + } +} + async fn collect_body( body_stream: &mut HttpResponseBodyStream, ) -> std::result::Result, StreamableHttpError> { diff --git a/codex-rs/rmcp-client/src/rmcp_client.rs b/codex-rs/rmcp-client/src/rmcp_client.rs index 90b09d724c..a28403e67c 100644 --- a/codex-rs/rmcp-client/src/rmcp_client.rs +++ b/codex-rs/rmcp-client/src/rmcp_client.rs @@ -74,6 +74,13 @@ use crate::utils::apply_default_headers; use crate::utils::build_default_headers; use codex_config::types::OAuthCredentialsStoreMode; +#[path = "streamable_http_retry.rs"] +mod streamable_http_retry; + +use self::streamable_http_retry::HandshakeError; +use self::streamable_http_retry::STREAMABLE_HTTP_RETRY_DELAYS_MS; +use self::streamable_http_retry::sleep_with_retry_deadline; + enum PendingTransport { InProcess { transport: tokio::io::DuplexStream, @@ -223,6 +230,25 @@ enum ClientOperationError { Timeout { label: String, duration: Duration }, } +fn remaining_operation_timeout( + label: &str, + timeout: Option, + deadline: Option, +) -> std::result::Result, ClientOperationError> { + let Some(deadline) = deadline else { + return Ok(None); + }; + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + Err(ClientOperationError::Timeout { + label: label.to_string(), + duration: timeout.unwrap_or(remaining), + }) + } else { + Ok(Some(remaining)) + } +} + pub type Elicitation = CreateElicitationRequestParams; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -396,9 +422,13 @@ impl RmcpClient { } }; - let (service, oauth_persistor) = - Self::connect_pending_transport(pending_transport, client_service.clone(), timeout) - .await?; + let (service, oauth_persistor) = self + .connect_pending_transport_with_initialize_retries( + pending_transport, + client_service.clone(), + timeout, + ) + .await?; let initialize_result_rmcp = service .peer() @@ -845,14 +875,31 @@ impl RmcpClient { ), }; - let service = match timeout { - Some(duration) => time::timeout(duration, transport) - .await - .map_err(|_| anyhow!("timed out handshaking with MCP server after {duration:?}"))? - .map_err(|err| anyhow!("handshaking with MCP server failed: {err}"))?, + let service_result = match timeout { + Some(duration) => match time::timeout(duration, transport).await { + Ok(result) => { + result.map_err(|source| anyhow::Error::from(HandshakeError { source })) + } + Err(_elapsed) => Err(anyhow!( + "timed out handshaking with MCP server after {duration:?}" + )), + }, None => transport .await - .map_err(|err| anyhow!("handshaking with MCP server failed: {err}"))?, + .map_err(|source| anyhow::Error::from(HandshakeError { source })), + }; + let service = match service_result { + Ok(service) => service, + Err(error) => { + if let Some(runtime) = oauth_persistor.as_ref() + && let Err(persist_error) = runtime.persist_if_needed().await + { + warn!( + "failed to persist OAuth tokens after failed initialize: {persist_error}" + ); + } + return Err(error); + } }; Ok((Arc::new(service), oauth_persistor)) @@ -869,7 +916,7 @@ impl RmcpClient { Fut: std::future::Future>, { let service = self.service().await?; - match Self::run_service_operation_once( + match Self::run_service_operation_with_transient_retries( Arc::clone(&service), label, timeout, @@ -882,7 +929,7 @@ impl RmcpClient { Err(error) if Self::is_session_expired_404(&error) => { self.reinitialize_after_session_expiry(&service).await?; let recovered_service = self.service().await?; - Self::run_service_operation_once( + Self::run_service_operation_with_transient_retries( recovered_service, label, timeout, @@ -896,6 +943,62 @@ impl RmcpClient { } } + async fn run_service_operation_with_transient_retries( + service: Arc>, + label: &str, + timeout: Option, + pause_state: ElicitationPauseState, + operation: &F, + ) -> std::result::Result + where + F: Fn(Arc>) -> Fut, + Fut: std::future::Future>, + { + let retry_deadline = timeout.map(|duration| Instant::now() + duration); + for (attempt, retry_delay_ms) in STREAMABLE_HTTP_RETRY_DELAYS_MS + .iter() + .copied() + .map(Some) + .chain(std::iter::once(None)) + .enumerate() + { + let attempt_timeout = remaining_operation_timeout(label, timeout, retry_deadline)?; + match Self::run_service_operation_once( + Arc::clone(&service), + label, + attempt_timeout, + pause_state.clone(), + operation, + ) + .await + { + Ok(result) => return Ok(result), + Err(error) if Self::is_retryable_tools_list_error(label, &error) => { + let Some(retry_delay_ms) = retry_delay_ms else { + return Err(error); + }; + let delay = Duration::from_millis(retry_delay_ms); + warn!( + attempt = attempt + 1, + max_attempts = STREAMABLE_HTTP_RETRY_DELAYS_MS.len() + 1, + delay_ms = delay.as_millis(), + error = %error, + "streamable HTTP MCP tools/list failed with a retryable error; retrying" + ); + if !sleep_with_retry_deadline(delay, retry_deadline).await { + return Err(ClientOperationError::Timeout { + label: label.to_string(), + duration: timeout.unwrap_or(delay), + }); + } + } + Err(error) => return Err(error), + } + } + + unreachable!("service operation retry loop should return on success or final error") + } + async fn run_service_operation_once( service: Arc>, label: &str, @@ -921,6 +1024,22 @@ impl RmcpClient { } } + fn is_retryable_tools_list_error(label: &str, error: &ClientOperationError) -> bool { + if label != "tools/list" { + return false; + } + let ClientOperationError::Service(rmcp::service::ServiceError::TransportSend(error)) = + error + else { + return false; + }; + + error + .error + .downcast_ref::>() + .is_some_and(Self::is_retryable_streamable_http_error) + } + fn is_session_expired_404(error: &ClientOperationError) -> bool { let ClientOperationError::Service(rmcp::service::ServiceError::TransportSend(error)) = error @@ -974,12 +1093,13 @@ impl RmcpClient { .clone() .ok_or_else(|| anyhow!("MCP client cannot recover before initialize succeeds"))?; let pending_transport = Self::create_pending_transport(&self.transport_recipe).await?; - let (service, oauth_persistor) = Self::connect_pending_transport( - pending_transport, - initialize_context.client_service, - initialize_context.timeout, - ) - .await?; + let (service, oauth_persistor) = self + .connect_pending_transport_with_initialize_retries( + pending_transport, + initialize_context.client_service, + initialize_context.timeout, + ) + .await?; { let mut guard = self.state.lock().await; diff --git a/codex-rs/rmcp-client/src/streamable_http_retry.rs b/codex-rs/rmcp-client/src/streamable_http_retry.rs new file mode 100644 index 0000000000..73da95de58 --- /dev/null +++ b/codex-rs/rmcp-client/src/streamable_http_retry.rs @@ -0,0 +1,239 @@ +use std::sync::Arc; +use std::time::Duration; +use std::time::Instant; + +use anyhow::Result; +use anyhow::anyhow; +use codex_exec_server::ExecServerError; +use reqwest::StatusCode; +use rmcp::service::RoleClient; +use rmcp::service::RunningService; +use rmcp::transport::streamable_http_client::StreamableHttpError; +use tokio::time; +use tracing::warn; + +use crate::elicitation_client_service::ElicitationClientService; +use crate::http_client_adapter::StreamableHttpClientAdapterError; +use crate::oauth::OAuthPersistor; + +use super::PendingTransport; +use super::RmcpClient; + +const JSON_RPC_INTERNAL_ERROR_CODE: i64 = -32603; +pub(super) const STREAMABLE_HTTP_RETRY_DELAYS_MS: [u64; 2] = [250, 1_000]; + +impl RmcpClient { + pub(super) async fn connect_pending_transport_with_initialize_retries( + &self, + initial_transport: PendingTransport, + client_service: ElicitationClientService, + timeout: Option, + ) -> Result<( + Arc>, + Option, + )> { + let should_retry = match &initial_transport { + PendingTransport::InProcess { .. } | PendingTransport::Stdio { .. } => false, + PendingTransport::StreamableHttp { .. } + | PendingTransport::StreamableHttpWithOAuth { .. } => true, + }; + let retry_deadline = timeout.map(|duration| Instant::now() + duration); + let mut pending_transport = Some(initial_transport); + + for (attempt, retry_delay_ms) in STREAMABLE_HTTP_RETRY_DELAYS_MS + .iter() + .copied() + .map(Some) + .chain(std::iter::once(None)) + .enumerate() + { + let transport = match pending_transport.take() { + Some(transport) => transport, + None => { + let remaining = remaining_initialize_timeout(timeout, retry_deadline)?; + match remaining { + Some(remaining) => time::timeout( + remaining, + Self::create_pending_transport(&self.transport_recipe), + ) + .await + .map_err(|_| initialize_timeout_error(timeout, remaining))??, + None => Self::create_pending_transport(&self.transport_recipe).await?, + } + } + }; + let attempt_timeout = remaining_initialize_timeout(timeout, retry_deadline)?; + + match Self::connect_pending_transport( + transport, + client_service.clone(), + attempt_timeout, + ) + .await + { + Ok(result) => return Ok(result), + Err(error) if should_retry && Self::is_retryable_initialize_error(&error) => { + let Some(retry_delay_ms) = retry_delay_ms else { + return Err(error); + }; + let delay = Duration::from_millis(retry_delay_ms); + warn!( + attempt = attempt + 1, + max_attempts = STREAMABLE_HTTP_RETRY_DELAYS_MS.len() + 1, + delay_ms = delay.as_millis(), + error = %error, + "streamable HTTP MCP initialize failed with a retryable error; retrying" + ); + if !sleep_with_retry_deadline(delay, retry_deadline).await { + let duration = timeout.unwrap_or(delay); + return Err(anyhow!( + "timed out handshaking with MCP server after {duration:?}" + )); + } + } + Err(error) => return Err(error), + } + } + + unreachable!("initialize retry loop should return on success or final error") + } + + fn is_retryable_initialize_error(error: &anyhow::Error) -> bool { + error.chain().any(|source| { + source + .downcast_ref::() + .is_some_and(|error| Self::is_retryable_client_initialize_error(&error.source)) + || source + .downcast_ref::() + .is_some_and(Self::is_retryable_client_initialize_error) + }) + } + + fn is_retryable_client_initialize_error(error: &rmcp::service::ClientInitializeError) -> bool { + match error { + rmcp::service::ClientInitializeError::TransportError { error, context } + if context.as_ref() == "send initialize request" => + { + error + .error + .downcast_ref::>() + .is_some_and(Self::is_retryable_streamable_http_error) + } + rmcp::service::ClientInitializeError::TransportError { error, context } + if context.as_ref() == "send initialized notification" => + { + error + .error + .downcast_ref::>() + .is_some_and(|error| { + matches!(error, StreamableHttpError::TransportChannelClosed) + || Self::is_retryable_streamable_http_error(error) + }) + } + _ => false, + } + } + + pub(super) fn is_retryable_streamable_http_error( + error: &StreamableHttpError, + ) -> bool { + match error { + StreamableHttpError::Client(StreamableHttpClientAdapterError::HttpRequest( + ExecServerError::HttpRequest(_), + )) => true, + StreamableHttpError::Client(StreamableHttpClientAdapterError::HttpRequest( + ExecServerError::Server { code, message }, + )) => { + *code == JSON_RPC_INTERNAL_ERROR_CODE && message.starts_with("http/request failed:") + } + StreamableHttpError::Client(StreamableHttpClientAdapterError::HttpRequest( + ExecServerError::Protocol(message), + )) => message.starts_with("http response stream `") && message.contains("` failed:"), + StreamableHttpError::UnexpectedServerResponse(message) => { + is_retryable_unexpected_server_response(message.as_ref()) + } + StreamableHttpError::AuthRequired(_) + | StreamableHttpError::InsufficientScope(_) + | StreamableHttpError::SessionExpired + | StreamableHttpError::UnexpectedContentType(_) + | StreamableHttpError::ServerDoesNotSupportSse + | StreamableHttpError::Deserialize(_) + | StreamableHttpError::Client(StreamableHttpClientAdapterError::SessionExpired404) + | StreamableHttpError::Client(StreamableHttpClientAdapterError::Header(_)) => false, + _ => false, + } + } +} + +fn is_retryable_unexpected_server_response(message: &str) -> bool { + let Some(message) = message.strip_prefix("HTTP ") else { + return false; + }; + let status_code = message + .chars() + .take_while(char::is_ascii_digit) + .collect::(); + let Ok(status) = status_code.parse::() else { + return false; + }; + let Ok(status) = StatusCode::from_u16(status) else { + return false; + }; + is_retryable_http_status(status) +} + +fn is_retryable_http_status(status: StatusCode) -> bool { + matches!( + status, + StatusCode::REQUEST_TIMEOUT + | StatusCode::TOO_MANY_REQUESTS + | StatusCode::INTERNAL_SERVER_ERROR + | StatusCode::BAD_GATEWAY + | StatusCode::SERVICE_UNAVAILABLE + | StatusCode::GATEWAY_TIMEOUT + ) +} + +fn remaining_initialize_timeout( + timeout: Option, + deadline: Option, +) -> Result> { + let Some(deadline) = deadline else { + return Ok(None); + }; + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + Err(initialize_timeout_error(timeout, remaining)) + } else { + Ok(Some(remaining)) + } +} + +fn initialize_timeout_error(timeout: Option, fallback: Duration) -> anyhow::Error { + let duration = timeout.unwrap_or(fallback); + anyhow!("timed out handshaking with MCP server after {duration:?}") +} + +pub(super) async fn sleep_with_retry_deadline(delay: Duration, deadline: Option) -> bool { + if let Some(deadline) = deadline { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return false; + } + time::timeout(remaining, time::sleep(delay)).await.is_ok() + } else { + time::sleep(delay).await; + true + } +} + +#[derive(Debug, thiserror::Error)] +#[error("handshaking with MCP server failed: {source}")] +pub(super) struct HandshakeError { + #[source] + pub(super) source: rmcp::service::ClientInitializeError, +} + +#[cfg(test)] +#[path = "streamable_http_retry_tests.rs"] +mod tests; diff --git a/codex-rs/rmcp-client/src/streamable_http_retry_tests.rs b/codex-rs/rmcp-client/src/streamable_http_retry_tests.rs new file mode 100644 index 0000000000..ed0d8030e6 --- /dev/null +++ b/codex-rs/rmcp-client/src/streamable_http_retry_tests.rs @@ -0,0 +1,74 @@ +use std::any::TypeId; + +use codex_exec_server::ExecServerError; +use pretty_assertions::assert_eq; +use rmcp::transport::DynamicTransportError; +use rmcp::transport::streamable_http_client::StreamableHttpError; + +use crate::http_client_adapter::StreamableHttpClientAdapterError; + +use super::*; + +#[test] +fn retryable_initialize_error_includes_initialized_notification_context() { + let contexts = [ + "send initialize request", + "send initialized notification", + "receive initialize response", + ]; + + assert_eq!( + contexts.map(|context| { + RmcpClient::is_retryable_client_initialize_error(&retryable_initialize_error(context)) + }), + [true, true, false], + ); +} + +#[test] +fn retryable_streamable_http_error_includes_remote_body_stream_failure() { + let errors = [ + StreamableHttpError::Client(StreamableHttpClientAdapterError::HttpRequest( + ExecServerError::HttpRequest("error sending request for url".to_string()), + )), + StreamableHttpError::Client(StreamableHttpClientAdapterError::HttpRequest( + ExecServerError::Server { + code: JSON_RPC_INTERNAL_ERROR_CODE, + message: "http/request failed: error sending request for url".to_string(), + }, + )), + StreamableHttpError::Client(StreamableHttpClientAdapterError::HttpRequest( + ExecServerError::Protocol( + "http response stream `http-1` failed: exec-server transport disconnected" + .to_string(), + ), + )), + StreamableHttpError::Client(StreamableHttpClientAdapterError::HttpRequest( + ExecServerError::Protocol( + "http response stream `http-1` received seq 2, expected 1".to_string(), + ), + )), + StreamableHttpError::UnexpectedServerResponse("HTTP 502: upstream failure".into()), + StreamableHttpError::UnexpectedServerResponse("HTTP 400: bad request".into()), + ]; + + assert_eq!( + errors.map(|error| RmcpClient::is_retryable_streamable_http_error(&error)), + [true, true, true, false, true, false], + ); +} + +fn retryable_initialize_error(context: &'static str) -> rmcp::service::ClientInitializeError { + rmcp::service::ClientInitializeError::TransportError { + error: DynamicTransportError::from_parts( + "streamable_http", + TypeId::of::<()>(), + Box::new(StreamableHttpError::Client( + StreamableHttpClientAdapterError::HttpRequest(ExecServerError::HttpRequest( + "error sending request for url".to_string(), + )), + )), + ), + context: context.into(), + } +} diff --git a/codex-rs/rmcp-client/tests/streamable_http_recovery.rs b/codex-rs/rmcp-client/tests/streamable_http_recovery.rs index 087d3d00df..29d2404ede 100644 --- a/codex-rs/rmcp-client/tests/streamable_http_recovery.rs +++ b/codex-rs/rmcp-client/tests/streamable_http_recovery.rs @@ -1,13 +1,229 @@ mod streamable_http_test_support; -use pretty_assertions::assert_eq; +use std::sync::Arc; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; +use std::time::Duration; +use codex_exec_server::Environment; +use codex_exec_server::ExecServerError; +use codex_exec_server::HttpClient; +use codex_exec_server::HttpRequestParams; +use codex_exec_server::HttpRequestResponse; +use codex_exec_server::HttpResponseBodyStream; +use futures::FutureExt as _; +use futures::future::BoxFuture; +use pretty_assertions::assert_eq; +use serde_json::Value; + +use streamable_http_test_support::arm_initialize_post_failure; +use streamable_http_test_support::arm_initialize_post_json_rpc_failure; +use streamable_http_test_support::arm_initialized_notification_post_json_rpc_failure; use streamable_http_test_support::arm_session_post_failure; +use streamable_http_test_support::arm_session_post_json_rpc_failure; use streamable_http_test_support::call_echo_tool; use streamable_http_test_support::create_client; +use streamable_http_test_support::create_client_with_http_client; use streamable_http_test_support::expected_echo_result; use streamable_http_test_support::spawn_streamable_http_server; +const JSON_RPC_INTERNAL_ERROR_CODE: i64 = -32603; +const SIMULATED_NO_RESPONSE_MESSAGE: &str = + "http/request failed: error sending request for url (simulated no response)"; + +#[derive(Clone)] +struct FailFirstInitializeHttpClient { + inner: Arc, + failures_remaining: Arc, + initialize_attempts: Arc, +} + +impl FailFirstInitializeHttpClient { + fn new(inner: Arc, failures_remaining: usize) -> Self { + Self { + inner, + failures_remaining: Arc::new(AtomicUsize::new(failures_remaining)), + initialize_attempts: Arc::new(AtomicUsize::new(0)), + } + } + + fn initialize_attempts(&self) -> usize { + self.initialize_attempts.load(Ordering::SeqCst) + } + + fn fail_next_initialize(&self) { + self.failures_remaining.store(1, Ordering::SeqCst); + } +} + +impl HttpClient for FailFirstInitializeHttpClient { + fn http_request( + &self, + params: HttpRequestParams, + ) -> BoxFuture<'_, Result> { + self.inner.http_request(params) + } + + fn http_request_stream( + &self, + params: HttpRequestParams, + ) -> BoxFuture<'_, Result<(HttpRequestResponse, HttpResponseBodyStream), ExecServerError>> { + let inner = Arc::clone(&self.inner); + let failures_remaining = Arc::clone(&self.failures_remaining); + let initialize_attempts = Arc::clone(&self.initialize_attempts); + + async move { + if is_initialize_post(¶ms) { + initialize_attempts.fetch_add(1, Ordering::SeqCst); + if failures_remaining.swap(0, Ordering::SeqCst) > 0 { + return Err(ExecServerError::Server { + code: JSON_RPC_INTERNAL_ERROR_CODE, + message: SIMULATED_NO_RESPONSE_MESSAGE.to_string(), + }); + } + } + + inner.http_request_stream(params).await + } + .boxed() + } +} + +fn is_initialize_post(params: &HttpRequestParams) -> bool { + params.method.eq_ignore_ascii_case("POST") + && params + .body + .as_ref() + .and_then(|body| serde_json::from_slice::(&body.0).ok()) + .and_then(|body| { + body.get("method") + .and_then(Value::as_str) + .map(|method| method == "initialize") + }) + .unwrap_or(false) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn streamable_http_initialize_retries_remote_no_response_error() -> anyhow::Result<()> { + let (_server, base_url) = spawn_streamable_http_server().await?; + let http_client = FailFirstInitializeHttpClient::new( + Environment::default_for_tests().get_http_client(), + /*failures_remaining*/ 1, + ); + + let client = create_client_with_http_client(&base_url, Arc::new(http_client.clone())).await?; + let result = call_echo_tool(&client, "after-init-retry").await?; + + assert_eq!(http_client.initialize_attempts(), 2); + assert_eq!(result, expected_echo_result("after-init-retry")); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn streamable_http_initialize_retries_transient_http_status() -> anyhow::Result<()> { + let (_server, base_url) = spawn_streamable_http_server().await?; + + arm_initialize_post_failure(&base_url, /*status*/ 502, /*remaining*/ 1).await?; + + let client = create_client(&base_url).await?; + let result = call_echo_tool(&client, "after-status-retry").await?; + + assert_eq!(result, expected_echo_result("after-status-retry")); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn streamable_http_initialize_retries_json_rpc_transient_status() -> anyhow::Result<()> { + let (_server, base_url) = spawn_streamable_http_server().await?; + + arm_initialize_post_json_rpc_failure(&base_url, /*status*/ 502, /*remaining*/ 1).await?; + + let client = create_client(&base_url).await?; + let result = call_echo_tool(&client, "after-json-status-retry").await?; + + assert_eq!(result, expected_echo_result("after-json-status-retry")); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn streamable_http_retries_initialized_notification_status() -> anyhow::Result<()> { + let (_server, base_url) = spawn_streamable_http_server().await?; + + arm_initialized_notification_post_json_rpc_failure( + &base_url, /*status*/ 502, /*remaining*/ 1, + ) + .await?; + + let client = create_client(&base_url).await?; + let result = call_echo_tool(&client, "after-notification-status-retry").await?; + + assert_eq!( + result, + expected_echo_result("after-notification-status-retry") + ); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn streamable_http_tools_list_retries_transient_http_status() -> anyhow::Result<()> { + let (_server, base_url) = spawn_streamable_http_server().await?; + let client = create_client(&base_url).await?; + + let expected = client + .list_tools( + /*params*/ None, + /*timeout*/ Some(Duration::from_secs(5)), + ) + .await?; + arm_session_post_failure( + &base_url, + /*status*/ 502, + /*remaining*/ 1, + /*www_authenticate_headers*/ &[], + ) + .await?; + + let result = client + .list_tools( + /*params*/ None, + /*timeout*/ Some(Duration::from_secs(5)), + ) + .await?; + + assert_eq!(result, expected); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn streamable_http_tools_list_retries_json_rpc_transient_status() -> anyhow::Result<()> { + let (_server, base_url) = spawn_streamable_http_server().await?; + let client = create_client(&base_url).await?; + + let expected = client + .list_tools( + /*params*/ None, + /*timeout*/ Some(Duration::from_secs(5)), + ) + .await?; + arm_session_post_json_rpc_failure(&base_url, /*status*/ 502, /*remaining*/ 1).await?; + + let result = client + .list_tools( + /*params*/ None, + /*timeout*/ Some(Duration::from_secs(5)), + ) + .await?; + + assert_eq!(result, expected); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] async fn streamable_http_404_session_expiry_recovers_and_retries_once() -> anyhow::Result<()> { let (_server, base_url) = spawn_streamable_http_server().await?; @@ -30,6 +246,34 @@ async fn streamable_http_404_session_expiry_recovers_and_retries_once() -> anyho Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn streamable_http_session_recovery_retries_initialize_failure() -> anyhow::Result<()> { + let (_server, base_url) = spawn_streamable_http_server().await?; + let http_client = FailFirstInitializeHttpClient::new( + Environment::default_for_tests().get_http_client(), + /*failures_remaining*/ 0, + ); + let client = create_client_with_http_client(&base_url, Arc::new(http_client.clone())).await?; + + let warmup = call_echo_tool(&client, "warmup").await?; + assert_eq!(warmup, expected_echo_result("warmup")); + + arm_session_post_failure( + &base_url, + /*status*/ 404, + /*remaining*/ 1, + /*www_authenticate_headers*/ &[], + ) + .await?; + http_client.fail_next_initialize(); + + let recovered = call_echo_tool(&client, "recovered-after-retry").await?; + assert_eq!(http_client.initialize_attempts(), 3); + assert_eq!(recovered, expected_echo_result("recovered-after-retry")); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] async fn streamable_http_401_does_not_trigger_recovery() -> anyhow::Result<()> { let (_server, base_url) = spawn_streamable_http_server().await?; @@ -129,11 +373,10 @@ async fn streamable_http_404_recovery_only_retries_once() -> anyhow::Result<()> .await?; let error = call_echo_tool(&client, "double-404").await.unwrap_err(); + let error_message = error.to_string(); assert!( - error - .to_string() - .contains("handshaking with MCP server failed") - || error.to_string().contains("Transport channel closed") + error_message.contains("404") || error_message.contains("session expired"), + "expected session-expiry error, got: {error:#}" ); let recovered = call_echo_tool(&client, "after-double-404").await?; diff --git a/codex-rs/rmcp-client/tests/streamable_http_test_support.rs b/codex-rs/rmcp-client/tests/streamable_http_test_support.rs index 03ee79392c..6b5df6e297 100644 --- a/codex-rs/rmcp-client/tests/streamable_http_test_support.rs +++ b/codex-rs/rmcp-client/tests/streamable_http_test_support.rs @@ -20,6 +20,7 @@ use anyhow::Context as _; use codex_config::types::OAuthCredentialsStoreMode; use codex_exec_server::Environment; use codex_exec_server::ExecServerClient; +use codex_exec_server::HttpClient; use codex_exec_server::RemoteExecServerConnectArgs; use codex_rmcp_client::ElicitationAction; use codex_rmcp_client::ElicitationResponse; @@ -44,6 +45,9 @@ use tokio::process::Command; use tokio::time::sleep; const SESSION_POST_FAILURE_CONTROL_PATH: &str = "/test/control/session-post-failure"; +const INITIALIZE_POST_FAILURE_CONTROL_PATH: &str = "/test/control/initialize-post-failure"; +const INITIALIZED_NOTIFICATION_POST_FAILURE_CONTROL_PATH: &str = + "/test/control/initialized-notification-post-failure"; fn streamable_http_server_bin() -> Result { codex_utils_cargo_bin::cargo_bin("test_streamable_http_server") @@ -74,6 +78,14 @@ pub(crate) fn expected_echo_result(message: &str) -> CallToolResult { } pub(crate) async fn create_client(base_url: &str) -> anyhow::Result { + create_client_with_http_client(base_url, Environment::default_for_tests().get_http_client()) + .await +} + +pub(crate) async fn create_client_with_http_client( + base_url: &str, + http_client: Arc, +) -> anyhow::Result { let client = RmcpClient::new_streamable_http_client( "test-streamable-http", &format!("{base_url}/mcp"), @@ -81,7 +93,7 @@ pub(crate) async fn create_client(base_url: &str) -> anyhow::Result /*http_headers*/ None, /*env_http_headers*/ None, OAuthCredentialsStoreMode::File, - Environment::default_for_tests().get_http_client(), + http_client, /*auth_provider*/ None, ) .await?; @@ -183,6 +195,107 @@ pub(crate) async fn arm_session_post_failure( Ok(()) } +pub(crate) async fn arm_session_post_json_rpc_failure( + base_url: &str, + status: u16, + remaining: usize, +) -> anyhow::Result<()> { + let response = reqwest::Client::new() + .post(format!("{base_url}{SESSION_POST_FAILURE_CONTROL_PATH}")) + .json(&json!({ + "status": status, + "remaining": remaining, + "content_type": "application/json", + "body": json!({ + "jsonrpc": "2.0", + "id": 1, + "error": { + "code": -32000, + "message": "transient session failure", + }, + }).to_string(), + })) + .send() + .await?; + + assert_eq!(response.status(), reqwest::StatusCode::NO_CONTENT); + Ok(()) +} + +pub(crate) async fn arm_initialized_notification_post_json_rpc_failure( + base_url: &str, + status: u16, + remaining: usize, +) -> anyhow::Result<()> { + let response = reqwest::Client::new() + .post(format!( + "{base_url}{INITIALIZED_NOTIFICATION_POST_FAILURE_CONTROL_PATH}" + )) + .json(&json!({ + "status": status, + "remaining": remaining, + "content_type": "application/json", + "body": json!({ + "jsonrpc": "2.0", + "id": 1, + "error": { + "code": -32000, + "message": "transient session failure", + }, + }).to_string(), + })) + .send() + .await?; + + assert_eq!(response.status(), reqwest::StatusCode::NO_CONTENT); + Ok(()) +} + +pub(crate) async fn arm_initialize_post_failure( + base_url: &str, + status: u16, + remaining: usize, +) -> anyhow::Result<()> { + let response = reqwest::Client::new() + .post(format!("{base_url}{INITIALIZE_POST_FAILURE_CONTROL_PATH}")) + .json(&json!({ + "status": status, + "remaining": remaining, + })) + .send() + .await?; + + assert_eq!(response.status(), reqwest::StatusCode::NO_CONTENT); + Ok(()) +} + +pub(crate) async fn arm_initialize_post_json_rpc_failure( + base_url: &str, + status: u16, + remaining: usize, +) -> anyhow::Result<()> { + let response = reqwest::Client::new() + .post(format!("{base_url}{INITIALIZE_POST_FAILURE_CONTROL_PATH}")) + .json(&json!({ + "status": status, + "remaining": remaining, + "content_type": "application/json", + "body": json!({ + "jsonrpc": "2.0", + "id": 1, + "error": { + "code": -32000, + "message": "transient initialize failure", + }, + }).to_string(), + })) + .send() + .await?; + + assert_eq!(response.status(), reqwest::StatusCode::NO_CONTENT); + Ok(()) +} + pub(crate) async fn spawn_streamable_http_server() -> anyhow::Result<(Child, String)> { let listener = TcpListener::bind("127.0.0.1:0")?; let port = listener.local_addr()?.port(); From 51b3cd51f6f94488c0e05564cbcad9512f73e3db Mon Sep 17 00:00:00 2001 From: Ahmed Ibrahim Date: Tue, 9 Jun 2026 16:07:34 -0700 Subject: [PATCH 18/47] [codex] Tighten MCP connection manager API visibility and order (#27257) ## Summary - order `McpConnectionManager` methods by visibility, with the primary constructor and public API first - restrict `list_available_server_infos` to `codex-mcp` - make `new_uninitialized` a private test-only helper ## Why The manager exposed methods that are only used inside `codex-mcp` or its unit tests. Tightening those methods keeps the exported API intentional, while the new ordering makes the supported surface easier to scan. ## Validation - `just fmt` - `git diff --check` - local tests not run; relying on CI --- codex-rs/codex-mcp/src/connection_manager.rs | 273 ++++++++++--------- 1 file changed, 137 insertions(+), 136 deletions(-) diff --git a/codex-rs/codex-mcp/src/connection_manager.rs b/codex-rs/codex-mcp/src/connection_manager.rs index d15a2e1500..5558916e5b 100644 --- a/codex-rs/codex-mcp/src/connection_manager.rs +++ b/codex-rs/codex-mcp/src/connection_manager.rs @@ -113,102 +113,6 @@ pub struct McpConnectionManager { } impl McpConnectionManager { - pub fn new_uninitialized( - approval_policy: &Constrained, - permission_profile: &Constrained, - prefix_mcp_tool_names: bool, - ) -> Self { - Self::new_uninitialized_with_permission_profile( - approval_policy, - permission_profile.get(), - prefix_mcp_tool_names, - ) - } - - pub fn new_uninitialized_with_permission_profile( - approval_policy: &Constrained, - permission_profile: &PermissionProfile, - prefix_mcp_tool_names: bool, - ) -> Self { - Self { - clients: HashMap::new(), - server_metadata: HashMap::new(), - tool_plugin_provenance: Arc::new(ToolPluginProvenance::default()), - host_owned_codex_apps_enabled: false, - prefix_mcp_tool_names, - elicitation_requests: ElicitationRequestManager::new( - approval_policy.value(), - permission_profile.clone(), - /*reviewer*/ None, - ), - startup_cancellation_token: CancellationToken::new(), - } - } - - pub fn has_servers(&self) -> bool { - !self.clients.is_empty() - } - - /// Drain all MCP clients from this manager and return a future that stops - /// them and terminates their stdio server processes. - pub fn begin_shutdown(&mut self) -> impl std::future::Future + Send + 'static { - self.startup_cancellation_token.cancel(); - let clients = std::mem::take(&mut self.clients); - self.server_metadata.clear(); - async move { - for client in clients.into_values() { - client.shutdown().await; - } - } - } - - /// Stop all MCP clients owned by this manager and terminate stdio server processes. - pub async fn shutdown(&mut self) { - self.begin_shutdown().await; - } - - pub fn server_origin(&self, server_name: &str) -> Option<&str> { - self.server_metadata - .get(server_name) - .and_then(|metadata| metadata.origin.as_ref()) - .map(super::server::McpServerOrigin::as_str) - } - - pub fn server_pollutes_memory(&self, server_name: &str) -> bool { - self.server_metadata - .get(server_name) - .is_none_or(|metadata| metadata.pollutes_memory) - } - - pub fn plugin_id_for_mcp_server_name(&self, server_name: &str) -> Option<&str> { - self.tool_plugin_provenance - .plugin_id_for_mcp_server_name(server_name) - } - - pub fn is_host_owned_codex_apps_server(&self, server_name: &str) -> bool { - self.host_owned_codex_apps_enabled && server_name == CODEX_APPS_MCP_SERVER_NAME - } - - pub fn set_approval_policy(&self, approval_policy: &Constrained) { - if let Ok(mut policy) = self.elicitation_requests.approval_policy.lock() { - *policy = approval_policy.value(); - } - } - - pub fn set_permission_profile(&self, permission_profile: PermissionProfile) { - if let Ok(mut profile) = self.elicitation_requests.permission_profile.lock() { - *profile = permission_profile; - } - } - - pub fn elicitations_auto_deny(&self) -> bool { - self.elicitation_requests.auto_deny() - } - - pub fn set_elicitations_auto_deny(&self, auto_deny: bool) { - self.elicitation_requests.set_auto_deny(auto_deny); - } - #[allow(clippy::new_ret_no_self, clippy::too_many_arguments)] pub async fn new( mcp_servers: &HashMap, @@ -364,6 +268,90 @@ impl McpConnectionManager { (manager, cancel_token) } + pub fn new_uninitialized_with_permission_profile( + approval_policy: &Constrained, + permission_profile: &PermissionProfile, + prefix_mcp_tool_names: bool, + ) -> Self { + Self { + clients: HashMap::new(), + server_metadata: HashMap::new(), + tool_plugin_provenance: Arc::new(ToolPluginProvenance::default()), + host_owned_codex_apps_enabled: false, + prefix_mcp_tool_names, + elicitation_requests: ElicitationRequestManager::new( + approval_policy.value(), + permission_profile.clone(), + /*reviewer*/ None, + ), + startup_cancellation_token: CancellationToken::new(), + } + } + + pub fn has_servers(&self) -> bool { + !self.clients.is_empty() + } + + /// Drain all MCP clients from this manager and return a future that stops + /// them and terminates their stdio server processes. + pub fn begin_shutdown(&mut self) -> impl std::future::Future + Send + 'static { + self.startup_cancellation_token.cancel(); + let clients = std::mem::take(&mut self.clients); + self.server_metadata.clear(); + async move { + for client in clients.into_values() { + client.shutdown().await; + } + } + } + + /// Stop all MCP clients owned by this manager and terminate stdio server processes. + pub async fn shutdown(&mut self) { + self.begin_shutdown().await; + } + + pub fn server_origin(&self, server_name: &str) -> Option<&str> { + self.server_metadata + .get(server_name) + .and_then(|metadata| metadata.origin.as_ref()) + .map(super::server::McpServerOrigin::as_str) + } + + pub fn server_pollutes_memory(&self, server_name: &str) -> bool { + self.server_metadata + .get(server_name) + .is_none_or(|metadata| metadata.pollutes_memory) + } + + pub fn plugin_id_for_mcp_server_name(&self, server_name: &str) -> Option<&str> { + self.tool_plugin_provenance + .plugin_id_for_mcp_server_name(server_name) + } + + pub fn is_host_owned_codex_apps_server(&self, server_name: &str) -> bool { + self.host_owned_codex_apps_enabled && server_name == CODEX_APPS_MCP_SERVER_NAME + } + + pub fn set_approval_policy(&self, approval_policy: &Constrained) { + if let Ok(mut policy) = self.elicitation_requests.approval_policy.lock() { + *policy = approval_policy.value(); + } + } + + pub fn set_permission_profile(&self, permission_profile: PermissionProfile) { + if let Ok(mut profile) = self.elicitation_requests.permission_profile.lock() { + *profile = permission_profile; + } + } + + pub fn elicitations_auto_deny(&self) -> bool { + self.elicitation_requests.auto_deny() + } + + pub fn set_elicitations_auto_deny(&self, auto_deny: bool) { + self.elicitation_requests.set_auto_deny(auto_deny); + } + pub async fn resolve_elicitation( &self, server_name: String, @@ -452,31 +440,6 @@ impl McpConnectionManager { normalize_tools_for_model_with_prefix(tools, self.prefix_mcp_tool_names) } - /// Returns presentation metadata without waiting for uncached clients still initializing. - /// Cached values will be used if available and the server is still starting up. - pub async fn list_available_server_infos(&self) -> HashMap { - let mut server_infos = HashMap::new(); - for (server_name, client) in &self.clients { - if !client.startup_complete.load(Ordering::Acquire) { - if let Some(server_info) = client.cached_server_info.clone() { - server_infos.insert(server_name.clone(), server_info); - } - continue; - } - match client.client().await { - Ok(managed_client) => { - server_infos.insert(server_name.clone(), managed_client.server_info); - } - Err(_) => { - if let Some(server_info) = client.cached_server_info.clone() { - server_infos.insert(server_name.clone(), server_info); - } - } - } - } - server_infos - } - /// Force-refresh codex apps tools by bypassing the in-process cache. /// /// On success, the refreshed tools replace the cache contents and the @@ -532,21 +495,6 @@ impl McpConnectionManager { )) } - fn with_server_metadata(&self, mut tool: ToolInfo) -> ToolInfo { - let Some(metadata) = self.server_metadata.get(&tool.server_name) else { - tool.supports_parallel_tool_calls = false; - tool.server_origin = None; - return tool; - }; - - tool.supports_parallel_tool_calls = metadata.supports_parallel_tool_calls; - tool.server_origin = metadata - .origin - .as_ref() - .map(|origin| origin.as_str().to_string()); - tool - } - /// Returns a single map that contains all resources. Each key is the /// server name and the value is a vector of resources. pub async fn list_all_resources(&self) -> HashMap> { @@ -778,6 +726,46 @@ impl McpConnectionManager { .with_context(|| format!("resources/read failed for `{server}` ({uri})")) } + /// Returns presentation metadata without waiting for uncached clients still initializing. + /// Cached values will be used if available and the server is still starting up. + pub(crate) async fn list_available_server_infos(&self) -> HashMap { + let mut server_infos = HashMap::new(); + for (server_name, client) in &self.clients { + if !client.startup_complete.load(Ordering::Acquire) { + if let Some(server_info) = client.cached_server_info.clone() { + server_infos.insert(server_name.clone(), server_info); + } + continue; + } + match client.client().await { + Ok(managed_client) => { + server_infos.insert(server_name.clone(), managed_client.server_info); + } + Err(_) => { + if let Some(server_info) = client.cached_server_info.clone() { + server_infos.insert(server_name.clone(), server_info); + } + } + } + } + server_infos + } + + fn with_server_metadata(&self, mut tool: ToolInfo) -> ToolInfo { + let Some(metadata) = self.server_metadata.get(&tool.server_name) else { + tool.supports_parallel_tool_calls = false; + tool.server_origin = None; + return tool; + }; + + tool.supports_parallel_tool_calls = metadata.supports_parallel_tool_calls; + tool.server_origin = metadata + .origin + .as_ref() + .map(|origin| origin.as_str().to_string()); + tool + } + async fn client_by_name(&self, name: &str) -> Result { self.clients .get(name) @@ -786,6 +774,19 @@ impl McpConnectionManager { .await .context("failed to get client") } + + #[cfg(test)] + fn new_uninitialized( + approval_policy: &Constrained, + permission_profile: &Constrained, + prefix_mcp_tool_names: bool, + ) -> Self { + Self::new_uninitialized_with_permission_profile( + approval_policy, + permission_profile.get(), + prefix_mcp_tool_names, + ) + } } impl Drop for McpConnectionManager { From fb8f1ea0d55bb0a6202961518440b40db37b3802 Mon Sep 17 00:00:00 2001 From: canvrno-oai Date: Tue, 9 Jun 2026 16:34:38 -0700 Subject: [PATCH 19/47] TUI Plugin Sharing 1 - add remote plugin identity (#26701) --- codex-rs/tui/src/app.rs | 1 + codex-rs/tui/src/app/background_requests.rs | 35 +- codex-rs/tui/src/app/event_dispatch.rs | 14 +- codex-rs/tui/src/app_event.rs | 19 +- codex-rs/tui/src/chatwidget/plugins.rs | 205 ++++++++--- codex-rs/tui/src/chatwidget/tests/helpers.rs | 28 ++ .../chatwidget/tests/popups_and_settings.rs | 345 ++++++++++++++++++ 7 files changed, 573 insertions(+), 74 deletions(-) diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index e5a3d07329..b239ff4895 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -11,6 +11,7 @@ use crate::app_event::ExitMode; use crate::app_event::FeedbackCategory; use crate::app_event::HistoryLookupResponse; use crate::app_event::PermissionProfileSelection; +use crate::app_event::PluginLocation; use crate::app_event::RateLimitRefreshOrigin; use crate::app_event::RealtimeAudioDeviceKind; #[cfg(target_os = "windows")] diff --git a/codex-rs/tui/src/app/background_requests.rs b/codex-rs/tui/src/app/background_requests.rs index 456676f796..a69d987447 100644 --- a/codex-rs/tui/src/app/background_requests.rs +++ b/codex-rs/tui/src/app/background_requests.rs @@ -241,7 +241,7 @@ impl App { &mut self, app_server: &AppServerSession, cwd: PathBuf, - marketplace_path: AbsolutePathBuf, + location: PluginLocation, plugin_name: String, plugin_display_name: String, ) { @@ -249,14 +249,14 @@ impl App { let app_event_tx = self.app_event_tx.clone(); tokio::spawn(async move { let cwd_for_event = cwd.clone(); - let marketplace_path_for_event = marketplace_path.clone(); + let location_for_event = location.clone(); let plugin_name_for_event = plugin_name.clone(); - let result = fetch_plugin_install(request_handle, marketplace_path, plugin_name) + let result = fetch_plugin_install(request_handle, location, plugin_name) .await .map_err(|err| format!("Failed to install plugin: {err}")); app_event_tx.send(AppEvent::PluginInstallLoaded { cwd: cwd_for_event, - marketplace_path: marketplace_path_for_event, + location: location_for_event, plugin_name: plugin_name_for_event, plugin_display_name, result, @@ -835,16 +835,17 @@ pub(super) async fn fetch_marketplace_upgrade( } pub(super) async fn fetch_plugin_install( request_handle: AppServerRequestHandle, - marketplace_path: AbsolutePathBuf, + location: PluginLocation, plugin_name: String, ) -> Result { let request_id = RequestId::String(format!("plugin-install-{}", Uuid::new_v4())); + let (marketplace_path, remote_marketplace_name) = location.into_request_params(); request_handle .request_typed(ClientRequest::PluginInstall { request_id, params: PluginInstallParams { - marketplace_path: Some(marketplace_path), - remote_marketplace_name: None, + marketplace_path, + remote_marketplace_name, plugin_name, }, }) @@ -1062,6 +1063,26 @@ mod tests { ); } + #[test] + fn plugin_location_request_params_select_exactly_one_location() { + let local_path = test_absolute_path("/marketplaces/local"); + + assert_eq!( + PluginLocation::Local { + marketplace_path: local_path.clone() + } + .into_request_params(), + (Some(local_path), None) + ); + assert_eq!( + PluginLocation::Remote { + marketplace_name: "workspace-directory".to_string() + } + .into_request_params(), + (None, Some("workspace-directory".to_string())) + ); + } + #[test] fn mcp_inventory_maps_prefix_tool_names_by_server() { let statuses = vec![ diff --git a/codex-rs/tui/src/app/event_dispatch.rs b/codex-rs/tui/src/app/event_dispatch.rs index 548aa54427..2d97a513c0 100644 --- a/codex-rs/tui/src/app/event_dispatch.rs +++ b/codex-rs/tui/src/app/event_dispatch.rs @@ -573,14 +573,14 @@ impl App { } AppEvent::FetchPluginInstall { cwd, - marketplace_path, + location, plugin_name, plugin_display_name, } => { self.fetch_plugin_install( app_server, cwd, - marketplace_path, + location, plugin_name, plugin_display_name, ); @@ -601,7 +601,7 @@ impl App { } AppEvent::PluginInstallLoaded { cwd, - marketplace_path, + location, plugin_name, plugin_display_name, result, @@ -612,7 +612,7 @@ impl App { } let should_refresh_plugin_detail = self.chat_widget.on_plugin_install_loaded( cwd.clone(), - marketplace_path.clone(), + location.clone(), plugin_name.clone(), plugin_display_name, result, @@ -621,12 +621,14 @@ impl App { { self.fetch_plugins_list(app_server, cwd.clone()); if should_refresh_plugin_detail { + let (marketplace_path, remote_marketplace_name) = + location.into_request_params(); self.fetch_plugin_detail( app_server, cwd, PluginReadParams { - marketplace_path: Some(marketplace_path), - remote_marketplace_name: None, + marketplace_path, + remote_marketplace_name, plugin_name, }, ); diff --git a/codex-rs/tui/src/app_event.rs b/codex-rs/tui/src/app_event.rs index c2f9ee3f31..41fa6bdce4 100644 --- a/codex-rs/tui/src/app_event.rs +++ b/codex-rs/tui/src/app_event.rs @@ -109,6 +109,21 @@ pub(crate) struct ConnectorsSnapshot { pub(crate) connectors: Vec, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum PluginLocation { + Local { marketplace_path: AbsolutePathBuf }, + Remote { marketplace_name: String }, +} + +impl PluginLocation { + pub(crate) fn into_request_params(self) -> (Option, Option) { + match self { + PluginLocation::Local { marketplace_path } => (Some(marketplace_path), None), + PluginLocation::Remote { marketplace_name } => (None, Some(marketplace_name)), + } + } +} + /// Distinguishes why a rate-limit refresh was requested so the completion /// handler can route the result correctly. /// @@ -493,7 +508,7 @@ pub(crate) enum AppEvent { /// Install a specific plugin from a marketplace. FetchPluginInstall { cwd: PathBuf, - marketplace_path: AbsolutePathBuf, + location: PluginLocation, plugin_name: String, plugin_display_name: String, }, @@ -501,7 +516,7 @@ pub(crate) enum AppEvent { /// Result of installing a plugin. PluginInstallLoaded { cwd: PathBuf, - marketplace_path: AbsolutePathBuf, + location: PluginLocation, plugin_name: String, plugin_display_name: String, result: Result, diff --git a/codex-rs/tui/src/chatwidget/plugins.rs b/codex-rs/tui/src/chatwidget/plugins.rs index 746f42fce9..9bc62fe261 100644 --- a/codex-rs/tui/src/chatwidget/plugins.rs +++ b/codex-rs/tui/src/chatwidget/plugins.rs @@ -5,6 +5,7 @@ use std::time::Instant; use super::ChatWidget; use crate::app_event::AppEvent; +use crate::app_event::PluginLocation; use crate::bottom_pane::ColumnWidthMode; use crate::bottom_pane::SelectionAction; use crate::bottom_pane::SelectionItem; @@ -25,17 +26,18 @@ use crate::tui::FrameRequester; use codex_app_server_protocol::MarketplaceAddResponse; use codex_app_server_protocol::MarketplaceRemoveResponse; use codex_app_server_protocol::MarketplaceUpgradeResponse; +use codex_app_server_protocol::PluginAvailability; use codex_app_server_protocol::PluginDetail; use codex_app_server_protocol::PluginInstallPolicy; use codex_app_server_protocol::PluginInstallResponse; use codex_app_server_protocol::PluginListResponse; use codex_app_server_protocol::PluginMarketplaceEntry; use codex_app_server_protocol::PluginReadResponse; +use codex_app_server_protocol::PluginSource; use codex_app_server_protocol::PluginSummary; use codex_app_server_protocol::PluginUninstallResponse; use codex_core_plugins::OPENAI_CURATED_MARKETPLACE_NAME; use codex_features::Feature; -use codex_utils_absolute_path::AbsolutePathBuf; use crossterm::event::KeyCode; use crossterm::event::KeyEvent; use crossterm::event::KeyEventKind; @@ -453,7 +455,7 @@ impl ChatWidget { pub(crate) fn on_plugin_install_loaded( &mut self, cwd: PathBuf, - _marketplace_path: AbsolutePathBuf, + _location: PluginLocation, _plugin_name: String, plugin_display_name: String, result: Result, @@ -1625,19 +1627,22 @@ impl ChatWidget { ) -> SelectionViewParams { let marketplace_label = plugin.marketplace_name.clone(); let display_name = plugin_display_name(&plugin.summary); - let detail_status_label = if plugin.summary.installed { - if plugin.summary.enabled { - "Installed" + let detail_status_label = + if plugin.summary.availability == PluginAvailability::DisabledByAdmin { + "Disabled by admin" + } else if plugin.summary.installed { + if plugin.summary.enabled { + "Installed" + } else { + "Disabled" + } } else { - "Disabled" - } - } else { - match plugin.summary.install_policy { - PluginInstallPolicy::NotAvailable => "Not installable", - PluginInstallPolicy::Available => "Can be installed", - PluginInstallPolicy::InstalledByDefault => "Available by default", - } - }; + match plugin.summary.install_policy { + PluginInstallPolicy::NotAvailable => "Not installable", + PluginInstallPolicy::Available => "Can be installed", + PluginInstallPolicy::InstalledByDefault => "Available by default", + } + }; let mut header = ColumnRenderable::new(); header.push(Line::from("Plugins".bold())); header.push(Line::from( @@ -1676,23 +1681,40 @@ impl ChatWidget { }]; if plugin.summary.installed { - let uninstall_cwd = self.config.cwd.to_path_buf(); - let plugin_id = plugin.summary.id.clone(); - let plugin_display_name = display_name; + if let Some(plugin_id) = plugin_uninstall_id(&plugin.summary) { + let uninstall_cwd = self.config.cwd.to_path_buf(); + let plugin_display_name = display_name; + items.push(SelectionItem { + name: "Uninstall plugin".to_string(), + description: Some("Remove this plugin now.".to_string()), + selected_description: Some("Remove this plugin now.".to_string()), + actions: vec![Box::new(move |tx| { + tx.send(AppEvent::OpenPluginUninstallLoading { + plugin_display_name: plugin_display_name.clone(), + }); + tx.send(AppEvent::FetchPluginUninstall { + cwd: uninstall_cwd.clone(), + plugin_id: plugin_id.clone(), + plugin_display_name: plugin_display_name.clone(), + }); + })], + ..Default::default() + }); + } else { + items.push(SelectionItem { + name: "Uninstall plugin".to_string(), + description: Some( + "This remote plugin did not provide an uninstall identity.".to_string(), + ), + is_disabled: true, + ..Default::default() + }); + } + } else if plugin.summary.availability == PluginAvailability::DisabledByAdmin { items.push(SelectionItem { - name: "Uninstall plugin".to_string(), - description: Some("Remove this plugin now.".to_string()), - selected_description: Some("Remove this plugin now.".to_string()), - actions: vec![Box::new(move |tx| { - tx.send(AppEvent::OpenPluginUninstallLoading { - plugin_display_name: plugin_display_name.clone(), - }); - tx.send(AppEvent::FetchPluginUninstall { - cwd: uninstall_cwd.clone(), - plugin_id: plugin_id.clone(), - plugin_display_name: plugin_display_name.clone(), - }); - })], + name: "Install plugin".to_string(), + description: Some("This plugin is disabled by your workspace admin.".to_string()), + is_disabled: true, ..Default::default() }); } else if plugin.summary.install_policy == PluginInstallPolicy::NotAvailable { @@ -1704,9 +1726,9 @@ impl ChatWidget { is_disabled: true, ..Default::default() }); - } else if let Some(marketplace_path) = plugin.marketplace_path.clone() { + } else if let Some(location) = plugin_detail_location(plugin) { let install_cwd = self.config.cwd.to_path_buf(); - let plugin_name = plugin.summary.name.clone(); + let plugin_name = plugin_request_name(&plugin.summary); let plugin_display_name = display_name; items.push(SelectionItem { name: "Install plugin".to_string(), @@ -1718,7 +1740,7 @@ impl ChatWidget { }); tx.send(AppEvent::FetchPluginInstall { cwd: install_cwd.clone(), - marketplace_path: marketplace_path.clone(), + location: location.clone(), plugin_name: plugin_name.clone(), plugin_display_name: plugin_display_name.clone(), }); @@ -1728,7 +1750,7 @@ impl ChatWidget { } else { items.push(SelectionItem { name: "Install plugin".to_string(), - description: Some("Installing remote plugins is not supported yet.".to_string()), + description: Some("This plugin did not provide an install location.".to_string()), is_disabled: true, ..Default::default() }); @@ -1792,9 +1814,12 @@ impl ChatWidget { } else { plugin_brief_description_without_marketplace(plugin, status_label_width) }; - let can_view_details = marketplace.path.is_some(); + let plugin_detail_request = plugin_detail_request_for_entry(marketplace, plugin); + let can_view_details = plugin_detail_request.is_some(); + let disabled_by_admin = plugin.availability == PluginAvailability::DisabledByAdmin; + let can_toggle_plugin = plugin.installed && !disabled_by_admin; let selected_status_label = format!("{status_label: = if let Some(marketplace_path) = marketplace_path { - vec![Box::new(move |tx| { - tx.send(AppEvent::OpenPluginDetailLoading { - plugin_display_name: plugin_display_name.clone(), - }); - tx.send(AppEvent::FetchPluginDetail { - cwd: cwd.clone(), - params: codex_app_server_protocol::PluginReadParams { - marketplace_path: Some(marketplace_path.clone()), - remote_marketplace_name: None, - plugin_name: plugin_name.clone(), - }, - }); - })] - } else { - Vec::new() - }; + let actions: Vec = + if let Some((location, plugin_name)) = plugin_detail_request { + vec![Box::new(move |tx| { + tx.send(AppEvent::OpenPluginDetailLoading { + plugin_display_name: plugin_display_name.clone(), + }); + let (marketplace_path, remote_marketplace_name) = + location.clone().into_request_params(); + tx.send(AppEvent::FetchPluginDetail { + cwd: cwd.clone(), + params: codex_app_server_protocol::PluginReadParams { + marketplace_path, + remote_marketplace_name, + plugin_name: plugin_name.clone(), + }, + }); + })] + } else { + Vec::new() + }; let is_disabled = !can_view_details && !plugin.installed; - let disabled_reason = - is_disabled.then(|| "remote plugin details are not available yet".to_string()); + let disabled_reason = is_disabled.then(|| "plugin details are unavailable".to_string()); items.push(SelectionItem { name: display_name, toggle, - toggle_placeholder: (!plugin.installed).then_some("[-] "), + toggle_placeholder: (!can_toggle_plugin).then_some("[-] "), description: Some(description), selected_description: Some(selected_description), search_value: Some(search_value), @@ -2082,6 +2113,9 @@ fn plugin_brief_description_without_marketplace( } fn plugin_status_label(plugin: &PluginSummary) -> &'static str { + if plugin.availability == PluginAvailability::DisabledByAdmin { + return "Disabled by admin"; + } if plugin.installed { if plugin.enabled { "Installed" @@ -2097,6 +2131,59 @@ fn plugin_status_label(plugin: &PluginSummary) -> &'static str { } } +fn plugin_location_for_marketplace( + marketplace: &PluginMarketplaceEntry, + plugin: &PluginSummary, +) -> Option { + if let Some(marketplace_path) = marketplace.path.clone() { + return Some(PluginLocation::Local { marketplace_path }); + } + plugin_remote_identity(plugin).map(|_| PluginLocation::Remote { + marketplace_name: marketplace.name.clone(), + }) +} + +fn plugin_detail_location(plugin: &PluginDetail) -> Option { + if let Some(marketplace_path) = plugin.marketplace_path.clone() { + return Some(PluginLocation::Local { marketplace_path }); + } + plugin_remote_identity(&plugin.summary).map(|_| PluginLocation::Remote { + marketplace_name: plugin.marketplace_name.clone(), + }) +} + +fn plugin_detail_request_for_entry( + marketplace: &PluginMarketplaceEntry, + plugin: &PluginSummary, +) -> Option<(PluginLocation, String)> { + plugin_location_for_marketplace(marketplace, plugin) + .map(|location| (location, plugin_request_name(plugin))) +} + +fn plugin_request_name(plugin: &PluginSummary) -> String { + if matches!(&plugin.source, PluginSource::Remote) + && let Some(remote_plugin_id) = plugin_remote_identity(plugin) + { + return remote_plugin_id; + } + plugin.name.clone() +} + +fn plugin_remote_identity(plugin: &PluginSummary) -> Option { + plugin + .share_context + .as_ref() + .map(|context| context.remote_plugin_id.clone()) + .or_else(|| plugin.remote_plugin_id.clone()) +} + +fn plugin_uninstall_id(plugin: &PluginSummary) -> Option { + if matches!(&plugin.source, PluginSource::Remote) { + return plugin_remote_identity(plugin); + } + Some(plugin.id.clone()) +} + fn plugin_description(plugin: &PluginSummary) -> Option { plugin .interface diff --git a/codex-rs/tui/src/chatwidget/tests/helpers.rs b/codex-rs/tui/src/chatwidget/tests/helpers.rs index b058b47cca..214ca969aa 100644 --- a/codex-rs/tui/src/chatwidget/tests/helpers.rs +++ b/codex-rs/tui/src/chatwidget/tests/helpers.rs @@ -1337,6 +1337,34 @@ pub(super) fn plugins_test_summary( } } +pub(super) fn plugins_test_remote_summary( + remote_plugin_id: &str, + name: &str, + display_name: Option<&str>, + description: Option<&str>, + installed: bool, +) -> PluginSummary { + PluginSummary { + id: remote_plugin_id.to_string(), + remote_plugin_id: Some(remote_plugin_id.to_string()), + local_version: None, + name: name.to_string(), + share_context: None, + source: PluginSource::Remote, + installed, + enabled: true, + install_policy: PluginInstallPolicy::Available, + auth_policy: PluginAuthPolicy::OnInstall, + availability: PluginAvailability::Available, + interface: Some(plugins_test_interface( + display_name, + description, + /*long_description*/ None, + )), + keywords: Vec::new(), + } +} + pub(super) fn plugins_test_curated_marketplace( plugins: Vec, ) -> PluginMarketplaceEntry { diff --git a/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs b/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs index 89c332a4ef..ae44cf00cd 100644 --- a/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs +++ b/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs @@ -6,6 +6,7 @@ use codex_app_server_protocol::HookErrorInfo; use codex_app_server_protocol::HooksListEntry; use codex_app_server_protocol::HooksListResponse; use codex_app_server_protocol::MarketplaceRemoveResponse; +use codex_app_server_protocol::PluginAvailability; use codex_features::Stage; use pretty_assertions::assert_eq; @@ -714,6 +715,350 @@ async fn plugin_detail_popup_hides_disclosure_for_installed_plugins() { ); } +#[tokio::test] +async fn plugins_popup_remote_row_opens_remote_detail() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + chat.set_feature_enabled(Feature::Plugins, /*enabled*/ true); + + let popup = render_loaded_plugins_popup( + &mut chat, + plugins_test_response(vec![PluginMarketplaceEntry { + name: "workspace-directory".to_string(), + path: None, + interface: Some(MarketplaceInterface { + display_name: Some("Workspace".to_string()), + }), + plugins: vec![plugins_test_remote_summary( + "plugins~Plugin_calendar", + "calendar", + Some("Calendar"), + Some("Workspace schedules."), + /*installed*/ false, + )], + }]), + ); + let remote_row = popup + .lines() + .find(|line| line.contains("Calendar")) + .expect("expected remote plugin row"); + assert!( + remote_row.contains("Available") + && remote_row.contains("Press Enter to install or view plugin details."), + "expected remote plugin row to be viewable, got:\n{remote_row}" + ); + + while rx.try_recv().is_ok() {} + chat.handle_key_event(KeyEvent::from(KeyCode::Enter)); + + match rx.try_recv() { + Ok(AppEvent::OpenPluginDetailLoading { + plugin_display_name, + }) => { + assert_eq!(plugin_display_name, "Calendar"); + } + other => panic!("expected OpenPluginDetailLoading event, got {other:?}"), + } + match rx.try_recv() { + Ok(AppEvent::FetchPluginDetail { cwd: _, params }) => { + assert_eq!(params.marketplace_path, None); + assert_eq!( + params.remote_marketplace_name, + Some("workspace-directory".to_string()) + ); + assert_eq!(params.plugin_name, "plugins~Plugin_calendar"); + } + other => panic!("expected FetchPluginDetail event, got {other:?}"), + } +} + +#[tokio::test] +async fn plugin_detail_remote_install_uses_remote_location() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + chat.set_feature_enabled(Feature::Plugins, /*enabled*/ true); + + let summary = plugins_test_remote_summary( + "plugins~Plugin_linear", + "linear", + Some("Linear"), + Some("Issue tracking."), + /*installed*/ false, + ); + let cwd = chat.config.cwd.clone(); + chat.on_plugins_loaded( + cwd.to_path_buf(), + Ok(plugins_test_response(vec![PluginMarketplaceEntry { + name: "workspace-shared-with-me-private".to_string(), + path: None, + interface: Some(MarketplaceInterface { + display_name: Some("Shared with me".to_string()), + }), + plugins: vec![summary.clone()], + }])), + ); + chat.add_plugins_output(); + chat.on_plugin_detail_loaded( + cwd.to_path_buf(), + Ok(PluginReadResponse { + plugin: PluginDetail { + marketplace_name: "workspace-shared-with-me-private".to_string(), + marketplace_path: None, + summary, + description: Some("Install shared Linear plugin.".to_string()), + skills: Vec::new(), + hooks: Vec::new(), + apps: Vec::new(), + app_templates: Vec::new(), + mcp_servers: Vec::new(), + }, + }), + ); + let popup = render_bottom_popup(&chat, /*width*/ 100); + assert!( + popup.contains("Install plugin") && popup.contains("Install this plugin now."), + "expected remote detail to offer install, got:\n{popup}" + ); + + while rx.try_recv().is_ok() {} + chat.handle_key_event(KeyEvent::from(KeyCode::Down)); + chat.handle_key_event(KeyEvent::from(KeyCode::Enter)); + + match rx.try_recv() { + Ok(AppEvent::OpenPluginInstallLoading { + plugin_display_name, + }) => { + assert_eq!(plugin_display_name, "Linear"); + } + other => panic!("expected OpenPluginInstallLoading event, got {other:?}"), + } + match rx.try_recv() { + Ok(AppEvent::FetchPluginInstall { + cwd: _, + location: crate::app_event::PluginLocation::Remote { marketplace_name }, + plugin_name, + plugin_display_name, + }) => { + assert_eq!(marketplace_name, "workspace-shared-with-me-private"); + assert_eq!(plugin_name, "plugins~Plugin_linear"); + assert_eq!(plugin_display_name, "Linear"); + } + other => panic!("expected remote FetchPluginInstall event, got {other:?}"), + } +} + +#[tokio::test] +async fn plugin_detail_remote_uninstall_uses_remote_plugin_id() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + chat.set_feature_enabled(Feature::Plugins, /*enabled*/ true); + + let summary = plugins_test_remote_summary( + "plugins~Plugin_linear", + "linear", + Some("Linear"), + Some("Issue tracking."), + /*installed*/ true, + ); + let cwd = chat.config.cwd.clone(); + chat.on_plugins_loaded( + cwd.to_path_buf(), + Ok(plugins_test_response(vec![PluginMarketplaceEntry { + name: "workspace-shared-with-me-private".to_string(), + path: None, + interface: Some(MarketplaceInterface { + display_name: Some("Shared with me".to_string()), + }), + plugins: vec![summary.clone()], + }])), + ); + chat.add_plugins_output(); + chat.on_plugin_detail_loaded( + cwd.to_path_buf(), + Ok(PluginReadResponse { + plugin: PluginDetail { + marketplace_name: "workspace-shared-with-me-private".to_string(), + marketplace_path: None, + summary, + description: Some("Installed shared Linear plugin.".to_string()), + skills: Vec::new(), + hooks: Vec::new(), + apps: Vec::new(), + app_templates: Vec::new(), + mcp_servers: Vec::new(), + }, + }), + ); + + while rx.try_recv().is_ok() {} + chat.handle_key_event(KeyEvent::from(KeyCode::Down)); + chat.handle_key_event(KeyEvent::from(KeyCode::Enter)); + + match rx.try_recv() { + Ok(AppEvent::OpenPluginUninstallLoading { + plugin_display_name, + }) => { + assert_eq!(plugin_display_name, "Linear"); + } + other => panic!("expected OpenPluginUninstallLoading event, got {other:?}"), + } + match rx.try_recv() { + Ok(AppEvent::FetchPluginUninstall { + plugin_id, + plugin_display_name, + .. + }) => { + assert_eq!(plugin_id, "plugins~Plugin_linear"); + assert_eq!(plugin_display_name, "Linear"); + } + other => panic!("expected remote FetchPluginUninstall event, got {other:?}"), + } +} + +#[tokio::test] +async fn plugin_detail_remote_without_remote_id_disables_uninstall_action() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + chat.set_feature_enabled(Feature::Plugins, /*enabled*/ true); + + let summary = PluginSummary { + source: PluginSource::Remote, + ..plugins_test_summary( + "linear@workspace-shared-with-me-private", + "linear", + Some("Linear"), + Some("Issue tracking."), + /*installed*/ true, + /*enabled*/ true, + PluginInstallPolicy::Available, + ) + }; + let cwd = chat.config.cwd.clone(); + chat.on_plugins_loaded( + cwd.to_path_buf(), + Ok(plugins_test_response(vec![PluginMarketplaceEntry { + name: "workspace-shared-with-me-private".to_string(), + path: None, + interface: Some(MarketplaceInterface { + display_name: Some("Shared with me".to_string()), + }), + plugins: vec![summary.clone()], + }])), + ); + chat.add_plugins_output(); + chat.on_plugin_detail_loaded( + cwd.to_path_buf(), + Ok(PluginReadResponse { + plugin: PluginDetail { + marketplace_name: "workspace-shared-with-me-private".to_string(), + marketplace_path: None, + summary, + description: Some("Installed shared Linear plugin.".to_string()), + skills: Vec::new(), + hooks: Vec::new(), + apps: Vec::new(), + app_templates: Vec::new(), + mcp_servers: Vec::new(), + }, + }), + ); + + let popup = render_bottom_popup(&chat, /*width*/ 120); + assert!( + popup.contains("This remote plugin did not provide an uninstall identity.") + && !popup.contains("Remove this plugin now."), + "expected missing remote ID to disable uninstall, got:\n{popup}" + ); + + while rx.try_recv().is_ok() {} + assert!( + rx.try_recv().is_err(), + "expected no action after rendering disabled uninstall state" + ); +} + +#[tokio::test] +async fn plugin_detail_admin_disabled_plugin_blocks_install() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + chat.set_feature_enabled(Feature::Plugins, /*enabled*/ true); + + let summary = PluginSummary { + availability: PluginAvailability::DisabledByAdmin, + ..plugins_test_summary( + "plugin-admin-blocked", + "admin-blocked", + Some("Admin Blocked"), + Some("Blocked by policy."), + /*installed*/ false, + /*enabled*/ true, + PluginInstallPolicy::Available, + ) + }; + let response = plugins_test_response(vec![plugins_test_curated_marketplace(vec![ + summary.clone(), + ])]); + let cwd = chat.config.cwd.clone(); + chat.on_plugins_loaded(cwd.to_path_buf(), Ok(response)); + chat.add_plugins_output(); + chat.on_plugin_detail_loaded( + cwd.to_path_buf(), + Ok(PluginReadResponse { + plugin: plugins_test_detail(summary, Some("Blocked by policy."), &[], &[], &[], &[]), + }), + ); + + let popup = render_bottom_popup(&chat, /*width*/ 100); + assert!( + popup.contains("Admin Blocked · Disabled by admin") + && popup.contains("This plugin is disabled by your workspace admin.") + && !popup.contains("Install this plugin now."), + "expected admin-disabled detail to block install, got:\n{popup}" + ); + + while rx.try_recv().is_ok() {} + assert!( + rx.try_recv().is_err(), + "expected no action after rendering disabled install state" + ); +} + +#[tokio::test] +async fn plugins_popup_admin_disabled_installed_plugin_has_no_toggle_hint() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + chat.set_feature_enabled(Feature::Plugins, /*enabled*/ true); + + let summary = PluginSummary { + availability: PluginAvailability::DisabledByAdmin, + ..plugins_test_summary( + "plugin-admin-blocked", + "admin-blocked", + Some("Admin Blocked"), + Some("Blocked by policy."), + /*installed*/ true, + /*enabled*/ true, + PluginInstallPolicy::Available, + ) + }; + render_loaded_plugins_popup( + &mut chat, + plugins_test_response(vec![plugins_test_curated_marketplace(vec![summary])]), + ); + + let popup = render_bottom_popup(&chat, /*width*/ 100); + assert!( + popup.contains("Disabled by admin") + && popup.contains("Press Enter to view plugin details.") + && !popup.contains("Space to disable"), + "expected admin-disabled installed plugin to omit toggle hint, got:\n{popup}" + ); + + while rx.try_recv().is_ok() {} + let before = render_bottom_popup(&chat, /*width*/ 100); + chat.handle_key_event(KeyEvent::new(KeyCode::Char(' '), KeyModifiers::NONE)); + let after = render_bottom_popup(&chat, /*width*/ 100); + assert!( + rx.try_recv().is_err(), + "space should not toggle admin-disabled installed plugins" + ); + assert_eq!(after, before); +} + #[tokio::test] async fn plugin_detail_error_popup_skips_disabled_row_numbering() { let (mut chat, _rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; From 51fc4b0559c08d12caec6f4d7d2b6a35e84c7339 Mon Sep 17 00:00:00 2001 From: Celia Chen Date: Tue, 9 Jun 2026 16:49:09 -0700 Subject: [PATCH 20/47] feat: use provider defaults for memory models (#27129) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why Memory startup used hardcoded OpenAI model slugs for extraction and consolidation. That works for the default OpenAI-compatible path, but provider-specific backends can require different model identifiers. In particular, Amazon Bedrock should use its Bedrock model ID for these background memory requests instead of the OpenAI `gpt-5.4-mini` / `gpt-5.4` slugs. ## What Changed - Added provider-owned preferred memory model methods alongside `approval_review_preferred_model`. - Updated memory extraction and consolidation to resolve their default model through the active `ModelProvider`. - Added Amazon Bedrock overrides so both memory stages use `openai.gpt-5.4` through Bedrock’s provider-specific model ID. - Kept explicit `memories.extract_model` and `memories.consolidation_model` config overrides taking precedence. - Added startup coverage for default OpenAI and Bedrock memory model selection. #closes #26288 --- codex-rs/Cargo.lock | 2 + codex-rs/memories/write/Cargo.toml | 2 + codex-rs/memories/write/src/lib.rs | 2 - codex-rs/memories/write/src/phase1.rs | 11 +- codex-rs/memories/write/src/phase2.rs | 7 +- codex-rs/memories/write/src/runtime.rs | 54 +++ codex-rs/memories/write/src/startup_tests.rs | 318 +++++++++++++++++- .../model-provider/src/amazon_bedrock/mod.rs | 8 + codex-rs/model-provider/src/provider.rs | 22 ++ 9 files changed, 413 insertions(+), 13 deletions(-) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index cbc237a84c..511fca7250 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -3342,6 +3342,8 @@ dependencies = [ "codex-features", "codex-git-utils", "codex-login", + "codex-model-provider", + "codex-model-provider-info", "codex-models-manager", "codex-otel", "codex-protocol", diff --git a/codex-rs/memories/write/Cargo.toml b/codex-rs/memories/write/Cargo.toml index ebe6b0f2a8..bf031a6a16 100644 --- a/codex-rs/memories/write/Cargo.toml +++ b/codex-rs/memories/write/Cargo.toml @@ -21,6 +21,7 @@ codex-config = { workspace = true } codex-features = { workspace = true } codex-git-utils = { workspace = true } codex-login = { workspace = true } +codex-model-provider = { workspace = true } codex-otel = { workspace = true } codex-protocol = { workspace = true } codex-rollout = { workspace = true } @@ -39,6 +40,7 @@ tracing = { workspace = true, features = ["log"] } uuid = { workspace = true, features = ["v4", "v5"] } [dev-dependencies] +codex-model-provider-info = { workspace = true } codex-models-manager = { workspace = true } core_test_support = { workspace = true } pretty_assertions = { workspace = true } diff --git a/codex-rs/memories/write/src/lib.rs b/codex-rs/memories/write/src/lib.rs index 6764ebf5c6..ff86068ba6 100644 --- a/codex-rs/memories/write/src/lib.rs +++ b/codex-rs/memories/write/src/lib.rs @@ -76,7 +76,6 @@ signal to remove stale memories derived only from those resources. } mod stage_one { - pub(super) const MODEL: &str = "gpt-5.4-mini"; pub(super) const REASONING_EFFORT: codex_protocol::openai_models::ReasoningEffort = codex_protocol::openai_models::ReasoningEffort::Low; pub(super) const CONCURRENCY_LIMIT: usize = 8; @@ -101,7 +100,6 @@ mod stage_one { } mod stage_two { - pub(super) const MODEL: &str = "gpt-5.4"; pub(super) const REASONING_EFFORT: codex_protocol::openai_models::ReasoningEffort = codex_protocol::openai_models::ReasoningEffort::Medium; pub(super) const JOB_LEASE_SECONDS: i64 = 3_600; diff --git a/codex-rs/memories/write/src/phase1.rs b/codex-rs/memories/write/src/phase1.rs index 21b20759b1..693b7a1680 100644 --- a/codex-rs/memories/write/src/phase1.rs +++ b/codex-rs/memories/write/src/phase1.rs @@ -190,11 +190,12 @@ async fn build_request_context( context: &MemoryStartupContext, config: &Config, ) -> StageOneRequestContext { - let model_name = config - .memories - .extract_model - .clone() - .unwrap_or(crate::stage_one::MODEL.to_string()); + let model_name = config.memories.extract_model.clone().unwrap_or_else(|| { + context + .provider() + .memory_extraction_preferred_model() + .to_string() + }); context .stage_one_request_context(config, &model_name, crate::stage_one::REASONING_EFFORT) .await diff --git a/codex-rs/memories/write/src/phase2.rs b/codex-rs/memories/write/src/phase2.rs index c78032d9c2..0c588a6070 100644 --- a/codex-rs/memories/write/src/phase2.rs +++ b/codex-rs/memories/write/src/phase2.rs @@ -16,6 +16,7 @@ use crate::workspace::write_workspace_diff; use codex_config::Constrained; use codex_core::config::Config; use codex_features::Feature; +use codex_model_provider::ModelProvider; use codex_protocol::ThreadId; use codex_protocol::protocol::AgentStatus; use codex_protocol::protocol::AskForApproval; @@ -76,7 +77,7 @@ pub async fn run(context: Arc, config: Arc) { } // 3. Build the locked-down config used by the consolidation agent. - let Some(agent_config) = agent::get_config(config.as_ref()) else { + let Some(agent_config) = agent::get_config(config.as_ref(), context.provider()) else { // If we can't get the config, we can't consolidate. tracing::error!("failed to get agent config"); job::failed( @@ -297,7 +298,7 @@ mod agent { use super::*; use tracing::warn; - pub(super) fn get_config(config: &Config) -> Option { + pub(super) fn get_config(config: &Config, provider: &dyn ModelProvider) -> Option { let root = memory_root(&config.codex_home); let mut agent_config = config.clone(); @@ -339,7 +340,7 @@ mod agent { .memories .consolidation_model .clone() - .unwrap_or(crate::stage_two::MODEL.to_string()), + .unwrap_or_else(|| provider.memory_consolidation_preferred_model().to_string()), ); agent_config.model_reasoning_effort = Some(crate::stage_two::REASONING_EFFORT); diff --git a/codex-rs/memories/write/src/runtime.rs b/codex-rs/memories/write/src/runtime.rs index d524a2e224..d2b1cc36b2 100644 --- a/codex-rs/memories/write/src/runtime.rs +++ b/codex-rs/memories/write/src/runtime.rs @@ -13,6 +13,9 @@ use codex_login::AuthManager; use codex_login::CodexAuth; use codex_login::auth_env_telemetry::collect_auth_env_telemetry; use codex_login::default_client::originator; +use codex_model_provider::ModelProvider; +use codex_model_provider::SharedModelProvider; +use codex_model_provider::create_model_provider; use codex_otel::SessionTelemetry; use codex_otel::TelemetryAuthMode; use codex_protocol::SessionId; @@ -68,6 +71,7 @@ pub(crate) struct MemoryStartupContext { thread: Arc, thread_manager: Arc, auth_manager: Arc, + provider: SharedModelProvider, session_telemetry: SessionTelemetry, } @@ -79,6 +83,51 @@ impl MemoryStartupContext { thread: Arc, config: &Config, source: SessionSource, + ) -> Self { + let provider = create_model_provider( + config.model_provider.clone(), + Some(Arc::clone(&auth_manager)), + ); + Self::new_with_provider( + thread_manager, + auth_manager, + thread_id, + thread, + config, + source, + provider, + ) + } + + #[cfg(test)] + pub(crate) fn new_for_testing( + thread_manager: Arc, + auth_manager: Arc, + thread_id: ThreadId, + thread: Arc, + config: &Config, + source: SessionSource, + provider: SharedModelProvider, + ) -> Self { + Self::new_with_provider( + thread_manager, + auth_manager, + thread_id, + thread, + config, + source, + provider, + ) + } + + fn new_with_provider( + thread_manager: Arc, + auth_manager: Arc, + thread_id: ThreadId, + thread: Arc, + config: &Config, + source: SessionSource, + provider: SharedModelProvider, ) -> Self { let auth = auth_manager.auth_cached(); let auth = auth.as_ref(); @@ -109,6 +158,7 @@ impl MemoryStartupContext { thread, thread_manager, auth_manager, + provider, session_telemetry, } } @@ -121,6 +171,10 @@ impl MemoryStartupContext { self.thread.state_db() } + pub(crate) fn provider(&self) -> &dyn ModelProvider { + self.provider.as_ref() + } + pub(crate) fn counter(&self, name: &str, inc: i64, tags: &[(&str, &str)]) { self.session_telemetry.counter(name, inc, tags); } diff --git a/codex-rs/memories/write/src/startup_tests.rs b/codex-rs/memories/write/src/startup_tests.rs index bd0a9904e6..d5e46e09e0 100644 --- a/codex-rs/memories/write/src/startup_tests.rs +++ b/codex-rs/memories/write/src/startup_tests.rs @@ -1,12 +1,30 @@ +use crate::extensions::seed_extension_instructions; +use crate::memory_root; +use crate::phase1; +use crate::phase2; +use crate::runtime::MemoryStartupContext; use crate::start_memories_startup_task; +use codex_config::types::MemoriesConfig; use codex_features::Feature; use codex_git_utils::diff_since_latest_init; use codex_git_utils::reset_git_repository; +use codex_login::AuthManager; +use codex_login::CodexAuth; +use codex_model_provider::ModelProvider; +use codex_model_provider::ProviderAccountResult; +use codex_model_provider::SharedModelProvider; +use codex_model_provider::create_model_provider; +use codex_model_provider_info::ModelProviderInfo; use codex_protocol::ThreadId; use codex_protocol::config_types::ServiceTier; +use codex_protocol::models::ContentItem; +use codex_protocol::models::ResponseItem; +use codex_protocol::openai_models::ModelsResponse; use codex_protocol::openai_models::ReasoningEffort; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::Op; +use codex_protocol::protocol::RolloutItem; +use codex_protocol::protocol::RolloutLine; use codex_protocol::protocol::SessionSource; use core_test_support::responses::ResponseMock; use core_test_support::responses::ResponsesRequest; @@ -20,7 +38,10 @@ use core_test_support::test_codex::TestCodex; use core_test_support::test_codex::test_codex; use core_test_support::wait_for_event; use pretty_assertions::assert_eq; +use std::future::Future; use std::path::Path; +use std::path::PathBuf; +use std::pin::Pin; use std::sync::Arc; use tempfile::TempDir; use tokio::time::Duration; @@ -329,18 +350,186 @@ async fn memories_startup_phase1_uses_live_thread_service_tier_and_detached_meta Ok(()) } +#[tokio::test] +async fn memories_startup_phase1_provider_default_drives_request_model() -> anyhow::Result<()> { + let server = start_mock_server().await; + let home = Arc::new(TempDir::new()?); + let request = + run_memory_phase_one_model_request_test(&server, home, startup_test_memories_config()) + .await?; + + assert_eq!( + request.body_json()["model"].as_str(), + Some(MOCK_PROVIDER_PHASE_ONE_MODEL) + ); + + Ok(()) +} + +#[tokio::test] +async fn memories_startup_phase2_provider_default_drives_request_model() -> anyhow::Result<()> { + let server = start_mock_server().await; + let home = Arc::new(TempDir::new()?); + let request = + run_memory_phase_two_model_request_test(&server, home, startup_test_memories_config()) + .await?; + + assert_eq!( + request.body_json()["model"].as_str(), + Some(MOCK_PROVIDER_PHASE_TWO_MODEL) + ); + + Ok(()) +} + +#[tokio::test] +async fn memories_startup_phase1_explicit_model_override_drives_request_model() -> anyhow::Result<()> +{ + let server = start_mock_server().await; + let home = Arc::new(TempDir::new()?); + let mut memories = startup_test_memories_config(); + memories.extract_model = Some("override.phase-one".to_string()); + let request = run_memory_phase_one_model_request_test(&server, home, memories).await?; + + assert_eq!( + request.body_json()["model"].as_str(), + Some("override.phase-one") + ); + + Ok(()) +} + +#[tokio::test] +async fn memories_startup_phase2_explicit_model_override_drives_request_model() -> anyhow::Result<()> +{ + let server = start_mock_server().await; + let home = Arc::new(TempDir::new()?); + let mut memories = startup_test_memories_config(); + memories.consolidation_model = Some("override.phase-two".to_string()); + let request = run_memory_phase_two_model_request_test(&server, home, memories).await?; + + assert_eq!( + request.body_json()["model"].as_str(), + Some("override.phase-two") + ); + + Ok(()) +} + +async fn run_memory_phase_one_model_request_test( + server: &wiremock::MockServer, + home: Arc, + memories: MemoriesConfig, +) -> anyhow::Result { + let test = build_test_codex_with_memories_config(server, Arc::clone(&home), memories).await?; + let provider = Arc::new(MockMemoryModelProvider::new( + test.config.model_provider.clone(), + Some(test.thread_manager.auth_manager()), + )); + let db = test + .codex + .state_db() + .ok_or_else(|| anyhow::anyhow!("state db should be enabled for memory startup test"))?; + seed_stage1_candidate( + db.as_ref(), + home.path(), + chrono::Utc::now() - chrono::Duration::hours(2), + "startup-models", + ) + .await?; + let response = mount_sse_once( + server, + sse(vec![ + ev_response_created("resp-phase1"), + ev_assistant_message( + "msg-phase1", + r#"{"raw_memory":"raw memory","rollout_summary":"rollout summary","rollout_slug":"startup-models"}"#, + ), + ev_completed("resp-phase1"), + ]), + ) + .await; + + let (context, config) = memory_startup_context_with_provider(&test, provider).await; + phase1::run(context, config).await; + let request = wait_for_single_request(&response).await; + shutdown_test_codex(&test).await?; + Ok(request) +} + +async fn run_memory_phase_two_model_request_test( + server: &wiremock::MockServer, + home: Arc, + memories: MemoriesConfig, +) -> anyhow::Result { + let test = build_test_codex_with_memories_config(server, home.clone(), memories).await?; + let provider = Arc::new(MockMemoryModelProvider::new( + test.config.model_provider.clone(), + Some(test.thread_manager.auth_manager()), + )); + let db = test + .codex + .state_db() + .ok_or_else(|| anyhow::anyhow!("state db should be enabled for memory startup test"))?; + seed_stage1_output( + db.as_ref(), + home.path(), + chrono::Utc::now(), + "raw memory for phase two", + "rollout summary for phase two", + "startup-models-phase-two", + ) + .await?; + + let response = mount_sse_once( + server, + sse(vec![ + ev_response_created("resp-phase2"), + ev_assistant_message("msg-phase2", "phase2 complete"), + ev_completed("resp-phase2"), + ]), + ) + .await; + + let (context, config) = memory_startup_context_with_provider(&test, provider).await; + let root = memory_root(&config.codex_home); + tokio::fs::create_dir_all(&root).await?; + seed_extension_instructions(&root).await?; + phase2::run(context, config).await; + let request = wait_for_single_request(&response).await; + wait_for_phase2_workspace_reset(&home.path().join("memories")).await?; + shutdown_test_codex(&test).await?; + Ok(request) +} + +fn startup_test_memories_config() -> MemoriesConfig { + MemoriesConfig { + max_raw_memories_for_consolidation: 1, + min_rollout_idle_hours: 0, + ..MemoriesConfig::default() + } +} + async fn build_test_codex( server: &wiremock::MockServer, home: Arc, +) -> anyhow::Result { + build_test_codex_with_memories_config(server, home, startup_test_memories_config()).await +} + +async fn build_test_codex_with_memories_config( + server: &wiremock::MockServer, + home: Arc, + memories: MemoriesConfig, ) -> anyhow::Result { test_codex() .with_home(home) - .with_config(|config| { + .with_config(move |config| { config .features .enable(Feature::Sqlite) .expect("test config should allow feature update"); - config.memories.max_raw_memories_for_consolidation = 1; + config.memories = memories; }) .build(server) .await @@ -370,6 +559,88 @@ async fn trigger_memories_startup(test: &TestCodex) { ); } +async fn memory_startup_context_with_provider( + test: &TestCodex, + provider: SharedModelProvider, +) -> (Arc, Arc) { + let config_snapshot = test.codex.config_snapshot().await; + let mut config = test.config.clone(); + config + .features + .enable(Feature::MemoryTool) + .expect("test config should allow feature update"); + let config = Arc::new(config); + let context = Arc::new(MemoryStartupContext::new_for_testing( + Arc::clone(&test.thread_manager), + test.thread_manager.auth_manager(), + test.session_configured.thread_id, + Arc::clone(&test.codex), + config.as_ref(), + config_snapshot.session_source, + provider, + )); + + (context, config) +} + +const MOCK_PROVIDER_PHASE_ONE_MODEL: &str = "mock.phase-one"; +const MOCK_PROVIDER_PHASE_TWO_MODEL: &str = "mock.phase-two"; + +#[derive(Debug)] +struct MockMemoryModelProvider { + delegate: SharedModelProvider, +} + +impl MockMemoryModelProvider { + fn new(info: ModelProviderInfo, auth_manager: Option>) -> Self { + Self { + delegate: create_model_provider(info, auth_manager), + } + } +} + +impl ModelProvider for MockMemoryModelProvider { + fn info(&self) -> &ModelProviderInfo { + self.delegate.info() + } + + fn memory_extraction_preferred_model(&self) -> &'static str { + MOCK_PROVIDER_PHASE_ONE_MODEL + } + + fn memory_consolidation_preferred_model(&self) -> &'static str { + MOCK_PROVIDER_PHASE_TWO_MODEL + } + + fn auth_manager(&self) -> Option> { + self.delegate.auth_manager() + } + + fn auth<'life0, 'async_trait>( + &'life0 self, + ) -> Pin> + Send + 'async_trait>> + where + 'life0: 'async_trait, + Self: 'async_trait, + { + let delegate = Arc::clone(&self.delegate); + Box::pin(async move { delegate.auth().await }) + } + + fn account_state(&self) -> ProviderAccountResult { + self.delegate.account_state() + } + + fn models_manager( + &self, + codex_home: PathBuf, + config_model_catalog: Option, + ) -> codex_models_manager::manager::SharedModelsManager { + self.delegate + .models_manager(codex_home, config_model_catalog) + } +} + async fn seed_stage1_output( db: &codex_state::StateRuntime, codex_home: &Path, @@ -404,6 +675,46 @@ async fn seed_stage1_output( Ok(thread_id) } +async fn seed_stage1_candidate( + db: &codex_state::StateRuntime, + codex_home: &Path, + updated_at: chrono::DateTime, + rollout_slug: &str, +) -> anyhow::Result { + let thread_id = ThreadId::new(); + let rollout_path = codex_home.join(format!("rollout-{thread_id}.jsonl")); + let line = RolloutLine { + timestamp: updated_at.to_rfc3339(), + item: RolloutItem::ResponseItem(ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::InputText { + text: "remember this startup test conversation".to_string(), + }], + phase: None, + }), + }; + let jsonl = serde_json::to_string(&line)?; + tokio::fs::write(&rollout_path, format!("{jsonl}\n")).await?; + + let mut metadata_builder = codex_state::ThreadMetadataBuilder::new( + thread_id, + rollout_path, + updated_at, + SessionSource::Cli, + ); + metadata_builder.cwd = codex_home.join(format!("workspace-{rollout_slug}")); + metadata_builder.model_provider = Some("test-provider".to_string()); + metadata_builder.git_branch = Some(format!("branch-{rollout_slug}")); + let mut metadata = metadata_builder.build("test-provider"); + metadata.preview = Some("remember this startup test conversation".to_string()); + metadata.first_user_message = metadata.preview.clone(); + db.upsert_thread(&metadata).await?; + db.set_thread_memory_mode(thread_id, "enabled").await?; + + Ok(thread_id) +} + async fn wait_for_single_request(mock: &ResponseMock) -> ResponsesRequest { wait_for_request(mock, /*expected_count*/ 1).await.remove(0) } @@ -447,7 +758,8 @@ async fn wait_for_request(mock: &ResponseMock, expected_count: usize) -> Vec &'static str { + AMAZON_BEDROCK_GPT_5_4_MODEL_ID + } + + fn memory_consolidation_preferred_model(&self) -> &'static str { + AMAZON_BEDROCK_GPT_5_4_MODEL_ID + } + fn auth_manager(&self) -> Option> { None } diff --git a/codex-rs/model-provider/src/provider.rs b/codex-rs/model-provider/src/provider.rs index afd07065d4..e786192433 100644 --- a/codex-rs/model-provider/src/provider.rs +++ b/codex-rs/model-provider/src/provider.rs @@ -74,6 +74,14 @@ pub type ProviderAccountResult = std::result::Result &'static str { + DEFAULT_MEMORY_EXTRACTION_PREFERRED_MODEL + } + + /// Returns the preferred model used for memory consolidation. + /// + /// Providers that require backend-specific model IDs should override this. + fn memory_consolidation_preferred_model(&self) -> &'static str { + DEFAULT_MEMORY_CONSOLIDATION_PREFERRED_MODEL + } + /// Returns whether requests made through this provider should include attestation. fn supports_attestation(&self) -> bool { false From 5ac640ac49ae5c8b781d51bbaf0467c98cda2643 Mon Sep 17 00:00:00 2001 From: mchen-oai Date: Tue, 9 Jun 2026 17:49:59 -0700 Subject: [PATCH 21/47] Add spans to build_tool_router (#27094) ## Why - Local profiling shows `append_tool_search_executor` averages ~113ms per call. Adding a span lets us track this cost as we optimize in follow-up PRs, either by reducing the work or avoiding repeated rebuilds when inputs have not changed. - While we're here, we can add spans to `build_tool_router` and other sub-calls which code analysis shows may have additional opportunities for improvement. ## What changed Add function-level trace spans around `build_tool_router`, `build_tool_specs_and_registry`, `add_tool_sources`, `append_tool_search_executor`, and `build_model_visible_specs_and_registry` ## Verification Trigger Codex rollout and observe new spans are included --- codex-rs/core/src/tools/spec_plan.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/codex-rs/core/src/tools/spec_plan.rs b/codex-rs/core/src/tools/spec_plan.rs index fed62a0ee1..19f4c9f897 100644 --- a/codex-rs/core/src/tools/spec_plan.rs +++ b/codex-rs/core/src/tools/spec_plan.rs @@ -86,6 +86,7 @@ use codex_tools::shell_type_for_model_and_features; use std::collections::BTreeMap; use std::collections::HashSet; use std::sync::Arc; +use tracing::instrument; use tracing::warn; const MULTI_AGENT_V2_NAMESPACE_DESCRIPTION: &str = "Tools for spawning and managing sub-agents."; @@ -148,6 +149,7 @@ struct CoreToolPlanContext<'a> { wait_agent_timeouts: WaitAgentTimeoutOptions, } +#[instrument(level = "trace", skip_all)] pub(crate) fn build_tool_router( turn_context: &TurnContext, params: ToolRouterParams<'_>, @@ -156,6 +158,7 @@ pub(crate) fn build_tool_router( ToolRouter::from_parts(registry, model_visible_specs) } +#[instrument(level = "trace", skip_all)] fn build_tool_specs_and_registry( turn_context: &TurnContext, params: ToolRouterParams<'_>, @@ -186,6 +189,7 @@ fn build_tool_specs_and_registry( build_model_visible_specs_and_registry(turn_context, planned_tools) } +#[instrument(level = "trace", skip_all)] fn build_model_visible_specs_and_registry( turn_context: &TurnContext, planned_tools: PlannedTools, @@ -557,6 +561,7 @@ fn code_mode_namespace_descriptions( namespace_descriptions } +#[instrument(level = "trace", skip_all)] fn add_tool_sources(context: &CoreToolPlanContext<'_>, planned_tools: &mut PlannedTools) { add_shell_tools(context, planned_tools); add_mcp_resource_tools(context, planned_tools); @@ -837,6 +842,7 @@ fn add_extension_tools(context: &CoreToolPlanContext<'_>, planned_tools: &mut Pl ); } +#[instrument(level = "trace", skip_all)] fn append_tool_search_executor( context: &CoreToolPlanContext<'_>, planned_tools: &mut PlannedTools, From 4a3eac214494a5cefebe53308dc48ebabf03201c Mon Sep 17 00:00:00 2001 From: guinness-oai Date: Tue, 9 Jun 2026 17:54:32 -0700 Subject: [PATCH 22/47] Add per-session realtime model and version overrides (#24999) ## Why Clients need to select a realtime session configuration for an individual start without rewriting persisted configuration or restarting the app-server process. ## What Changed - Add optional `model` and `version` fields to `thread/realtime/start` - Forward those optional values through the realtime start operation and apply them only for that session - Preserve existing configured/default behavior when the new fields are omitted - Update generated protocol schema and app-server documentation ## Validation - Added/updated protocol serialization coverage for the new optional request fields - Added focused core coverage for a session override taking precedence over configured realtime selection - Added focused app-server coverage that a request override reaches the realtime WebSocket handshake --- .../schema/json/ClientRequest.json | 7 ++ .../src/protocol/common.rs | 14 ++++ .../src/protocol/v2/realtime.rs | 6 ++ codex-rs/app-server/README.md | 4 +- .../src/request_processors/turn_processor.rs | 2 + .../tests/suite/v2/experimental_api.rs | 4 + .../tests/suite/v2/realtime_conversation.rs | 18 +++++ codex-rs/core/src/realtime_conversation.rs | 21 ++--- codex-rs/core/tests/suite/compact_remote.rs | 2 + .../core/tests/suite/realtime_conversation.rs | 76 ++++++++++++++++++- codex-rs/protocol/src/protocol.rs | 4 + 11 files changed, 145 insertions(+), 13 deletions(-) diff --git a/codex-rs/app-server-protocol/schema/json/ClientRequest.json b/codex-rs/app-server-protocol/schema/json/ClientRequest.json index b360d4bddd..2701dfb34d 100644 --- a/codex-rs/app-server-protocol/schema/json/ClientRequest.json +++ b/codex-rs/app-server-protocol/schema/json/ClientRequest.json @@ -1993,6 +1993,13 @@ ], "type": "object" }, + "RealtimeConversationVersion": { + "enum": [ + "v1", + "v2" + ], + "type": "string" + }, "RealtimeOutputModality": { "enum": [ "text", diff --git a/codex-rs/app-server-protocol/src/protocol/common.rs b/codex-rs/app-server-protocol/src/protocol/common.rs index d3c8eae22b..d3245091a9 100644 --- a/codex-rs/app-server-protocol/src/protocol/common.rs +++ b/codex-rs/app-server-protocol/src/protocol/common.rs @@ -2942,10 +2942,12 @@ mod tests { request_id: RequestId::Integer(9), params: v2::ThreadRealtimeStartParams { thread_id: "thr_123".to_string(), + model: Some("realtime-treatment-model".to_string()), output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("You are on a call".to_string())), realtime_session_id: Some("sess_456".to_string()), transport: None, + version: Some(RealtimeConversationVersion::V1), voice: Some(RealtimeVoice::Marin), }, }; @@ -2955,10 +2957,12 @@ mod tests { "id": 9, "params": { "threadId": "thr_123", + "model": "realtime-treatment-model", "outputModality": "audio", "prompt": "You are on a call", "realtimeSessionId": "sess_456", "transport": null, + "version": "v1", "voice": "marin" } }), @@ -2973,10 +2977,12 @@ mod tests { request_id: RequestId::Integer(9), params: v2::ThreadRealtimeStartParams { thread_id: "thr_123".to_string(), + model: None, output_modality: RealtimeOutputModality::Audio, prompt: None, realtime_session_id: None, transport: None, + version: None, voice: None, }, }; @@ -2986,9 +2992,11 @@ mod tests { "id": 9, "params": { "threadId": "thr_123", + "model": null, "outputModality": "audio", "realtimeSessionId": null, "transport": null, + "version": null, "voice": null } }), @@ -2999,10 +3007,12 @@ mod tests { request_id: RequestId::Integer(9), params: v2::ThreadRealtimeStartParams { thread_id: "thr_123".to_string(), + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(None), realtime_session_id: None, transport: None, + version: None, voice: None, }, }; @@ -3012,10 +3022,12 @@ mod tests { "id": 9, "params": { "threadId": "thr_123", + "model": null, "outputModality": "audio", "prompt": null, "realtimeSessionId": null, "transport": null, + "version": null, "voice": null } }), @@ -3168,10 +3180,12 @@ mod tests { request_id: RequestId::Integer(1), params: v2::ThreadRealtimeStartParams { thread_id: "thr_123".to_string(), + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("You are on a call".to_string())), realtime_session_id: None, transport: None, + version: None, voice: None, }, }; diff --git a/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs b/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs index c6ea0744de..a86cd8067d 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs @@ -65,6 +65,9 @@ impl From for CoreRealtimeAudioFrame { #[ts(export_to = "v2/")] pub struct ThreadRealtimeStartParams { pub thread_id: String, + /// Overrides the configured realtime model for this session only. + #[ts(optional = nullable)] + pub model: Option, /// Selects text or audio output for the realtime session. Transport and voice stay /// independent so clients can choose how they connect separately from what the model emits. pub output_modality: RealtimeOutputModality, @@ -80,6 +83,9 @@ pub struct ThreadRealtimeStartParams { pub realtime_session_id: Option, #[ts(optional = nullable)] pub transport: Option, + /// Overrides the configured realtime protocol version for this session only. + #[ts(optional = nullable)] + pub version: Option, #[ts(optional = nullable)] pub voice: Option, } diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index e09866e196..999cc8fa37 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -162,7 +162,7 @@ Example with notification opt-out: - `thread/inject_items` — append raw Responses API items to a loaded thread’s model-visible history without starting a user turn; returns `{}` on success. - `turn/steer` — add user input to an already in-flight regular turn without starting a new turn; returns the active `turnId` that accepted the input. `clientUserMessageId` is optional; when supplied, the corresponding `userMessage` item echoes it as `clientId`. Review and manual compaction turns reject `turn/steer`. - `turn/interrupt` — request cancellation of an in-flight turn by `(thread_id, turn_id)`; success is an empty `{}` response and the turn finishes with `status: "interrupted"`. -- `thread/realtime/start` — start a thread-scoped realtime session (experimental); pass `outputModality: "text"` or `outputModality: "audio"` to choose model output, returns `{}` and streams `thread/realtime/*` notifications. Omit `transport` for the websocket transport, or pass `{ "type": "webrtc", "sdp": "..." }` to create a WebRTC session from a browser-generated SDP offer; the remote answer SDP is emitted as `thread/realtime/sdp`. +- `thread/realtime/start` — start a thread-scoped realtime session (experimental); pass `outputModality: "text"` or `outputModality: "audio"` to choose model output, and optionally pass `model` and `version` to override configured realtime selection for this session only. Returns `{}` and streams `thread/realtime/*` notifications. Omit `transport` for the websocket transport, or pass `{ "type": "webrtc", "sdp": "..." }` to create a WebRTC session from a browser-generated SDP offer; the remote answer SDP is emitted as `thread/realtime/sdp`. - `thread/realtime/appendAudio` — append an input audio chunk to the active realtime session (experimental); returns `{}`. - `thread/realtime/appendText` — append text input to the active realtime session (experimental); returns `{}`. - `thread/realtime/stop` — stop the active realtime session for the thread (experimental); returns `{}`. @@ -837,6 +837,8 @@ Then send `offer.sdp` to app-server. Core uses `experimental_realtime_ws_backend Omit `prompt` to use Codex's default realtime backend prompt. Send `prompt: null` or `prompt: ""` when the session should start without that default backend prompt. +Clients may also pass `model` and `version` on `thread/realtime/start` to select a +different realtime session configuration without changing thread or user config. ```javascript await pc.setRemoteDescription({ diff --git a/codex-rs/app-server/src/request_processors/turn_processor.rs b/codex-rs/app-server/src/request_processors/turn_processor.rs index 3d44781946..b2e6ddc0b9 100644 --- a/codex-rs/app-server/src/request_processors/turn_processor.rs +++ b/codex-rs/app-server/src/request_processors/turn_processor.rs @@ -934,6 +934,7 @@ impl TurnRequestProcessor { request_id, thread.as_ref(), Op::RealtimeConversationStart(ConversationStartParams { + model: params.model, output_modality: params.output_modality, prompt: params.prompt, realtime_session_id: params.realtime_session_id, @@ -945,6 +946,7 @@ impl TurnRequestProcessor { ConversationStartTransport::Webrtc { sdp } } }), + version: params.version, voice: params.voice, }), ) diff --git a/codex-rs/app-server/tests/suite/v2/experimental_api.rs b/codex-rs/app-server/tests/suite/v2/experimental_api.rs index fc1100d17b..03c9d455f0 100644 --- a/codex-rs/app-server/tests/suite/v2/experimental_api.rs +++ b/codex-rs/app-server/tests/suite/v2/experimental_api.rs @@ -80,10 +80,12 @@ async fn realtime_conversation_start_requires_experimental_api_capability() -> R let request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { thread_id: "thr_123".to_string(), + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("hello".to_string())), realtime_session_id: None, transport: None, + version: None, voice: None, }) .await?; @@ -186,12 +188,14 @@ async fn realtime_webrtc_start_requires_experimental_api_capability() -> Result< let request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { thread_id: "thr_123".to_string(), + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("hello".to_string())), realtime_session_id: None, transport: Some(ThreadRealtimeStartTransport::Webrtc { sdp: "v=offer\r\n".to_string(), }), + version: None, voice: None, }) .await?; diff --git a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs index f88fb54929..bd03af8e02 100644 --- a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs +++ b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs @@ -311,12 +311,14 @@ impl RealtimeE2eHarness { .mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { thread_id: self.thread_id.clone(), + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: Some(ThreadRealtimeStartTransport::Webrtc { sdp: offer_sdp.to_string(), }), + version: None, voice: None, }) .await?; @@ -556,10 +558,12 @@ async fn realtime_conversation_streams_v2_notifications() -> Result<()> { let start_request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { thread_id: thread_start.thread.id.clone(), + model: Some("realtime-treatment-model".to_string()), output_modality: RealtimeOutputModality::Audio, prompt: None, realtime_session_id: None, transport: None, + version: None, voice: Some(RealtimeVoice::Cedar), }) .await?; @@ -588,6 +592,10 @@ async fn realtime_conversation_streams_v2_notifications() -> Result<()> { startup_context_request.body_json()["session"]["audio"]["output"]["voice"], "cedar" ); + assert_eq!( + realtime_server.single_handshake().uri(), + "/v1/realtime?model=realtime-treatment-model" + ); assert_eq!( startup_context_request.body_json()["session"]["output_modalities"], json!(["audio"]) @@ -805,10 +813,12 @@ async fn realtime_text_output_modality_requests_text_output_and_final_transcript let start_request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { thread_id: thread_start.thread.id.clone(), + model: None, output_modality: RealtimeOutputModality::Text, prompt: None, realtime_session_id: None, transport: None, + version: None, voice: None, }) .await?; @@ -979,10 +989,12 @@ async fn realtime_conversation_stop_emits_closed_notification() -> Result<()> { let start_request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { thread_id: thread_start.thread.id.clone(), + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, + version: None, voice: None, }) .await?; @@ -1076,12 +1088,14 @@ async fn realtime_webrtc_start_emits_sdp_notification() -> Result<()> { let start_request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { thread_id: thread_id.clone(), + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: Some(ThreadRealtimeStartTransport::Webrtc { sdp: "v=offer\r\n".to_string(), }), + version: None, voice: None, }) .await?; @@ -1991,12 +2005,14 @@ async fn realtime_webrtc_start_surfaces_backend_error() -> Result<()> { let start_request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { thread_id: thread_start.thread.id, + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: Some(ThreadRealtimeStartTransport::Webrtc { sdp: "v=offer\r\n".to_string(), }), + version: None, voice: None, }) .await?; @@ -2050,10 +2066,12 @@ async fn realtime_conversation_requires_feature_flag() -> Result<()> { let start_request_id = mcp .send_thread_realtime_start_request(ThreadRealtimeStartParams { thread_id: thread_start.thread.id.clone(), + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, + version: None, voice: None, }) .await?; diff --git a/codex-rs/core/src/realtime_conversation.rs b/codex-rs/core/src/realtime_conversation.rs index 47bdc8c0c1..66dc653e56 100644 --- a/codex-rs/core/src/realtime_conversation.rs +++ b/codex-rs/core/src/realtime_conversation.rs @@ -625,12 +625,14 @@ async fn prepare_realtime_start( if let Some(realtime_ws_base_url) = &config.experimental_realtime_ws_base_url { api_provider.base_url = realtime_ws_base_url.clone(); } - let version = config.realtime.version; + let version = params.version.unwrap_or(config.realtime.version); let session_config = build_realtime_session_config( sess, + params.model, params.prompt, params.realtime_session_id, params.output_modality, + version, params.voice, ) .await?; @@ -664,9 +666,11 @@ async fn prepare_realtime_start( pub(crate) async fn build_realtime_session_config( sess: &Arc, + model: Option, prompt: Option>, realtime_session_id: Option, output_modality: RealtimeOutputModality, + version: RealtimeWsVersion, voice: Option, ) -> CodexResult { let config = sess.get_config().await; @@ -689,18 +693,15 @@ pub(crate) async fn build_realtime_session_config( (false, false) => format!("{prompt}\n\n{startup_context}"), }; let model = Some( - config - .experimental_realtime_ws_model - .clone() + model + .or_else(|| config.experimental_realtime_ws_model.clone()) .unwrap_or_else(|| DEFAULT_REALTIME_MODEL.to_string()), ); - let event_parser = match config.realtime.version { + let event_parser = match version { RealtimeWsVersion::V1 => RealtimeEventParser::V1, RealtimeWsVersion::V2 => RealtimeEventParser::RealtimeV2, }; - if config.realtime.version == RealtimeWsVersion::V1 - && matches!(output_modality, RealtimeOutputModality::Text) - { + if version == RealtimeWsVersion::V1 && matches!(output_modality, RealtimeOutputModality::Text) { return Err(CodexErr::InvalidRequest( "text realtime output modality requires realtime v2".to_string(), )); @@ -711,8 +712,8 @@ pub(crate) async fn build_realtime_session_config( }; let voice = voice .or(config.realtime.voice) - .unwrap_or_else(|| default_realtime_voice(config.realtime.version)); - validate_realtime_voice(config.realtime.version, voice)?; + .unwrap_or_else(|| default_realtime_voice(version)); + validate_realtime_voice(version, voice)?; Ok(RealtimeSessionConfig { instructions: prompt, model, diff --git a/codex-rs/core/tests/suite/compact_remote.rs b/codex-rs/core/tests/suite/compact_remote.rs index 73a500f04d..eee445ea9f 100644 --- a/codex-rs/core/tests/suite/compact_remote.rs +++ b/codex-rs/core/tests/suite/compact_remote.rs @@ -194,10 +194,12 @@ async fn start_remote_realtime_server() -> responses::WebSocketTestServer { async fn start_realtime_conversation(codex: &codex_core::CodexThread) -> Result<()> { codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, + version: None, voice: None, })) .await?; diff --git a/codex-rs/core/tests/suite/realtime_conversation.rs b/codex-rs/core/tests/suite/realtime_conversation.rs index 095b3f59d4..25f0516b0d 100644 --- a/codex-rs/core/tests/suite/realtime_conversation.rs +++ b/codex-rs/core/tests/suite/realtime_conversation.rs @@ -282,10 +282,12 @@ async fn conversation_start_audio_text_close_round_trip() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, + version: None, voice: None, })) .await?; @@ -420,10 +422,12 @@ async fn conversation_start_defaults_to_v2_and_gpt_realtime_1_5() -> Result<()> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, + version: None, voice: None, })) .await?; @@ -502,18 +506,20 @@ async fn conversation_webrtc_start_posts_generated_session() -> Result<()> { config.experimental_realtime_ws_model = Some("realtime-test-model".to_string()); config.experimental_realtime_ws_startup_context = Some("startup context".to_string()); config.experimental_realtime_ws_base_url = Some(realtime_ws_base_url); - config.realtime.version = RealtimeWsVersion::V1; + config.realtime.version = RealtimeWsVersion::V2; }); let test = builder.build(&server).await?; test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + model: Some("session-override-model".to_string()), output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: Some(ConversationStartTransport::Webrtc { sdp: "v=offer\r\n".to_string(), }), + version: Some(RealtimeConversationVersion::V1), voice: None, })) .await?; @@ -572,7 +578,7 @@ async fn conversation_webrtc_start_posts_generated_session() -> Result<()> { Some("multipart/form-data; boundary=codex-realtime-call-boundary") ); let body = String::from_utf8(request.body).context("multipart body should be utf-8")?; - let session = r#"{"audio":{"input":{"format":{"type":"audio/pcm","rate":24000}},"output":{"voice":"cove"}},"type":"quicksilver","model":"realtime-test-model","instructions":"backend prompt\n\nstartup context"}"#; + let session = r#"{"audio":{"input":{"format":{"type":"audio/pcm","rate":24000}},"output":{"voice":"cove"}},"type":"quicksilver","model":"session-override-model","instructions":"backend prompt\n\nstartup context"}"#; let session = normalized_json_string(session)?; assert_eq!( body, @@ -678,12 +684,14 @@ async fn conversation_webrtc_close_while_sideband_connecting_drops_pending_join( test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: Some(ConversationStartTransport::Webrtc { sdp: "v=offer\r\n".to_string(), }), + version: None, voice: None, })) .await?; @@ -762,12 +770,14 @@ async fn conversation_webrtc_sideband_connect_failure_closes_with_error() -> Res test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: Some(ConversationStartTransport::Webrtc { sdp: "v=offer\r\n".to_string(), }), + version: None, voice: None, })) .await?; @@ -847,10 +857,12 @@ async fn conversation_start_uses_openai_env_key_fallback_with_chatgpt_auth() -> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, + version: None, voice: None, })) .await?; @@ -913,10 +925,12 @@ async fn conversation_transport_close_emits_closed_event() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, + version: None, voice: None, })) .await?; @@ -1003,10 +1017,12 @@ async fn conversation_start_preflight_failure_emits_realtime_error_only() -> Res test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, + version: None, voice: None, })) .await?; @@ -1047,10 +1063,12 @@ async fn conversation_start_connect_failure_emits_realtime_error_only() -> Resul test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, + version: None, voice: None, })) .await?; @@ -1138,10 +1156,12 @@ async fn conversation_second_start_replaces_runtime() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("old".to_string())), realtime_session_id: Some("conv_old".to_string()), transport: None, + version: None, voice: None, })) .await?; @@ -1161,10 +1181,12 @@ async fn conversation_second_start_replaces_runtime() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("new".to_string())), realtime_session_id: Some("conv_new".to_string()), transport: None, + version: None, voice: None, })) .await?; @@ -1255,10 +1277,12 @@ async fn conversation_uses_experimental_realtime_ws_base_url_override() -> Resul test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, + version: None, voice: None, })) .await?; @@ -1317,10 +1341,12 @@ async fn conversation_uses_default_realtime_backend_prompt() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + model: None, output_modality: RealtimeOutputModality::Audio, prompt: None, realtime_session_id: None, transport: None, + version: None, voice: None, })) .await?; @@ -1387,10 +1413,12 @@ async fn conversation_uses_empty_instructions_for_null_or_empty_prompt() -> Resu ] { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + model: None, output_modality: RealtimeOutputModality::Audio, prompt, realtime_session_id: None, transport: None, + version: None, voice: None, })) .await?; @@ -1450,10 +1478,12 @@ async fn conversation_uses_explicit_start_voice() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, + version: None, voice: Some(RealtimeVoice::Breeze), })) .await?; @@ -1505,10 +1535,12 @@ async fn conversation_uses_configured_realtime_voice() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, + version: None, voice: None, })) .await?; @@ -1548,10 +1580,12 @@ async fn conversation_rejects_voice_for_wrong_realtime_version() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, + version: None, voice: Some(RealtimeVoice::Cove), })) .await?; @@ -1592,10 +1626,12 @@ async fn conversation_uses_experimental_realtime_ws_backend_prompt_override() -> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("prompt from op".to_string())), realtime_session_id: None, transport: None, + version: None, voice: None, })) .await?; @@ -1662,10 +1698,12 @@ async fn conversation_uses_experimental_realtime_ws_startup_context_override() - test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("prompt from op".to_string())), realtime_session_id: None, transport: None, + version: None, voice: None, })) .await?; @@ -1726,10 +1764,12 @@ async fn conversation_disables_realtime_startup_context_with_empty_override() -> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("prompt from op".to_string())), realtime_session_id: None, transport: None, + version: None, voice: None, })) .await?; @@ -1783,10 +1823,12 @@ async fn conversation_start_injects_startup_context_from_thread_history() -> Res test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, + version: None, voice: None, })) .await?; @@ -1892,10 +1934,12 @@ async fn conversation_startup_context_current_thread_selects_many_turns_by_budge codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, + version: None, voice: None, })) .await?; @@ -1997,10 +2041,12 @@ async fn conversation_startup_context_falls_back_to_workspace_map() -> Result<() test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, + version: None, voice: None, })) .await?; @@ -2054,10 +2100,12 @@ async fn conversation_startup_context_is_truncated_and_sent_once_per_start() -> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, + version: None, voice: None, })) .await?; @@ -2131,10 +2179,12 @@ async fn conversation_user_text_turn_is_not_sent_to_realtime() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, + version: None, voice: None, })) .await?; @@ -2225,10 +2275,12 @@ async fn realtime_v2_noop_tool_call_returns_empty_function_output_without_respon test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, + version: None, voice: None, })) .await?; @@ -2321,10 +2373,12 @@ async fn conversation_mirrors_assistant_message_text_to_realtime_handoff() -> Re test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, + version: None, voice: None, })) .await?; @@ -2455,10 +2509,12 @@ async fn conversation_handoff_persists_across_item_done_until_turn_complete() -> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, + version: None, voice: None, })) .await?; @@ -2604,10 +2660,12 @@ async fn inbound_handoff_request_starts_turn() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, + version: None, voice: None, })) .await?; @@ -2703,10 +2761,12 @@ async fn inbound_handoff_request_uses_active_transcript() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, + version: None, voice: None, })) .await?; @@ -2803,10 +2863,12 @@ async fn inbound_handoff_request_sends_transcript_delta_after_each_handoff() -> test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, + version: None, voice: None, })) .await?; @@ -2901,10 +2963,12 @@ async fn inbound_conversation_item_does_not_start_turn_and_still_forwards_audio( test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, + version: None, voice: None, })) .await?; @@ -3021,10 +3085,12 @@ async fn delegated_turn_user_role_echo_does_not_redelegate_and_still_forwards_au test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, + version: None, voice: None, })) .await?; @@ -3171,10 +3237,12 @@ async fn inbound_handoff_request_does_not_block_realtime_event_forwarding() -> R test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, + version: None, voice: None, })) .await?; @@ -3305,10 +3373,12 @@ async fn inbound_handoff_request_steers_active_turn() -> Result<()> { test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, + version: None, voice: None, })) .await?; @@ -3455,10 +3525,12 @@ async fn inbound_handoff_request_starts_turn_and_does_not_block_realtime_audio() test.codex .submit(Op::RealtimeConversationStart(ConversationStartParams { + model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), realtime_session_id: None, transport: None, + version: None, voice: None, })) .await?; diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index 5abc9c6399..e93801d29f 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -183,11 +183,15 @@ pub struct McpServerRefreshConfig { #[derive(Debug, Clone, PartialEq)] pub struct ConversationStartParams { + /// Overrides the configured realtime model for this session only. + pub model: Option, /// Selects whether the realtime session should produce text or audio output. pub output_modality: RealtimeOutputModality, pub prompt: Option>, pub realtime_session_id: Option, pub transport: Option, + /// Overrides the configured realtime protocol version for this session only. + pub version: Option, pub voice: Option, } From 608b8b1cc6ce91064e1fd12e0810e1772b5e4710 Mon Sep 17 00:00:00 2001 From: marksteinbrick-oai Date: Tue, 9 Jun 2026 18:45:54 -0700 Subject: [PATCH 23/47] [codex-analytics] emit goal lifecycle analytics (#27078) ## Why - Currently, there is no analytics event for `/goal` behavior - Existing events cannot identify goal execution or its resulting outcome - The original update in [#26182](https://github.com/openai/codex/pull/26182) was implemented before `/goal` moved into `codex-goal-extension`. ## What Changed - Adds `codex_goal_event` serialization and enrichment to `codex-analytics` - Emits goal events from the canonical `codex-goal-extension` mutation and accounting paths: - `created` when a new logical goal is persisted - `usage_accounted` when cumulative goal usage is persisted - `status_changed` when the stored goal status changes - `cleared` when the goal is deleted - Preserves causal `turn_id` for turn driven events and uses null attribution for external or idle lifecycle events - Changes goal deletion to return the deleted row so `cleared` retains the stable goal ID ## Event Details Includes standard analytics metadata along with goal specific fields: - `goal_id`: Stable ID stored in the local SQLite goal row and shared across the goal's events - `event_kind`: Observed operation (see the 4 lifecycle events cited in the above bullet) - `goal_status`: Resulting or last stored status: `active`, `paused`, `blocked`, `usage_limited`, etc. - `has_token_budget`: Indicates whether a token budget is configured - `turn_id`: Causal turn ID, or null when no causal turn exists - `cumulative_tokens_accounted`: Cumulative tokens on `usage_accounted` events; null otherwise - `cumulative_time_accounted_seconds`: Cumulative active time on `usage_accounted` events; null otherwise ## Validation - `just test -p codex-analytics -p codex-state -p codex-goal-extension` - `just test -p codex-core -E 'test(/goal/)'` - `just test -p codex-app-server` - `cargo build -p codex-analytics -p codex-core -p codex-state -p codex-app-server` --- codex-rs/Cargo.lock | 2 + codex-rs/analytics/Cargo.toml | 1 + codex-rs/analytics/src/client.rs | 7 ++ codex-rs/analytics/src/events.rs | 54 +++++++++++++ codex-rs/analytics/src/facts.rs | 22 ++++++ codex-rs/analytics/src/lib.rs | 2 + codex-rs/analytics/src/reducer.rs | 36 +++++++++ codex-rs/app-server/src/extensions.rs | 4 +- codex-rs/app-server/src/mcp_refresh.rs | 1 + codex-rs/app-server/src/message_processor.rs | 1 + .../app-server/tests/suite/v2/analytics.rs | 30 +++++++- .../tests/suite/v2/thread_resume.rs | 77 +++++++++++++++++-- codex-rs/ext/goal/Cargo.toml | 1 + codex-rs/ext/goal/src/analytics.rs | 77 +++++++++++++++++++ codex-rs/ext/goal/src/api.rs | 8 +- codex-rs/ext/goal/src/extension.rs | 11 +++ codex-rs/ext/goal/src/lib.rs | 1 + codex-rs/ext/goal/src/runtime.rs | 38 ++++++++- codex-rs/ext/goal/src/tool.rs | 25 ++++++ .../ext/goal/tests/goal_extension_backend.rs | 3 + codex-rs/state/src/model/thread_goal.rs | 4 +- codex-rs/state/src/runtime/goals.rs | 29 +++++-- codex-rs/state/src/runtime/threads.rs | 2 +- 23 files changed, 412 insertions(+), 24 deletions(-) create mode 100644 codex-rs/ext/goal/src/analytics.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 511fca7250..46bb21bf62 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1892,6 +1892,7 @@ dependencies = [ "codex-model-provider", "codex-plugin", "codex-protocol", + "codex-state", "codex-utils-absolute-path", "os_info", "pretty_assertions", @@ -3049,6 +3050,7 @@ dependencies = [ "anyhow", "async-trait", "chrono", + "codex-analytics", "codex-core", "codex-extension-api", "codex-otel", diff --git a/codex-rs/analytics/Cargo.toml b/codex-rs/analytics/Cargo.toml index 918e7edc72..8546427469 100644 --- a/codex-rs/analytics/Cargo.toml +++ b/codex-rs/analytics/Cargo.toml @@ -19,6 +19,7 @@ codex-login = { workspace = true } codex-model-provider = { workspace = true } codex-plugin = { workspace = true } codex-protocol = { workspace = true } +codex-state = { workspace = true } os_info = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } diff --git a/codex-rs/analytics/src/client.rs b/codex-rs/analytics/src/client.rs index b99a2ec86f..193de9ef65 100644 --- a/codex-rs/analytics/src/client.rs +++ b/codex-rs/analytics/src/client.rs @@ -9,6 +9,7 @@ use crate::facts::AnalyticsJsonRpcError; use crate::facts::AppInvocation; use crate::facts::AppMentionedInput; use crate::facts::AppUsedInput; +use crate::facts::CodexGoalEvent; use crate::facts::CustomAnalyticsFact; use crate::facts::HookRunFact; use crate::facts::HookRunInput; @@ -246,6 +247,12 @@ impl AnalyticsEventsClient { ))); } + pub fn track_goal_event(&self, event: CodexGoalEvent) { + self.record_fact(AnalyticsFact::Custom(CustomAnalyticsFact::Goal(Box::new( + event, + )))); + } + pub fn track_turn_resolved_config(&self, fact: TurnResolvedConfigFact) { self.record_fact(AnalyticsFact::Custom( CustomAnalyticsFact::TurnResolvedConfig(Box::new(fact)), diff --git a/codex-rs/analytics/src/events.rs b/codex-rs/analytics/src/events.rs index 0b6c9a0122..8a25215ec8 100644 --- a/codex-rs/analytics/src/events.rs +++ b/codex-rs/analytics/src/events.rs @@ -4,12 +4,14 @@ use crate::facts::AcceptedLineFingerprint; use crate::facts::AppInvocation; use crate::facts::CodexCompactionEvent; use crate::facts::CodexErrKind; +use crate::facts::CodexGoalEvent; use crate::facts::CompactionImplementation; use crate::facts::CompactionPhase; use crate::facts::CompactionReason; use crate::facts::CompactionStatus; use crate::facts::CompactionStrategy; use crate::facts::CompactionTrigger; +use crate::facts::GoalEventKind; use crate::facts::HookRunFact; use crate::facts::InvocationType; use crate::facts::PluginState; @@ -63,6 +65,7 @@ pub(crate) enum TrackEventRequest { AppUsed(CodexAppUsedEventRequest), HookRun(CodexHookRunEventRequest), Compaction(Box), + Goal(Box), TurnEvent(Box), TurnSteer(CodexTurnSteerEventRequest), CommandExecution(CodexCommandExecutionEventRequest), @@ -771,6 +774,30 @@ pub(crate) struct CodexCompactionEventRequest { pub(crate) event_params: CodexCompactionEventParams, } +#[derive(Serialize)] +pub(crate) struct CodexGoalEventParams { + pub(crate) thread_id: String, + pub(crate) session_id: String, + pub(crate) turn_id: Option, + pub(crate) app_server_client: CodexAppServerClientMetadata, + pub(crate) runtime: CodexRuntimeMetadata, + pub(crate) thread_source: Option, + pub(crate) subagent_source: Option, + pub(crate) parent_thread_id: Option, + pub(crate) goal_id: String, + pub(crate) event_kind: GoalEventKind, + pub(crate) goal_status: codex_state::ThreadGoalStatus, + pub(crate) has_token_budget: bool, + pub(crate) cumulative_tokens_accounted: Option, + pub(crate) cumulative_time_accounted_seconds: Option, +} + +#[derive(Serialize)] +pub(crate) struct CodexGoalEventRequest { + pub(crate) event_type: &'static str, + pub(crate) event_params: CodexGoalEventParams, +} + #[derive(Serialize)] pub(crate) struct CodexTurnEventParams { pub(crate) thread_id: String, @@ -979,6 +1006,33 @@ pub(crate) fn codex_compaction_event_params( } } +pub(crate) fn codex_goal_event_params( + input: CodexGoalEvent, + session_id: String, + app_server_client: CodexAppServerClientMetadata, + runtime: CodexRuntimeMetadata, + thread_source: Option, + subagent_source: Option, + parent_thread_id: Option, +) -> CodexGoalEventParams { + CodexGoalEventParams { + thread_id: input.thread_id, + session_id, + turn_id: input.turn_id, + app_server_client, + runtime, + thread_source, + subagent_source, + parent_thread_id, + goal_id: input.goal_id, + event_kind: input.event_kind, + goal_status: input.goal_status, + has_token_budget: input.has_token_budget, + cumulative_tokens_accounted: input.cumulative_tokens_accounted, + cumulative_time_accounted_seconds: input.cumulative_time_accounted_seconds, + } +} + pub(crate) fn codex_plugin_used_metadata( tracking: &TrackEventsContext, plugin: PluginTelemetryMetadata, diff --git a/codex-rs/analytics/src/facts.rs b/codex-rs/analytics/src/facts.rs index a035bc98b9..a19fb05d6b 100644 --- a/codex-rs/analytics/src/facts.rs +++ b/codex-rs/analytics/src/facts.rs @@ -417,6 +417,27 @@ pub struct CodexCompactionEvent { pub duration_ms: Option, } +#[derive(Clone, Copy, Debug, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum GoalEventKind { + Created, + UsageAccounted, + StatusChanged, + Cleared, +} + +#[derive(Clone)] +pub struct CodexGoalEvent { + pub thread_id: String, + pub turn_id: Option, + pub goal_id: String, + pub event_kind: GoalEventKind, + pub goal_status: codex_state::ThreadGoalStatus, + pub has_token_budget: bool, + pub cumulative_tokens_accounted: Option, + pub cumulative_time_accounted_seconds: Option, +} + #[allow(dead_code)] pub(crate) enum AnalyticsFact { Initialize { @@ -468,6 +489,7 @@ pub(crate) enum AnalyticsFact { pub(crate) enum CustomAnalyticsFact { SubAgentThreadStarted(SubAgentThreadStartedInput), Compaction(Box), + Goal(Box), GuardianReview(Box), TurnResolvedConfig(Box), TurnTokenUsage(Box), diff --git a/codex-rs/analytics/src/lib.rs b/codex-rs/analytics/src/lib.rs index e2e16dfacd..35bd539cd4 100644 --- a/codex-rs/analytics/src/lib.rs +++ b/codex-rs/analytics/src/lib.rs @@ -24,6 +24,7 @@ pub use facts::AcceptedLineFingerprint; pub use facts::AnalyticsJsonRpcError; pub use facts::AppInvocation; pub use facts::CodexCompactionEvent; +pub use facts::CodexGoalEvent; pub use facts::CodexTurnSteerEvent; pub use facts::CompactionImplementation; pub use facts::CompactionPhase; @@ -31,6 +32,7 @@ pub use facts::CompactionReason; pub use facts::CompactionStatus; pub use facts::CompactionStrategy; pub use facts::CompactionTrigger; +pub use facts::GoalEventKind; pub use facts::HookRunFact; pub use facts::InputError; pub use facts::InvocationType; diff --git a/codex-rs/analytics/src/reducer.rs b/codex-rs/analytics/src/reducer.rs index f8abe36e4e..f0eb0ac9ab 100644 --- a/codex-rs/analytics/src/reducer.rs +++ b/codex-rs/analytics/src/reducer.rs @@ -15,6 +15,7 @@ use crate::events::CodexDynamicToolCallEventParams; use crate::events::CodexDynamicToolCallEventRequest; use crate::events::CodexFileChangeEventParams; use crate::events::CodexFileChangeEventRequest; +use crate::events::CodexGoalEventRequest; use crate::events::CodexHookRunEventRequest; use crate::events::CodexImageGenerationEventParams; use crate::events::CodexImageGenerationEventRequest; @@ -51,6 +52,7 @@ use crate::events::TrackEventRequest; use crate::events::WebSearchActionKind; use crate::events::codex_app_metadata; use crate::events::codex_compaction_event_params; +use crate::events::codex_goal_event_params; use crate::events::codex_hook_run_metadata; use crate::events::codex_plugin_metadata; use crate::events::codex_plugin_used_metadata; @@ -62,6 +64,7 @@ use crate::facts::AnalyticsJsonRpcError; use crate::facts::AppMentionedInput; use crate::facts::AppUsedInput; use crate::facts::CodexCompactionEvent; +use crate::facts::CodexGoalEvent; use crate::facts::CustomAnalyticsFact; use crate::facts::HookRunInput; use crate::facts::PluginState; @@ -192,6 +195,16 @@ impl<'a> AnalyticsDropSite<'a> { } } + fn goal(input: &'a CodexGoalEvent) -> Self { + Self { + event_name: "goal", + thread_id: &input.thread_id, + turn_id: input.turn_id.as_deref(), + review_id: None, + item_id: None, + } + } + fn tool_item( notification: &'a codex_app_server_protocol::ItemCompletedNotification, item_id: &'a str, @@ -461,6 +474,9 @@ impl AnalyticsReducer { CustomAnalyticsFact::Compaction(input) => { self.ingest_compaction(*input, out); } + CustomAnalyticsFact::Goal(input) => { + self.ingest_goal(*input, out); + } CustomAnalyticsFact::GuardianReview(input) => { self.ingest_guardian_review(*input, out); } @@ -1271,6 +1287,26 @@ impl AnalyticsReducer { ))); } + fn ingest_goal(&mut self, input: CodexGoalEvent, out: &mut Vec) { + let Some((connection_state, thread_metadata)) = + self.thread_context_or_warn(AnalyticsDropSite::goal(&input)) + else { + return; + }; + out.push(TrackEventRequest::Goal(Box::new(CodexGoalEventRequest { + event_type: "codex_goal_event", + event_params: codex_goal_event_params( + input, + thread_metadata.session_id.clone(), + connection_state.app_server_client.clone(), + connection_state.runtime.clone(), + thread_metadata.thread_source, + thread_metadata.subagent_source.clone(), + thread_metadata.parent_thread_id.clone(), + ), + }))); + } + fn ingest_guardian_review_completed( &mut self, notification: codex_app_server_protocol::ItemGuardianApprovalReviewCompletedNotification, diff --git a/codex-rs/app-server/src/extensions.rs b/codex-rs/app-server/src/extensions.rs index a0a44fd80d..9e0a6e1c4d 100644 --- a/codex-rs/app-server/src/extensions.rs +++ b/codex-rs/app-server/src/extensions.rs @@ -1,6 +1,7 @@ use std::sync::Arc; use std::sync::Weak; +use codex_analytics::AnalyticsEventsClient; use codex_app_server_protocol::ServerNotification; use codex_app_server_protocol::ThreadGoal; use codex_app_server_protocol::ThreadGoalUpdatedNotification; @@ -30,6 +31,7 @@ pub(crate) fn thread_extensions( event_sink: Arc, auth_manager: Arc, state_db: Option, + analytics_events_client: AnalyticsEventsClient, thread_manager: Weak, goal_service: Arc, executor_skill_provider: Arc, @@ -42,6 +44,7 @@ where codex_goal_extension::install_with_backend( &mut builder, state_db, + analytics_events_client, codex_otel::global(), thread_manager, goal_service, @@ -140,7 +143,6 @@ pub(crate) fn guardian_agent_spawner( mod tests { use std::time::Duration; - use codex_analytics::AnalyticsEventsClient; use codex_protocol::protocol::ThreadGoal as CoreThreadGoal; use codex_protocol::protocol::ThreadGoalStatus; use codex_protocol::protocol::ThreadGoalUpdatedEvent; diff --git a/codex-rs/app-server/src/mcp_refresh.rs b/codex-rs/app-server/src/mcp_refresh.rs index 2cd1027754..ff71048830 100644 --- a/codex-rs/app-server/src/mcp_refresh.rs +++ b/codex-rs/app-server/src/mcp_refresh.rs @@ -196,6 +196,7 @@ mod tests { Arc::new(NoopExtensionEventSink), auth_manager.clone(), Some(state_db.clone()), + codex_analytics::AnalyticsEventsClient::disabled(), thread_manager.clone(), Arc::new(codex_goal_extension::GoalService::new()), Arc::clone(&executor_skill_provider), diff --git a/codex-rs/app-server/src/message_processor.rs b/codex-rs/app-server/src/message_processor.rs index 78ba2c12cf..2069d61c73 100644 --- a/codex-rs/app-server/src/message_processor.rs +++ b/codex-rs/app-server/src/message_processor.rs @@ -327,6 +327,7 @@ impl MessageProcessor { app_server_extension_event_sink(outgoing.clone(), thread_state_manager.clone()), auth_manager.clone(), state_db.clone(), + analytics_events_client.clone(), thread_manager.clone(), Arc::clone(&goal_service), Arc::clone(&executor_skill_provider), diff --git a/codex-rs/app-server/tests/suite/v2/analytics.rs b/codex-rs/app-server/tests/suite/v2/analytics.rs index 8e8f2a7557..ea876940ab 100644 --- a/codex-rs/app-server/tests/suite/v2/analytics.rs +++ b/codex-rs/app-server/tests/suite/v2/analytics.rs @@ -124,6 +124,31 @@ pub(crate) async fn wait_for_analytics_event( server: &MockServer, read_timeout: Duration, event_type: &str, +) -> Result { + wait_for_matching_analytics_event(server, read_timeout, |event| { + event["event_type"] == event_type + }) + .await +} + +pub(crate) async fn wait_for_goal_event( + server: &MockServer, + read_timeout: Duration, + event_kind: &str, + goal_status: &str, +) -> Result { + wait_for_matching_analytics_event(server, read_timeout, |event| { + event["event_type"] == "codex_goal_event" + && event["event_params"]["event_kind"] == event_kind + && event["event_params"]["goal_status"] == goal_status + }) + .await +} + +async fn wait_for_matching_analytics_event( + server: &MockServer, + read_timeout: Duration, + matches: impl Fn(&Value) -> bool, ) -> Result { timeout(read_timeout, async { loop { @@ -142,10 +167,7 @@ pub(crate) async fn wait_for_analytics_event( let Some(events) = payload["events"].as_array() else { continue; }; - if let Some(event) = events - .iter() - .find(|event| event["event_type"] == event_type) - { + if let Some(event) = events.iter().find(|event| matches(event)) { return Ok::(event.clone()); } } diff --git a/codex-rs/app-server/tests/suite/v2/thread_resume.rs b/codex-rs/app-server/tests/suite/v2/thread_resume.rs index 582177ee7d..cf797ff6be 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_resume.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_resume.rs @@ -103,6 +103,7 @@ use super::analytics::assert_basic_thread_initialized_event; use super::analytics::mount_analytics_capture; use super::analytics::thread_initialized_event; use super::analytics::wait_for_analytics_payload; +use super::analytics::wait_for_goal_event; #[cfg(windows)] const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(25); @@ -1316,19 +1317,30 @@ async fn thread_goal_set_edits_objective_without_resetting_usage() -> Result<()> } #[tokio::test] -async fn thread_goal_clear_deletes_goal_and_notifies() -> Result<()> { - let server = create_mock_responses_server_repeating_assistant("Done").await; +async fn thread_goal_lifecycle_emits_analytics_and_clear_deletes_goal() -> Result<()> { + let server = create_mock_responses_server_sequence_unchecked(vec![ + responses::sse(vec![ + responses::ev_response_created("materialize-thread"), + responses::ev_completed("materialize-thread"), + ]), + responses::sse(vec![ + responses::ev_response_created("goal-continuation"), + responses::ev_completed_with_tokens("goal-continuation", /*total_tokens*/ 200), + ]), + ]) + .await; let codex_home = TempDir::new()?; - create_config_toml(codex_home.path(), &server.uri())?; + create_config_toml_with_chatgpt_base_url(codex_home.path(), &server.uri(), &server.uri())?; let config_path = codex_home.path().join("config.toml"); let config = std::fs::read_to_string(&config_path)?; std::fs::write( &config_path, config.replace("personality = true\n", "personality = true\ngoals = true\n"), )?; + mount_analytics_capture(&server, codex_home.path()).await?; let mut mcp = TestAppServer::new_without_managed_config(codex_home.path()).await?; - timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + timeout(DEFAULT_READ_TIMEOUT.saturating_mul(2), mcp.initialize()).await??; let start_id = mcp .send_thread_start_request(ThreadStartParams { @@ -1370,7 +1382,8 @@ async fn thread_goal_clear_deletes_goal_and_notifies() -> Result<()> { "thread/goal/set", Some(json!({ "threadId": thread.id, - "objective": "keep polishing", + "objective": "do not serialize this objective", + "tokenBudget": 100, })), ) .await?; @@ -1386,6 +1399,55 @@ async fn thread_goal_clear_deletes_goal_and_notifies() -> Result<()> { ) .await??; + let created = wait_for_goal_event(&server, DEFAULT_READ_TIMEOUT, "created", "active").await?; + let persisted_goal_id = created["event_params"]["goal_id"] + .as_str() + .expect("created goal id"); + assert_eq!(created["event_params"]["thread_id"], thread.id); + assert_eq!(created["event_params"]["turn_id"], serde_json::Value::Null); + assert_eq!(created["event_params"]["has_token_budget"], true); + assert!(created["event_params"]["session_id"].is_string()); + assert!(created["event_params"]["app_server_client"].is_object()); + assert!(created["event_params"]["runtime"].is_object()); + assert!(created["event_params"].get("objective").is_none()); + assert!(created["event_params"].get("token_budget").is_none()); + + let usage = wait_for_goal_event( + &server, + DEFAULT_READ_TIMEOUT, + "usage_accounted", + "budget_limited", + ) + .await?; + let causal_turn_id = usage["event_params"]["turn_id"] + .as_str() + .expect("accounted usage turn id"); + assert_eq!(usage["event_params"]["goal_id"], persisted_goal_id); + assert_eq!(usage["event_params"]["cumulative_tokens_accounted"], 200); + assert!( + usage["event_params"]["cumulative_time_accounted_seconds"] + .as_i64() + .is_some() + ); + + let status = wait_for_goal_event( + &server, + DEFAULT_READ_TIMEOUT, + "status_changed", + "budget_limited", + ) + .await?; + assert_eq!(status["event_params"]["goal_id"], persisted_goal_id); + assert_eq!(status["event_params"]["turn_id"], causal_turn_id); + assert_eq!( + status["event_params"]["cumulative_tokens_accounted"], + serde_json::Value::Null + ); + assert_eq!( + status["event_params"]["cumulative_time_accounted_seconds"], + serde_json::Value::Null + ); + let clear_id = mcp .send_raw_request( "thread/goal/clear", @@ -1408,6 +1470,11 @@ async fn thread_goal_clear_deletes_goal_and_notifies() -> Result<()> { ) .await??; + let cleared = + wait_for_goal_event(&server, DEFAULT_READ_TIMEOUT, "cleared", "budget_limited").await?; + assert_eq!(cleared["event_params"]["goal_id"], persisted_goal_id); + assert_eq!(cleared["event_params"]["turn_id"], serde_json::Value::Null); + let get_id = mcp .send_raw_request( "thread/goal/get", diff --git a/codex-rs/ext/goal/Cargo.toml b/codex-rs/ext/goal/Cargo.toml index 6116383243..1e21f5ac88 100644 --- a/codex-rs/ext/goal/Cargo.toml +++ b/codex-rs/ext/goal/Cargo.toml @@ -15,6 +15,7 @@ workspace = true [dependencies] async-trait = { workspace = true } +codex-analytics = { workspace = true } codex-core = { workspace = true } codex-extension-api = { workspace = true } codex-otel = { workspace = true } diff --git a/codex-rs/ext/goal/src/analytics.rs b/codex-rs/ext/goal/src/analytics.rs new file mode 100644 index 0000000000..82d34962d1 --- /dev/null +++ b/codex-rs/ext/goal/src/analytics.rs @@ -0,0 +1,77 @@ +use codex_analytics::AnalyticsEventsClient; +use codex_analytics::CodexGoalEvent; +use codex_analytics::GoalEventKind; + +#[derive(Clone)] +pub(crate) struct GoalAnalytics { + client: AnalyticsEventsClient, +} + +pub(crate) enum GoalEventAttribution<'a> { + Turn(&'a str), + NoTurn, +} + +impl GoalAnalytics { + pub(crate) fn new(client: AnalyticsEventsClient) -> Self { + Self { client } + } + + pub(crate) fn created( + &self, + goal: &codex_state::ThreadGoal, + attribution: GoalEventAttribution<'_>, + ) { + self.track(goal, attribution, GoalEventKind::Created); + } + + pub(crate) fn usage_accounted( + &self, + goal: &codex_state::ThreadGoal, + attribution: GoalEventAttribution<'_>, + ) { + self.track(goal, attribution, GoalEventKind::UsageAccounted); + } + + pub(crate) fn status_changed( + &self, + goal: &codex_state::ThreadGoal, + previous_status: Option, + attribution: GoalEventAttribution<'_>, + ) { + if previous_status.is_some_and(|status| status != goal.status) { + self.track(goal, attribution, GoalEventKind::StatusChanged); + } + } + + pub(crate) fn cleared(&self, goal: &codex_state::ThreadGoal) { + self.track(goal, GoalEventAttribution::NoTurn, GoalEventKind::Cleared); + } + + fn track( + &self, + goal: &codex_state::ThreadGoal, + attribution: GoalEventAttribution<'_>, + event_kind: GoalEventKind, + ) { + let (cumulative_tokens_accounted, cumulative_time_accounted_seconds) = match event_kind { + GoalEventKind::UsageAccounted => (Some(goal.tokens_used), Some(goal.time_used_seconds)), + GoalEventKind::Created | GoalEventKind::StatusChanged | GoalEventKind::Cleared => { + (None, None) + } + }; + self.client.track_goal_event(CodexGoalEvent { + thread_id: goal.thread_id.to_string(), + turn_id: match attribution { + GoalEventAttribution::Turn(turn_id) => Some(turn_id.to_string()), + GoalEventAttribution::NoTurn => None, + }, + goal_id: goal.goal_id.clone(), + event_kind, + goal_status: goal.status, + has_token_budget: goal.token_budget.is_some(), + cumulative_tokens_accounted, + cumulative_time_accounted_seconds, + }); + } +} diff --git a/codex-rs/ext/goal/src/api.rs b/codex-rs/ext/goal/src/api.rs index 5123e6a5ce..a4dc93409e 100644 --- a/codex-rs/ext/goal/src/api.rs +++ b/codex-rs/ext/goal/src/api.rs @@ -259,19 +259,19 @@ impl GoalService { tracing::warn!("failed to prepare external goal mutation: {err}"); } - let cleared = state_db + let cleared_goal = state_db .thread_goals() .delete_thread_goal(thread_id) .await .map_err(|err| { GoalServiceError::Internal(format!("failed to clear thread goal: {err}")) })?; + let cleared = cleared_goal.is_some(); drop(goal_state_permit); drop(runtime); - if cleared - && let Some(runtime) = self.runtime_for_thread(thread_id) - && let Err(err) = runtime.apply_external_goal_clear().await + if let (Some(runtime), Some(goal)) = (self.runtime_for_thread(thread_id), cleared_goal) + && let Err(err) = runtime.apply_external_goal_clear(goal).await { tracing::warn!("failed to apply external goal clear runtime effects: {err}"); } diff --git a/codex-rs/ext/goal/src/extension.rs b/codex-rs/ext/goal/src/extension.rs index 78ae0b5b44..0696c2b868 100644 --- a/codex-rs/ext/goal/src/extension.rs +++ b/codex-rs/ext/goal/src/extension.rs @@ -2,6 +2,7 @@ use std::sync::Arc; use std::sync::Weak; use async_trait::async_trait; +use codex_analytics::AnalyticsEventsClient; use codex_core::ThreadManager; use codex_extension_api::ConfigContributor; use codex_extension_api::ExtensionData; @@ -33,6 +34,7 @@ use codex_protocol::protocol::TokenUsageInfo; use crate::accounting::BudgetLimitedGoalDisposition; use crate::accounting::GoalAccountingState; +use crate::analytics::GoalAnalytics; use crate::api::GoalService; use crate::events::GoalEventEmitter; use crate::metrics::GoalMetrics; @@ -57,6 +59,7 @@ impl GoalExtensionConfig { #[derive(Clone)] pub struct GoalExtension { state_dbs: Arc, + analytics: GoalAnalytics, event_emitter: GoalEventEmitter, metrics: GoalMetrics, thread_manager: Weak, @@ -73,6 +76,7 @@ impl std::fmt::Debug for GoalExtension { impl GoalExtension { pub(crate) fn new_with_host_capabilities( state_dbs: Arc, + analytics_events_client: AnalyticsEventsClient, event_sink: Arc, metrics_client: Option, thread_manager: Weak, @@ -81,6 +85,7 @@ impl GoalExtension { ) -> Self { Self { state_dbs, + analytics: GoalAnalytics::new(analytics_events_client), event_emitter: GoalEventEmitter::new(event_sink), metrics: GoalMetrics::new(metrics_client), thread_manager, @@ -120,6 +125,7 @@ where self.thread_manager.clone(), accounting_state, GoalRuntimeConfig { + analytics: self.analytics.clone(), enabled, tools_available_for_thread, }, @@ -403,6 +409,7 @@ where runtime.thread_id(), Arc::clone(&self.state_dbs), runtime.accounting_state(), + self.analytics.clone(), self.event_emitter.clone(), self.metrics.clone(), )), @@ -410,6 +417,7 @@ where runtime.thread_id(), Arc::clone(&self.state_dbs), runtime.accounting_state(), + self.analytics.clone(), self.event_emitter.clone(), self.metrics.clone(), )), @@ -417,6 +425,7 @@ where runtime.thread_id(), Arc::clone(&self.state_dbs), runtime.accounting_state(), + self.analytics.clone(), self.event_emitter.clone(), self.metrics.clone(), )), @@ -427,6 +436,7 @@ where pub fn install_with_backend( registry: &mut ExtensionRegistryBuilder, state_dbs: Arc, + analytics_events_client: AnalyticsEventsClient, metrics_client: Option, thread_manager: Weak, goal_service: Arc, @@ -436,6 +446,7 @@ pub fn install_with_backend( { let extension = Arc::new(GoalExtension::new_with_host_capabilities( state_dbs, + analytics_events_client, registry.event_sink(), metrics_client, thread_manager, diff --git a/codex-rs/ext/goal/src/lib.rs b/codex-rs/ext/goal/src/lib.rs index bb640c6ce8..ccd091affc 100644 --- a/codex-rs/ext/goal/src/lib.rs +++ b/codex-rs/ext/goal/src/lib.rs @@ -1,6 +1,7 @@ //! Extension crate for the `/goal` feature. mod accounting; +mod analytics; mod api; mod events; mod extension; diff --git a/codex-rs/ext/goal/src/runtime.rs b/codex-rs/ext/goal/src/runtime.rs index 2641dfb949..2f2202cbb8 100644 --- a/codex-rs/ext/goal/src/runtime.rs +++ b/codex-rs/ext/goal/src/runtime.rs @@ -10,6 +10,8 @@ use codex_protocol::protocol::ThreadGoal; use crate::accounting::BudgetLimitedGoalDisposition; use crate::accounting::GoalAccountingState; +use crate::analytics::GoalAnalytics; +use crate::analytics::GoalEventAttribution; use crate::events::GoalEventEmitter; use crate::metrics::GoalMetrics; use crate::steering::continuation_steering_item; @@ -24,6 +26,7 @@ pub struct GoalRuntimeHandle { } pub(crate) struct GoalRuntimeConfig { + pub(crate) analytics: GoalAnalytics, pub(crate) enabled: bool, pub(crate) tools_available_for_thread: bool, } @@ -36,6 +39,7 @@ pub(crate) enum ActiveGoalStopReason { struct GoalRuntimeInner { thread_id: ThreadId, state_dbs: Arc, + analytics: GoalAnalytics, event_emitter: GoalEventEmitter, metrics: GoalMetrics, thread_manager: Weak, @@ -87,6 +91,7 @@ impl GoalRuntimeHandle { inner: Arc::new(GoalRuntimeInner { thread_id, state_dbs, + analytics: config.analytics, event_emitter, metrics, thread_manager, @@ -165,6 +170,9 @@ impl GoalRuntimeHandle { .is_some_and(|previous_goal| previous_goal.goal_id != goal.goal_id); if previous_goal.is_none() || replaced_existing_goal { self.inner.metrics.record_created(); + self.inner + .analytics + .created(&goal, GoalEventAttribution::NoTurn); } let previous_status = previous_goal .as_ref() @@ -175,6 +183,9 @@ impl GoalRuntimeHandle { self.inner .metrics .record_terminal_if_status_changed(previous_status, &goal); + self.inner + .analytics + .status_changed(&goal, previous_status, GoalEventAttribution::NoTurn); let objective_changed = previous_goal.as_ref().is_some_and(|previous_goal| { !replaced_existing_goal && previous_goal.objective != goal.objective }); @@ -211,11 +222,15 @@ impl GoalRuntimeHandle { Ok(()) } - pub async fn apply_external_goal_clear(&self) -> Result<(), String> { + pub async fn apply_external_goal_clear( + &self, + goal: codex_state::ThreadGoal, + ) -> Result<(), String> { if !self.is_enabled() { return Ok(()); } + self.inner.analytics.cleared(&goal); self.inner.accounting_state.clear_active_goal(); Ok(()) } @@ -302,6 +317,11 @@ impl GoalRuntimeHandle { self.inner .metrics .record_terminal_if_status_changed(previous_status, &goal); + self.inner.analytics.status_changed( + &goal, + previous_status, + GoalEventAttribution::Turn(turn_id), + ); self.inner.accounting_state.clear_active_goal(); let goal = protocol_goal_from_state(goal); self.inner.event_emitter.thread_goal_updated( @@ -445,6 +465,14 @@ impl GoalRuntimeHandle { self.inner .metrics .record_terminal_if_status_changed(previous_status, &goal); + self.inner + .analytics + .usage_accounted(&goal, GoalEventAttribution::Turn(turn_id)); + self.inner.analytics.status_changed( + &goal, + previous_status, + GoalEventAttribution::Turn(turn_id), + ); accounting.mark_progress_accounted_for_status( turn_id, &snapshot, @@ -499,6 +527,14 @@ impl GoalRuntimeHandle { self.inner .metrics .record_terminal_if_status_changed(previous_status, &goal); + self.inner + .analytics + .usage_accounted(&goal, GoalEventAttribution::NoTurn); + self.inner.analytics.status_changed( + &goal, + previous_status, + GoalEventAttribution::NoTurn, + ); accounting.mark_idle_progress_accounted_for_status( &snapshot, goal.status, diff --git a/codex-rs/ext/goal/src/tool.rs b/codex-rs/ext/goal/src/tool.rs index 8c0f513004..c45ed81b0a 100644 --- a/codex-rs/ext/goal/src/tool.rs +++ b/codex-rs/ext/goal/src/tool.rs @@ -17,6 +17,8 @@ use serde::Serialize; use crate::accounting::BudgetLimitedGoalDisposition; use crate::accounting::GoalAccountingState; +use crate::analytics::GoalAnalytics; +use crate::analytics::GoalEventAttribution; use crate::events::GoalEventEmitter; use crate::metrics::GoalMetrics; use crate::spec::CREATE_GOAL_TOOL_NAME; @@ -32,6 +34,7 @@ pub(crate) struct GoalToolExecutor { thread_id: ThreadId, state_db: Arc, accounting_state: Arc, + analytics: GoalAnalytics, event_emitter: GoalEventEmitter, metrics: GoalMetrics, } @@ -75,6 +78,7 @@ impl GoalToolExecutor { thread_id: ThreadId, state_db: Arc, accounting_state: Arc, + analytics: GoalAnalytics, event_emitter: GoalEventEmitter, metrics: GoalMetrics, ) -> Self { @@ -83,6 +87,7 @@ impl GoalToolExecutor { thread_id, state_db, accounting_state, + analytics, event_emitter, metrics, } @@ -92,6 +97,7 @@ impl GoalToolExecutor { thread_id: ThreadId, state_db: Arc, accounting_state: Arc, + analytics: GoalAnalytics, event_emitter: GoalEventEmitter, metrics: GoalMetrics, ) -> Self { @@ -100,6 +106,7 @@ impl GoalToolExecutor { thread_id, state_db, accounting_state, + analytics, event_emitter, metrics, } @@ -109,6 +116,7 @@ impl GoalToolExecutor { thread_id: ThreadId, state_db: Arc, accounting_state: Arc, + analytics: GoalAnalytics, event_emitter: GoalEventEmitter, metrics: GoalMetrics, ) -> Self { @@ -117,6 +125,7 @@ impl GoalToolExecutor { thread_id, state_db, accounting_state, + analytics, event_emitter, metrics, } @@ -200,6 +209,10 @@ impl GoalToolExecutor { .accounting_state .mark_current_turn_goal_active(goal.goal_id.clone()); self.metrics.record_created(); + self.analytics.created( + &goal, + GoalEventAttribution::Turn(invocation.turn_id.as_str()), + ); let goal = protocol_goal_from_state(goal); self.emit_goal_updated_from_tool_call(&invocation, turn_id, goal.clone()); goal_response(Some(goal), CompletionBudgetReport::Omit) @@ -259,6 +272,11 @@ impl GoalToolExecutor { })?; self.metrics .record_terminal_if_status_changed(previous_status, &goal); + self.analytics.status_changed( + &goal, + previous_status, + GoalEventAttribution::Turn(invocation.turn_id.as_str()), + ); let goal = protocol_goal_from_state(goal); let turn_id = self.accounting_state.clear_current_turn_goal(); self.emit_goal_updated_from_tool_call(&invocation, turn_id, goal.clone()); @@ -324,6 +342,13 @@ impl GoalToolExecutor { codex_state::GoalAccountingOutcome::Updated(goal) => { self.metrics .record_terminal_if_status_changed(previous_status, &goal); + self.analytics + .usage_accounted(&goal, GoalEventAttribution::Turn(turn_id.as_str())); + self.analytics.status_changed( + &goal, + previous_status, + GoalEventAttribution::Turn(turn_id.as_str()), + ); self.accounting_state.mark_progress_accounted_for_status( turn_id.as_str(), &snapshot, diff --git a/codex-rs/ext/goal/tests/goal_extension_backend.rs b/codex-rs/ext/goal/tests/goal_extension_backend.rs index b922f5332d..60c992ec78 100644 --- a/codex-rs/ext/goal/tests/goal_extension_backend.rs +++ b/codex-rs/ext/goal/tests/goal_extension_backend.rs @@ -4,6 +4,7 @@ use std::sync::PoisonError; use std::sync::Weak; use std::time::Duration; +use codex_analytics::AnalyticsEventsClient; use codex_extension_api::ExtensionData; use codex_extension_api::ExtensionEventSink; use codex_extension_api::ExtensionRegistryBuilder; @@ -1114,6 +1115,7 @@ async fn installed_tools_with_start( install_with_backend( &mut builder, runtime, + AnalyticsEventsClient::disabled(), /*metrics_client*/ None, Weak::new(), goal_service, @@ -1164,6 +1166,7 @@ impl GoalExtensionHarness { install_with_backend( &mut builder, runtime, + AnalyticsEventsClient::disabled(), /*metrics_client*/ None, Weak::new(), Arc::clone(&goal_service), diff --git a/codex-rs/state/src/model/thread_goal.rs b/codex-rs/state/src/model/thread_goal.rs index 4806250069..12d2a78dac 100644 --- a/codex-rs/state/src/model/thread_goal.rs +++ b/codex-rs/state/src/model/thread_goal.rs @@ -3,12 +3,14 @@ use anyhow::anyhow; use chrono::DateTime; use chrono::Utc; use codex_protocol::ThreadId; +use serde::Serialize; use sqlx::Row; use sqlx::sqlite::SqliteRow; use super::epoch_millis_to_datetime; -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] pub enum ThreadGoalStatus { Active, Paused, diff --git a/codex-rs/state/src/runtime/goals.rs b/codex-rs/state/src/runtime/goals.rs index 50e8cf068e..d0fab47d05 100644 --- a/codex-rs/state/src/runtime/goals.rs +++ b/codex-rs/state/src/runtime/goals.rs @@ -377,18 +377,31 @@ WHERE thread_id = ? self.get_thread_goal(thread_id).await } - pub async fn delete_thread_goal(&self, thread_id: ThreadId) -> anyhow::Result { - let result = sqlx::query( + pub async fn delete_thread_goal( + &self, + thread_id: ThreadId, + ) -> anyhow::Result> { + let row = sqlx::query( r#" DELETE FROM thread_goals WHERE thread_id = ? +RETURNING + thread_id, + goal_id, + objective, + status, + token_budget, + tokens_used, + time_used_seconds, + created_at_ms, + updated_at_ms "#, ) .bind(thread_id.to_string()) - .execute(self.pool.as_ref()) + .fetch_optional(self.pool.as_ref()) .await?; - Ok(result.rows_affected() > 0) + row.map(|row| thread_goal_from_row(&row)).transpose() } pub async fn account_thread_goal_usage( @@ -622,7 +635,8 @@ mod tests { assert_eq!(0, replaced.tokens_used); assert_eq!(0, replaced.time_used_seconds); - assert!( + assert_eq!( + Some(replaced), runtime .thread_goals() .delete_thread_goal(thread_id) @@ -637,8 +651,9 @@ mod tests { .await .unwrap() ); - assert!( - !runtime + assert_eq!( + None, + runtime .thread_goals() .delete_thread_goal(thread_id) .await diff --git a/codex-rs/state/src/runtime/threads.rs b/codex-rs/state/src/runtime/threads.rs index 6f3c45f0e2..2c00500ca3 100644 --- a/codex-rs/state/src/runtime/threads.rs +++ b/codex-rs/state/src/runtime/threads.rs @@ -891,7 +891,7 @@ ON CONFLICT(id) DO UPDATE SET let rows_affected = result.rows_affected(); self.memories.delete_thread_memory(thread_id).await?; if rows_affected > 0 { - self.thread_goals.delete_thread_goal(thread_id).await?; + let _ = self.thread_goals.delete_thread_goal(thread_id).await?; } Ok(rows_affected) } From e0cb4ede4e44a371d595520b29d0c80336b8733e Mon Sep 17 00:00:00 2001 From: "Adam Perry @ OpenAI" Date: Tue, 9 Jun 2026 20:52:09 -0700 Subject: [PATCH 24/47] [codex] Fix post-merge analytics integration failures (#27285) ## Why Recent merges left `main` with analytics integration build failures. Local Cargo runs also made the trimmed-skills test depend on developer-installed skills, while Bazel used an isolated home. ## What changed - Clone `thread_metadata.thread_source` when constructing goal analytics event parameters. - Group app-server thread extension inputs into `ThreadExtensionDependencies`. - Isolate the trimmed-skills test home so its exact fixture count is stable across Cargo and Bazel. ## Validation - `cargo check -p codex-analytics` - `just test -p codex-analytics` (71 tests) - `just test -p codex-app-server` (837 tests; one unrelated zsh-fork timeout passed on retry) --- codex-rs/analytics/src/reducer.rs | 2 +- codex-rs/app-server/src/extensions.rs | 27 ++++++++++++++----- codex-rs/app-server/src/mcp_refresh.rs | 17 +++++++----- codex-rs/app-server/src/message_processor.rs | 20 +++++++++----- .../app-server/tests/suite/v2/turn_start.rs | 10 ++++++- 5 files changed, 53 insertions(+), 23 deletions(-) diff --git a/codex-rs/analytics/src/reducer.rs b/codex-rs/analytics/src/reducer.rs index f0eb0ac9ab..1be9f624f1 100644 --- a/codex-rs/analytics/src/reducer.rs +++ b/codex-rs/analytics/src/reducer.rs @@ -1300,7 +1300,7 @@ impl AnalyticsReducer { thread_metadata.session_id.clone(), connection_state.app_server_client.clone(), connection_state.runtime.clone(), - thread_metadata.thread_source, + thread_metadata.thread_source.clone(), thread_metadata.subagent_source.clone(), thread_metadata.parent_thread_id.clone(), ), diff --git a/codex-rs/app-server/src/extensions.rs b/codex-rs/app-server/src/extensions.rs index 9e0a6e1c4d..4916132c58 100644 --- a/codex-rs/app-server/src/extensions.rs +++ b/codex-rs/app-server/src/extensions.rs @@ -26,19 +26,32 @@ use crate::outgoing_message::OutgoingMessageSender; use crate::thread_state::ThreadListenerCommand; use crate::thread_state::ThreadStateManager; +pub(crate) struct ThreadExtensionDependencies { + pub(crate) event_sink: Arc, + pub(crate) auth_manager: Arc, + pub(crate) state_db: Option, + pub(crate) analytics_events_client: AnalyticsEventsClient, + pub(crate) thread_manager: Weak, + pub(crate) goal_service: Arc, + pub(crate) executor_skill_provider: Arc, +} + pub(crate) fn thread_extensions( guardian_agent_spawner: S, - event_sink: Arc, - auth_manager: Arc, - state_db: Option, - analytics_events_client: AnalyticsEventsClient, - thread_manager: Weak, - goal_service: Arc, - executor_skill_provider: Arc, + dependencies: ThreadExtensionDependencies, ) -> Arc> where S: AgentSpawner + 'static, { + let ThreadExtensionDependencies { + event_sink, + auth_manager, + state_db, + analytics_events_client, + thread_manager, + goal_service, + executor_skill_provider, + } = dependencies; let mut builder = ExtensionRegistryBuilder::::with_event_sink(event_sink); if let Some(state_db) = state_db { codex_goal_extension::install_with_backend( diff --git a/codex-rs/app-server/src/mcp_refresh.rs b/codex-rs/app-server/src/mcp_refresh.rs index ff71048830..b5431228d4 100644 --- a/codex-rs/app-server/src/mcp_refresh.rs +++ b/codex-rs/app-server/src/mcp_refresh.rs @@ -96,6 +96,7 @@ async fn queue_refresh( #[cfg(test)] mod tests { use super::*; + use crate::extensions::ThreadExtensionDependencies; use crate::extensions::guardian_agent_spawner; use crate::extensions::thread_extensions; use async_trait::async_trait; @@ -193,13 +194,15 @@ mod tests { Arc::clone(&environment_manager), thread_extensions( guardian_agent_spawner(thread_manager.clone()), - Arc::new(NoopExtensionEventSink), - auth_manager.clone(), - Some(state_db.clone()), - codex_analytics::AnalyticsEventsClient::disabled(), - thread_manager.clone(), - Arc::new(codex_goal_extension::GoalService::new()), - Arc::clone(&executor_skill_provider), + ThreadExtensionDependencies { + event_sink: Arc::new(NoopExtensionEventSink), + auth_manager: auth_manager.clone(), + state_db: Some(state_db.clone()), + analytics_events_client: codex_analytics::AnalyticsEventsClient::disabled(), + thread_manager: thread_manager.clone(), + goal_service: Arc::new(codex_goal_extension::GoalService::new()), + executor_skill_provider: Arc::clone(&executor_skill_provider), + }, ), /*analytics_events_client*/ None, Arc::clone(&thread_store), diff --git a/codex-rs/app-server/src/message_processor.rs b/codex-rs/app-server/src/message_processor.rs index 2069d61c73..6bb5121dc9 100644 --- a/codex-rs/app-server/src/message_processor.rs +++ b/codex-rs/app-server/src/message_processor.rs @@ -8,6 +8,7 @@ use crate::attestation::app_server_attestation_provider; use crate::config_manager::ConfigManager; use crate::connection_rpc_gate::ConnectionRpcGate; use crate::error_code::invalid_request; +use crate::extensions::ThreadExtensionDependencies; use crate::extensions::app_server_extension_event_sink; use crate::extensions::guardian_agent_spawner; use crate::extensions::thread_extensions; @@ -324,13 +325,18 @@ impl MessageProcessor { environment_manager, thread_extensions( guardian_agent_spawner(thread_manager.clone()), - app_server_extension_event_sink(outgoing.clone(), thread_state_manager.clone()), - auth_manager.clone(), - state_db.clone(), - analytics_events_client.clone(), - thread_manager.clone(), - Arc::clone(&goal_service), - Arc::clone(&executor_skill_provider), + ThreadExtensionDependencies { + event_sink: app_server_extension_event_sink( + outgoing.clone(), + thread_state_manager.clone(), + ), + auth_manager: auth_manager.clone(), + state_db: state_db.clone(), + analytics_events_client: analytics_events_client.clone(), + thread_manager: thread_manager.clone(), + goal_service: Arc::clone(&goal_service), + executor_skill_provider: Arc::clone(&executor_skill_provider), + }, ), Some(analytics_events_client.clone()), Arc::clone(&thread_store), diff --git a/codex-rs/app-server/tests/suite/v2/turn_start.rs b/codex-rs/app-server/tests/suite/v2/turn_start.rs index 5080937997..823851b715 100644 --- a/codex-rs/app-server/tests/suite/v2/turn_start.rs +++ b/codex-rs/app-server/tests/suite/v2/turn_start.rs @@ -591,7 +591,15 @@ async fn turn_start_emits_thread_scoped_warning_notification_for_trimmed_skills( write_test_skill(codex_home.path(), "alpha-skill")?; write_test_skill(codex_home.path(), "beta-skill")?; - let mut mcp = TestAppServer::new(codex_home.path()).await?; + let isolated_home = codex_home.path().to_string_lossy(); + let mut mcp = TestAppServer::new_with_env( + codex_home.path(), + &[ + ("HOME", Some(isolated_home.as_ref())), + ("USERPROFILE", Some(isolated_home.as_ref())), + ], + ) + .await?; timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; let thread_req = mcp From 00a25e1e0c6eecf076dcb989f4065c578c262ae7 Mon Sep 17 00:00:00 2001 From: mchen-oai Date: Tue, 9 Jun 2026 21:41:06 -0700 Subject: [PATCH 25/47] Add spans to run_turn (#27107) ## Why Codex app-server latency traces do not granularly cover turn orchestration, sampling-request preparation, and tool-loading work. These spans help separate local coordination/setup costs from model streaming and tool execution. ## What changed - Add `run_turn.*` spans around sampling-request input preparation and post-sampling state collection - Add function-level trace spans around turn setup, hook execution, compaction, prompt construction, and MCP tool exposure - Add `built_tools.*` spans around plugin loading and discoverable-tool loading ## Verification Trigger Codex rollout and observe new spans are included --- codex-rs/core/src/hook_runtime.rs | 4 ++ codex-rs/core/src/mcp_tool_exposure.rs | 2 + codex-rs/core/src/session/mod.rs | 1 + codex-rs/core/src/session/turn.rs | 89 ++++++++++++++++---------- 4 files changed, 62 insertions(+), 34 deletions(-) diff --git a/codex-rs/core/src/hook_runtime.rs b/codex-rs/core/src/hook_runtime.rs index 6e2d4fe33c..8501c6259c 100644 --- a/codex-rs/core/src/hook_runtime.rs +++ b/codex-rs/core/src/hook_runtime.rs @@ -36,6 +36,7 @@ use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::SubAgentSource; use codex_thread_store::ReadThreadParams; use serde_json::Value; +use tracing::instrument; use crate::context::ContextualUserFragment; use crate::context::HookAdditionalContext; @@ -97,6 +98,7 @@ impl From for ContextInjectingHookOutcome { } } +#[instrument(level = "trace", skip_all)] pub(crate) async fn run_pending_session_start_hooks( sess: &Arc, turn_context: &Arc, @@ -291,6 +293,7 @@ pub(crate) async fn run_post_tool_use_hooks( outcome } +#[instrument(level = "trace", skip_all)] pub(crate) async fn run_turn_stop_hooks( sess: &Arc, turn_context: &Arc, @@ -427,6 +430,7 @@ pub(crate) async fn run_post_compact_hooks( } } +#[instrument(level = "trace", skip_all)] pub(crate) async fn run_legacy_after_agent_hook( sess: &Arc, turn_context: &Arc, diff --git a/codex-rs/core/src/mcp_tool_exposure.rs b/codex-rs/core/src/mcp_tool_exposure.rs index 5dc66af734..d430851570 100644 --- a/codex-rs/core/src/mcp_tool_exposure.rs +++ b/codex-rs/core/src/mcp_tool_exposure.rs @@ -4,6 +4,7 @@ use codex_features::Feature; use codex_mcp::CODEX_APPS_MCP_SERVER_NAME; use codex_mcp::ToolInfo as McpToolInfo; use codex_mcp::tool_is_model_visible; +use tracing::instrument; use crate::config::Config; use crate::connectors; @@ -15,6 +16,7 @@ pub(crate) struct McpToolExposure { pub(crate) deferred_tools: Option>, } +#[instrument(level = "trace", skip_all)] pub(crate) fn build_mcp_tool_exposure( all_mcp_tools: &[McpToolInfo], connectors: Option<&[connectors::AppInfo]>, diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs index 3e71afee8a..2e5d5643da 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -3011,6 +3011,7 @@ impl Session { /// Mid-turn compaction is the other path that can re-establish that baseline when it /// reinjects full initial context into replacement history. Other non-regular tasks /// intentionally do not update the baseline. + #[instrument(level = "trace", skip_all)] pub(crate) async fn record_context_updates_and_set_reference_context_item( &self, turn_context: &TurnContext, diff --git a/codex-rs/core/src/session/turn.rs b/codex-rs/core/src/session/turn.rs index 959e28834a..95d24dc03d 100644 --- a/codex-rs/core/src/session/turn.rs +++ b/codex-rs/core/src/session/turn.rs @@ -186,10 +186,9 @@ pub(crate) async fn run_turn( let mut stop_hook_active = false; // Although from the perspective of codex.rs, TurnDiffTracker has the lifecycle of a Task which contains // many turns, from the perspective of the user, it is a single turn. + let display_roots = turn_diff_display_roots(turn_context.as_ref()).await; let turn_diff_tracker = Arc::new(tokio::sync::Mutex::new( - TurnDiffTracker::with_environment_display_roots( - turn_diff_display_roots(turn_context.as_ref()).await, - ), + TurnDiffTracker::with_environment_display_roots(display_roots), )); // `ModelClientSession` is turn-scoped and caches WebSocket + sticky routing state, so we reuse @@ -214,11 +213,13 @@ pub(crate) async fn run_turn( } // Construct the input that we will send to the model. - let sampling_request_input: Vec = { + let sampling_request_input: Vec = async { sess.clone_history() .await .for_prompt(&turn_context.model_info.input_modalities) - }; + } + .instrument(trace_span!("run_turn.prepare_sampling_request_input")) + .await; let window_id = sess.services.model_client.current_window_id(); let turn_metadata_header = turn_context @@ -242,15 +243,20 @@ pub(crate) async fn run_turn( last_agent_message: sampling_request_last_agent_message, } = sampling_request_output; can_drain_pending_input = true; - let has_pending_input = sess.input_queue.has_pending_input(&sess.active_turn).await; + let (has_pending_input, token_status, estimated_token_count) = async { + let has_pending_input = + sess.input_queue.has_pending_input(&sess.active_turn).await; + let token_status = + auto_compact_token_status(sess.as_ref(), turn_context.as_ref()).await; + let estimated_token_count = + sess.get_estimated_token_count(turn_context.as_ref()).await; + (has_pending_input, token_status, estimated_token_count) + } + .instrument(trace_span!("run_turn.collect_post_sampling_state")) + .await; let needs_follow_up = model_needs_follow_up || has_pending_input; - let token_status = - auto_compact_token_status(sess.as_ref(), turn_context.as_ref()).await; let token_limit_reached = token_status.token_limit_reached; - let estimated_token_count = - sess.get_estimated_token_count(turn_context.as_ref()).await; - trace!( turn_id = %turn_context.sub_id, total_usage_tokens = token_status.active_context_tokens, @@ -381,6 +387,7 @@ pub(crate) async fn run_turn( last_agent_message } +#[instrument(level = "trace", skip_all)] async fn turn_diff_display_roots(turn_context: &TurnContext) -> Vec<(String, PathBuf)> { let mut display_roots = Vec::new(); for turn_environment in &turn_context.environments.turn_environments { @@ -396,6 +403,7 @@ async fn turn_diff_display_roots(turn_context: &TurnContext) -> Vec<(String, Pat display_roots } +#[instrument(level = "trace", skip_all)] async fn run_hooks_and_record_inputs( sess: &Arc, turn_context: &Arc, @@ -428,6 +436,7 @@ async fn run_hooks_and_record_inputs( clippy::await_holding_invalid_type, reason = "MCP tool listing borrows the read guard across cancellation-aware await" )] +#[instrument(level = "trace", skip_all)] async fn build_skills_and_plugins( sess: &Arc, turn_context: &TurnContext, @@ -756,6 +765,7 @@ async fn auto_compact_token_status( } } +#[instrument(level = "trace", skip_all)] async fn run_pre_sampling_compact( sess: &Arc, turn_context: &Arc, @@ -834,6 +844,11 @@ async fn maybe_run_previous_model_inline_compact( Ok(()) } +#[instrument( + level = "trace", + skip_all, + fields(reason = ?reason, phase = ?phase) +)] async fn run_auto_compact( sess: &Arc, turn_context: &Arc, @@ -942,6 +957,7 @@ pub(super) fn collect_explicit_app_ids_from_skill_items( connector_ids } +#[instrument(level = "trace", skip_all)] pub(crate) fn build_prompt( input: Vec, router: &ToolRouter, @@ -1095,6 +1111,7 @@ pub(crate) async fn built_tools( .services .plugins_manager .plugins_for_config(&turn_context.config.plugins_config_input()) + .instrument(trace_span!("built_tools.load_plugins")) .await; let apps_enabled = turn_context.apps_enabled(); @@ -1125,35 +1142,39 @@ pub(crate) async fn built_tools( .into_iter() .map(|connector_id| connector_id.0) .collect::>(); - let discoverable_tools = if apps_enabled && tool_suggest_enabled(turn_context) { - if let Some(accessible_connectors) = accessible_connectors_with_enabled_state.as_ref() { - match connectors::list_tool_suggest_discoverable_tools_with_auth( - &turn_context.config, - sess.services.plugins_manager.as_ref(), - auth.as_ref(), - accessible_connectors.as_slice(), - &loaded_plugin_app_connector_ids, - ) - .await - .map(|discoverable_tools| { - filter_request_plugin_install_discoverable_tools_for_client( - discoverable_tools, - turn_context.app_server_client_name.as_deref(), + let discoverable_tools = async { + if apps_enabled && tool_suggest_enabled(turn_context) { + if let Some(accessible_connectors) = accessible_connectors_with_enabled_state.as_ref() { + match connectors::list_tool_suggest_discoverable_tools_with_auth( + &turn_context.config, + sess.services.plugins_manager.as_ref(), + auth.as_ref(), + accessible_connectors.as_slice(), + &loaded_plugin_app_connector_ids, ) - }) { - Ok(discoverable_tools) if discoverable_tools.is_empty() => None, - Ok(discoverable_tools) => Some(discoverable_tools), - Err(err) => { - warn!("failed to load discoverable tool suggestions: {err:#}"); - None + .await + .map(|discoverable_tools| { + filter_request_plugin_install_discoverable_tools_for_client( + discoverable_tools, + turn_context.app_server_client_name.as_deref(), + ) + }) { + Ok(discoverable_tools) if discoverable_tools.is_empty() => None, + Ok(discoverable_tools) => Some(discoverable_tools), + Err(err) => { + warn!("failed to load discoverable tool suggestions: {err:#}"); + None + } } + } else { + None } } else { None } - } else { - None - }; + } + .instrument(trace_span!("built_tools.load_discoverable_tools")) + .await; let mcp_tool_exposure = build_mcp_tool_exposure( &all_mcp_tools, From a7b6baecc63519fd1d7b7be4f779217fe6c82063 Mon Sep 17 00:00:00 2001 From: Ahmed Ibrahim Date: Wed, 10 Jun 2026 00:17:58 -0700 Subject: [PATCH 26/47] [codex] Make MCP connection startup fallible (#27261) ## Why Required MCP server startup was enforced in `Session::new` after `McpConnectionManager` had already created the clients. That split let other manager construction paths bypass the same requirement and exposed manager internals solely so the session could validate them. Keeping required-server readiness in the constructor gives every caller one consistent startup contract. ## What changed - make `McpConnectionManager::new` return `anyhow::Result` and fail when an enabled, required server cannot initialize - pass the startup cancellation token into the constructor so required-server waits remain cancellable - propagate constructor failures through resource reads, connector discovery, and MCP status collection - preserve the active manager and cancellation token when a refreshed replacement fails - keep required-startup failure collection private and cover the constructor error contract directly ## Validation - updated the focused connection-manager test to assert the complete required-server startup error - local tests not run; relying on CI --- codex-rs/codex-mcp/src/connection_manager.rs | 93 ++++++++++++------- .../codex-mcp/src/connection_manager_tests.rs | 21 +++-- codex-rs/codex-mcp/src/mcp/mod.rs | 9 +- codex-rs/core/src/connectors.rs | 5 +- codex-rs/core/src/mcp_tool_call_tests.rs | 4 +- codex-rs/core/src/session/mcp.rs | 19 ++-- codex-rs/core/src/session/session.rs | 64 +++---------- codex-rs/core/src/state/service.rs | 21 +++++ 8 files changed, 129 insertions(+), 107 deletions(-) diff --git a/codex-rs/codex-mcp/src/connection_manager.rs b/codex-rs/codex-mcp/src/connection_manager.rs index 5558916e5b..5b7b6b3e81 100644 --- a/codex-rs/codex-mcp/src/connection_manager.rs +++ b/codex-rs/codex-mcp/src/connection_manager.rs @@ -68,6 +68,7 @@ use serde_json::Value as JsonValue; use tokio::task::JoinSet; use tokio_util::sync::CancellationToken; use tracing::Instrument; +use tracing::info_span; use tracing::instrument; use tracing::trace; use tracing::trace_span; @@ -105,6 +106,7 @@ pub fn tool_is_model_visible(tool: &ToolInfo) -> bool { pub struct McpConnectionManager { clients: HashMap, server_metadata: HashMap, + required_servers: Vec, tool_plugin_provenance: Arc, host_owned_codex_apps_enabled: bool, prefix_mcp_tool_names: bool, @@ -113,7 +115,7 @@ pub struct McpConnectionManager { } impl McpConnectionManager { - #[allow(clippy::new_ret_no_self, clippy::too_many_arguments)] + #[allow(clippy::too_many_arguments)] pub async fn new( mcp_servers: &HashMap, store_mode: OAuthCredentialsStoreMode, @@ -121,6 +123,7 @@ impl McpConnectionManager { approval_policy: &Constrained, submit_id: String, tx_event: Sender, + startup_cancellation_token: CancellationToken, initial_permission_profile: PermissionProfile, runtime_context: McpRuntimeContext, codex_home: PathBuf, @@ -131,8 +134,13 @@ impl McpConnectionManager { tool_plugin_provenance: ToolPluginProvenance, auth: Option<&CodexAuth>, elicitation_reviewer: Option, - ) -> (Self, CancellationToken) { - let cancel_token = CancellationToken::new(); + ) -> Self { + let mut required_servers = mcp_servers + .iter() + .filter(|(_, server)| server.enabled() && server.required()) + .map(|(name, _)| name.clone()) + .collect::>(); + required_servers.sort(); let mut clients = HashMap::new(); let mut server_metadata = HashMap::new(); let mut join_set = JoinSet::new(); @@ -152,7 +160,7 @@ impl McpConnectionManager { .filter(|(_, server)| server.enabled()) { server_metadata.insert(server_name.clone(), McpServerMetadata::from(&server)); - let cancel_token = cancel_token.child_token(); + let cancel_token = startup_cancellation_token.child_token(); let _ = emit_update( startup_submit_id.as_str(), &tx_event, @@ -237,11 +245,12 @@ impl McpConnectionManager { let manager = Self { clients, server_metadata, + required_servers, tool_plugin_provenance, host_owned_codex_apps_enabled, prefix_mcp_tool_names, elicitation_requests: elicitation_requests.clone(), - startup_cancellation_token: cancel_token.clone(), + startup_cancellation_token: startup_cancellation_token.clone(), }; tokio::spawn(async move { let outcomes = join_set.join_all().await; @@ -265,7 +274,53 @@ impl McpConnectionManager { }) .await; }); - (manager, cancel_token) + manager + } + + /// Waits for every required server and reports their startup failures together. + /// + /// Callers must make the manager reachable to request handlers before awaiting this method, + /// because server initialization may require client elicitation. + pub async fn validate_required_servers(&self) -> Result<()> { + let failures = async { + let mut failures = Vec::new(); + for server_name in &self.required_servers { + let Some(async_managed_client) = self.clients.get(server_name).cloned() else { + failures.push(McpStartupFailure { + server: server_name.clone(), + error: format!("required MCP server `{server_name}` was not initialized"), + }); + continue; + }; + + match async_managed_client.client().await { + Ok(_) => {} + Err(error) => failures.push(McpStartupFailure { + server: server_name.clone(), + error: startup_outcome_error_message(error), + }), + } + } + failures + } + .instrument(info_span!( + "session_init.required_mcp_wait", + otel.name = "session_init.required_mcp_wait", + session_init.required_mcp_server_count = self.required_servers.len(), + )) + .await; + if failures.is_empty() { + return Ok(()); + } + + let details = failures + .iter() + .map(|failure| format!("{}: {}", failure.server, failure.error)) + .collect::>() + .join("; "); + Err(anyhow!( + "required MCP servers failed to initialize: {details}" + )) } pub fn new_uninitialized_with_permission_profile( @@ -276,6 +331,7 @@ impl McpConnectionManager { Self { clients: HashMap::new(), server_metadata: HashMap::new(), + required_servers: Vec::new(), tool_plugin_provenance: Arc::new(ToolPluginProvenance::default()), host_owned_codex_apps_enabled: false, prefix_mcp_tool_names, @@ -374,31 +430,6 @@ impl McpConnectionManager { } } - pub async fn required_startup_failures( - &self, - required_servers: &[String], - ) -> Vec { - let mut failures = Vec::new(); - for server_name in required_servers { - let Some(async_managed_client) = self.clients.get(server_name).cloned() else { - failures.push(McpStartupFailure { - server: server_name.clone(), - error: format!("required MCP server `{server_name}` was not initialized"), - }); - continue; - }; - - match async_managed_client.client().await { - Ok(_) => {} - Err(error) => failures.push(McpStartupFailure { - server: server_name.clone(), - error: startup_outcome_error_message(error), - }), - } - } - failures - } - /// Returns all tools with model-visible names normalized. #[instrument(level = "trace", skip_all, fields(mcp_server_count = self.clients.len()))] pub async fn list_all_tools(&self) -> Vec { diff --git a/codex-rs/codex-mcp/src/connection_manager_tests.rs b/codex-rs/codex-mcp/src/connection_manager_tests.rs index 60a0026eeb..4b90337ac2 100644 --- a/codex-rs/codex-mcp/src/connection_manager_tests.rs +++ b/codex-rs/codex-mcp/src/connection_manager_tests.rs @@ -1167,13 +1167,15 @@ async fn no_local_runtime_fails_local_stdio_but_keeps_local_http_server() { ), ]); - let (manager, cancel_token) = McpConnectionManager::new( + let cancel_token = CancellationToken::new(); + let manager = McpConnectionManager::new( &mcp_servers, OAuthCredentialsStoreMode::default(), HashMap::new(), &approval_policy, String::new(), tx_event, + cancel_token.clone(), PermissionProfile::default(), McpRuntimeContext::new( Arc::new(EnvironmentManager::without_environments()), @@ -1201,13 +1203,18 @@ async fn no_local_runtime_fails_local_stdio_but_keeps_local_http_server() { .wait_for_server_ready("stdio", Duration::from_millis(10)) .await ); - let failures = manager - .required_startup_failures(&["stdio".to_string()]) - .await; - assert_eq!(failures.len(), 1); - assert_eq!(failures[0].server, "stdio"); + let error = match manager + .clients + .get("stdio") + .expect("stdio client") + .client() + .await + { + Ok(_) => panic!("local stdio MCP startup should fail"), + Err(error) => error, + }; assert_eq!( - failures[0].error, + startup_outcome_error_message(error), "local stdio MCP server `stdio` requires a local environment" ); cancel_token.cancel(); diff --git a/codex-rs/codex-mcp/src/mcp/mod.rs b/codex-rs/codex-mcp/src/mcp/mod.rs index a74f95bae3..adc2cf615e 100644 --- a/codex-rs/codex-mcp/src/mcp/mod.rs +++ b/codex-rs/codex-mcp/src/mcp/mod.rs @@ -35,6 +35,7 @@ use rmcp::model::ElicitationCapability; use rmcp::model::ReadResourceRequestParams; use rmcp::model::ReadResourceResult; use serde_json::Value; +use tokio_util::sync::CancellationToken; use crate::codex_apps::codex_apps_tools_cache_key; use crate::connection_manager::McpConnectionManager; @@ -293,13 +294,15 @@ pub async fn read_mcp_resource( .await; let (tx_event, rx_event) = unbounded(); drop(rx_event); - let (manager, cancel_token) = McpConnectionManager::new( + let cancel_token = CancellationToken::new(); + let manager = McpConnectionManager::new( &mcp_servers, config.mcp_oauth_credentials_store_mode, auth_statuses, &config.approval_policy, String::new(), tx_event, + cancel_token.clone(), PermissionProfile::default(), runtime_context, config.codex_home.clone(), @@ -363,13 +366,15 @@ pub async fn collect_mcp_server_status_snapshot_with_detail( let (tx_event, rx_event) = unbounded(); drop(rx_event); - let (mcp_connection_manager, cancel_token) = McpConnectionManager::new( + let cancel_token = CancellationToken::new(); + let mcp_connection_manager = McpConnectionManager::new( &mcp_servers, config.mcp_oauth_credentials_store_mode, auth_status_entries.clone(), &config.approval_policy, submit_id, tx_event, + cancel_token.clone(), PermissionProfile::default(), runtime_context, config.codex_home.clone(), diff --git a/codex-rs/core/src/connectors.rs b/codex-rs/core/src/connectors.rs index 623cb44216..6d6ca52d8c 100644 --- a/codex-rs/core/src/connectors.rs +++ b/codex-rs/core/src/connectors.rs @@ -17,6 +17,7 @@ use codex_protocol::models::PermissionProfile; use codex_tools::DiscoverableTool; use rmcp::model::ToolAnnotations; use serde::Deserialize; +use tokio_util::sync::CancellationToken; use tracing::warn; use crate::config::Config; @@ -285,13 +286,15 @@ pub async fn list_accessible_connectors_from_mcp_tools_with_mcp_manager( let (tx_event, rx_event) = unbounded(); drop(rx_event); - let (mut mcp_connection_manager, cancel_token) = McpConnectionManager::new( + let cancel_token = CancellationToken::new(); + let mut mcp_connection_manager = McpConnectionManager::new( &mcp_servers, config.mcp_oauth_credentials_store_mode, auth_status_entries, &config.permissions.approval_policy, INITIAL_SUBMIT_ID.to_owned(), tx_event, + cancel_token.clone(), PermissionProfile::default(), // Connector discovery is threadless. Use an actually configured env if // one exists, but do not reintroduce the old hidden-local fallback. diff --git a/codex-rs/core/src/mcp_tool_call_tests.rs b/codex-rs/core/src/mcp_tool_call_tests.rs index bd71c3a054..da54fe1873 100644 --- a/codex-rs/core/src/mcp_tool_call_tests.rs +++ b/codex-rs/core/src/mcp_tool_call_tests.rs @@ -46,6 +46,7 @@ use std::path::Path; use std::path::PathBuf; use std::sync::Arc; use tempfile::tempdir; +use tokio_util::sync::CancellationToken; use tracing::Instrument; use tracing::Level; use tracing_subscriber::fmt::format::FmtSpan; @@ -1256,13 +1257,14 @@ fn codex_apps_auth_failure_metadata() -> McpToolApprovalMetadata { async fn install_host_owned_codex_apps_manager(session: &Session, turn_context: &TurnContext) { let auth = session.services.auth_manager.auth().await; - let (manager, _cancel_token) = codex_mcp::McpConnectionManager::new( + let manager = codex_mcp::McpConnectionManager::new( &HashMap::new(), turn_context.config.mcp_oauth_credentials_store_mode, HashMap::new(), &turn_context.approval_policy, turn_context.sub_id.clone(), session.get_tx_event(), + CancellationToken::new(), turn_context.permission_profile(), codex_mcp::McpRuntimeContext::new(Arc::clone(&session.services.environment_manager), { #[allow(deprecated)] diff --git a/codex-rs/core/src/session/mcp.rs b/codex-rs/core/src/session/mcp.rs index f2643d0e5c..7798136418 100644 --- a/codex-rs/core/src/session/mcp.rs +++ b/codex-rs/core/src/session/mcp.rs @@ -338,18 +338,21 @@ impl Session { turn_context.cwd.to_path_buf(), ), }; - { + let mcp_startup_cancellation_token = { let mut guard = self.services.mcp_startup_cancellation_token.lock().await; guard.cancel(); - *guard = CancellationToken::new(); - } - let (refreshed_manager, cancel_token) = McpConnectionManager::new( + let cancellation_token = CancellationToken::new(); + *guard = cancellation_token.clone(); + cancellation_token + }; + let refreshed_manager = McpConnectionManager::new( &mcp_servers, store_mode, auth_statuses, &turn_context.approval_policy, turn_context.sub_id.clone(), self.get_tx_event(), + mcp_startup_cancellation_token, turn_context.permission_profile(), mcp_runtime_context, config.codex_home.to_path_buf(), @@ -366,14 +369,6 @@ impl Session { let current_manager = self.services.mcp_connection_manager.read().await; refreshed_manager.set_elicitations_auto_deny(current_manager.elicitations_auto_deny()); } - { - let mut guard = self.services.mcp_startup_cancellation_token.lock().await; - if guard.is_cancelled() { - cancel_token.cancel(); - } - *guard = cancel_token; - } - let mut old_manager = { let mut manager = self.services.mcp_connection_manager.write().await; std::mem::replace(&mut *manager, refreshed_manager) diff --git a/codex-rs/core/src/session/session.rs b/codex-rs/core/src/session/session.rs index 50ff984231..98d19957c0 100644 --- a/codex-rs/core/src/session/session.rs +++ b/codex-rs/core/src/session/session.rs @@ -469,10 +469,6 @@ impl Session { #[instrument(name = "session_init", level = "info", skip_all)] #[allow(clippy::too_many_arguments)] - #[expect( - clippy::await_holding_invalid_type, - reason = "session initialization must serialize access through session-owned manager guards" - )] pub(crate) async fn new( mut session_configuration: SessionConfiguration, config: Arc, @@ -1116,15 +1112,6 @@ impl Session { sess.send_event_raw(event).await; } - let mut required_mcp_servers: Vec = mcp_servers - .iter() - .filter(|(_, server)| server.enabled() && server.required()) - .map(|(name, _)| name.clone()) - .collect(); - required_mcp_servers.sort(); - let enabled_mcp_server_count = - mcp_servers.values().filter(|server| server.enabled()).count(); - let required_mcp_server_count = required_mcp_servers.len(); let tool_plugin_provenance = mcp_manager.tool_plugin_provenance(config.as_ref()).await; let host_owned_codex_apps_enabled = config .features @@ -1137,11 +1124,13 @@ impl Session { } else { ElicitationCapability::default() }; - { + let mcp_startup_cancellation_token = { let mut cancel_guard = sess.services.mcp_startup_cancellation_token.lock().await; cancel_guard.cancel(); - *cancel_guard = CancellationToken::new(); - } + let cancel_token = CancellationToken::new(); + *cancel_guard = cancel_token.clone(); + cancel_token + }; let turn_environment = crate::environment_selection::resolve_environment_selections( sess.services.environment_manager.as_ref(), session_configuration.environment_selections(), @@ -1164,13 +1153,14 @@ impl Session { session_configuration.cwd().to_path_buf(), ), }; - let (mcp_connection_manager, cancel_token) = McpConnectionManager::new( + let mcp_connection_manager = McpConnectionManager::new( &mcp_servers, config.mcp_oauth_credentials_store_mode, - auth_statuses.clone(), + auth_statuses, &session_configuration.approval_policy, INITIAL_SUBMIT_ID.to_owned(), tx_event.clone(), + mcp_startup_cancellation_token, session_configuration.permission_profile(), mcp_runtime_context, config.codex_home.to_path_buf(), @@ -1185,43 +1175,11 @@ impl Session { .instrument(info_span!( "session_init.mcp_manager_init", otel.name = "session_init.mcp_manager_init", - session_init.enabled_mcp_server_count = enabled_mcp_server_count, - session_init.required_mcp_server_count = required_mcp_server_count, )) .await; - { - let mut manager_guard = sess.services.mcp_connection_manager.write().await; - *manager_guard = mcp_connection_manager; - } - { - let mut cancel_guard = sess.services.mcp_startup_cancellation_token.lock().await; - if cancel_guard.is_cancelled() { - cancel_token.cancel(); - } - *cancel_guard = cancel_token; - } - if !required_mcp_servers.is_empty() { - let failures = sess - .services - .mcp_connection_manager - .read() - .await - .required_startup_failures(&required_mcp_servers) - .instrument(info_span!( - "session_init.required_mcp_wait", - otel.name = "session_init.required_mcp_wait", - session_init.required_mcp_server_count = required_mcp_server_count, - )) - .await; - if !failures.is_empty() { - let details = failures - .iter() - .map(|failure| format!("{}: {}", failure.server, failure.error)) - .collect::>() - .join("; "); - anyhow::bail!("required MCP servers failed to initialize: {details}"); - } - } + sess.services + .install_mcp_connection_manager(mcp_connection_manager) + .await?; sess.schedule_startup_prewarm(session_configuration.base_instructions.clone()) .await; let session_start_source = match &initial_history { diff --git a/codex-rs/core/src/state/service.rs b/codex-rs/core/src/state/service.rs index 59def9c7ab..9bc4d73915 100644 --- a/codex-rs/core/src/state/service.rs +++ b/codex-rs/core/src/state/service.rs @@ -15,6 +15,7 @@ use crate::tools::code_mode::CodeModeService; use crate::tools::network_approval::NetworkApprovalService; use crate::tools::sandboxing::ApprovalStore; use crate::unified_exec::UnifiedExecProcessManager; +use anyhow::Result; use arc_swap::ArcSwap; use arc_swap::ArcSwapOption; use codex_analytics::AnalyticsEventsClient; @@ -82,3 +83,23 @@ pub(crate) struct SessionServices { /// the same manager through child-thread spawn paths without reconstructing it. pub(crate) environment_manager: Arc, } + +impl SessionServices { + /// Installs the manager before validating required servers so startup-time elicitation can + /// resolve through the session's manager while validation waits. + #[expect( + clippy::await_holding_invalid_type, + reason = "required MCP validation keeps the installed manager reachable for startup-time elicitation" + )] + pub(crate) async fn install_mcp_connection_manager( + &self, + manager: McpConnectionManager, + ) -> Result<()> { + *self.mcp_connection_manager.write().await = manager; + self.mcp_connection_manager + .read() + .await + .validate_required_servers() + .await + } +} From 0ffcefaf3ddb3a61d8683ca0703f7d8b39ad6c1e Mon Sep 17 00:00:00 2001 From: jif Date: Wed, 10 Jun 2026 11:45:49 +0200 Subject: [PATCH 27/47] feat: keep child MCP warnings out of parent transcript (#27174) ## Why MCP startup status notifications are thread-owned, but `ChatWidget` trusted upstream routing. If routing state delivered a tagged child notification to the active parent widget, the child MCP failure could still mutate the parent's startup state and transcript. Rejecting it only inside the MCP handler was also too late because shared notification handling could already restore and consume the parent's retry status. ## What changed - Validate a tagged MCP status notification against the visible `ChatWidget` thread before shared notification handling mutates any parent state. - Cover child `Starting` and `Failed` notifications delivered to a retrying parent widget, asserting that they preserve its visible retry error and saved status header while producing no history or MCP status mutation. ## User impact Subagent MCP startup failures remain scoped to the child transcript instead of appearing as duplicate warnings in the parent transcript. ## Testing - `just test -p codex-tui mcp_startup_ignores_status_for_other_thread` - `just test -p codex-tui primary_thread_ignores_child_mcp_startup_notifications` - `just fmt` --- codex-rs/tui/src/chatwidget/protocol.rs | 9 ++++ .../tui/src/chatwidget/tests/mcp_startup.rs | 41 +++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/codex-rs/tui/src/chatwidget/protocol.rs b/codex-rs/tui/src/chatwidget/protocol.rs index 9d29208075..3682f245f5 100644 --- a/codex-rs/tui/src/chatwidget/protocol.rs +++ b/codex-rs/tui/src/chatwidget/protocol.rs @@ -6,6 +6,15 @@ impl ChatWidget { notification: ServerNotification, replay_kind: Option, ) { + // Reject misrouted child updates before shared notification handling mutates parent state. + if let ServerNotification::McpServerStatusUpdated(notification) = ¬ification + && let (Some(notification_thread_id), Some(thread_id)) = + (notification.thread_id.as_deref(), self.thread_id()) + && notification_thread_id != thread_id.to_string() + { + return; + } + let from_replay = replay_kind.is_some(); let is_resume_initial_replay = matches!(replay_kind, Some(ReplayKind::ResumeInitialMessages)); diff --git a/codex-rs/tui/src/chatwidget/tests/mcp_startup.rs b/codex-rs/tui/src/chatwidget/tests/mcp_startup.rs index 8b0d5ca472..60efbe1353 100644 --- a/codex-rs/tui/src/chatwidget/tests/mcp_startup.rs +++ b/codex-rs/tui/src/chatwidget/tests/mcp_startup.rs @@ -25,6 +25,47 @@ fn notify_mcp_status_error(chat: &mut ChatWidget, name: &str, error: &str) { ); } +#[tokio::test] +async fn mcp_startup_ignores_status_for_other_thread() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + chat.show_welcome_banner = false; + chat.set_mcp_startup_expected_servers(["sentry".to_string()]); + let parent_thread_id = ThreadId::new(); + let child_thread_id = ThreadId::new(); + chat.thread_id = Some(parent_thread_id); + chat.on_stream_error( + "Connection interrupted, retrying".to_string(), + /*additional_details*/ None, + ); + let status_before = chat.status_state.current_status.clone(); + let retry_status_header_before = chat.status_state.retry_status_header.clone(); + + for status in [ + McpServerStartupState::Starting, + McpServerStartupState::Failed, + ] { + chat.handle_server_notification( + ServerNotification::McpServerStatusUpdated(McpServerStatusUpdatedNotification { + thread_id: Some(child_thread_id.to_string()), + name: "sentry".to_string(), + status, + error: matches!(status, McpServerStartupState::Failed) + .then(|| "sentry is not logged in".to_string()), + }), + /*replay_kind*/ None, + ); + } + + assert!(drain_insert_history(&mut rx).is_empty()); + assert!(!chat.bottom_pane.is_task_running()); + assert!(chat.mcp_startup_status.is_none()); + assert_eq!(chat.status_state.current_status, status_before); + assert_eq!( + chat.status_state.retry_status_header, + retry_status_header_before + ); +} + #[tokio::test] async fn mcp_startup_dedupes_same_round_duplicate_failure_warning() { let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; From 9cd11e9e62c48ea6127f8b9f6ae2e013e4e4ff00 Mon Sep 17 00:00:00 2001 From: jif Date: Wed, 10 Jun 2026 12:54:21 +0200 Subject: [PATCH 28/47] Use plugin-service MCP as the hosted plugin runtime (#27198) ## Stack - Base: #27191 - This PR is the third vertical and should be reviewed against `jif/external-plugins-2`, not `main`. ## Why #27191 moves the host-owned Apps MCP registration behind an extension contributor, but deliberately preserves the existing endpoint-selection feature while that contribution contract lands. App-server can therefore resolve the server through extensions, yet the hosted plugin endpoint is still selected through temporary `apps_mcp_path_override` plumbing. That is not the long-term plugin model. A plugin can bundle skills, connectors, MCP servers, and hooks, and those components do not all need the same source or execution environment. In particular, an authenticated HTTP MCP server can expose plugin capabilities directly from a backend without an executor or an orchestrator filesystem. This PR completes that hosted vertical. App-server's MCP extension now owns the aggregate hosted plugin runtime at `/ps/mcp`. Connector actions continue to arrive as MCP tools, while backend-provided skills arrive as MCP resources and use Codex's existing resource list/read paths. No second backend client, skill filesystem, or generic plugin activation framework is introduced. The backend route remains the hosted implementation. This change replaces Codex's temporary endpoint-selection mechanism, not the service behind the endpoint. ## What changed ### Hosted plugin runtime The MCP extension now contributes `codex_apps` as the hosted plugin runtime rather than as a configurable Apps endpoint: - `https://chatgpt.com` resolves to `https://chatgpt.com/backend-api/ps/mcp`; - a bare custom ChatGPT base resolves to `/api/codex/ps/mcp`; - the existing product-SKU header and ChatGPT authentication behavior are preserved; - executor availability is never consulted for this streamable HTTP transport. The same MCP connection carries both component shapes supported by the hosted endpoint: - connector actions are discovered and invoked as MCP tools; - hosted skills are enumerated and read as MCP resources through the existing `list_mcp_resources` and `read_mcp_resource` paths. This keeps component access in the subsystem that already owns the protocol instead of downloading backend skills into an orchestrator filesystem or inventing a parallel hosted-skill client. ### Explicit runtime ordering `McpManager` now resolves the reserved `codex_apps` entry in three ordered phases: 1. install the legacy Apps fallback for compatibility; 2. apply ordered extension `Set` or `Remove` overlays; 3. apply the final ChatGPT-auth gate without synthesizing the server again. This ordering is important: - an ordinary configured or plugin MCP server cannot claim the auth-bearing `codex_apps` name; - an extension-contributed hosted runtime wins over the fallback; - an extension `Remove` remains authoritative; - a host without the MCP extension retains the legacy Apps endpoint and current local-only behavior. The temporary `legacy_apps_mcp_loader_enabled` coordination flag is no longer needed. ### Remove the path override The `apps_mcp_path_override` feature and its runtime plumbing are removed, including: - the feature registry entry and structured feature config; - `Config` and `McpConfig` fields; - config schema output; - config-lock materialization; - URL override handling in `codex-mcp`. Existing boolean and structured forms still deserialize as ignored compatibility input. They are omitted from new serialized config, and config-lock comparison normalizes the removed input so older locks remain replayable. ### App-server coverage App-server MCP fixtures now serve the hosted route at `/api/codex/ps/mcp`. Existing resource-read and tool/elicitation flows therefore exercise the extension-owned endpoint rather than succeeding through the legacy fallback. The stack also adds the missing `codex_chatgpt::connectors` re-export for the manager-backed connector helper introduced in #27191. ## Compatibility - App-server installs the extension and uses `/ps/mcp` for the hosted runtime. - CLI and other hosts that do not install the extension retain the legacy Apps endpoint. - Apps disabled or non-ChatGPT authentication removes `codex_apps` from the effective runtime view. - Existing local plugins, local skills, executor-selected skills, configured MCP servers, and MCP OAuth behavior are otherwise unchanged. - Backend plugin enablement remains account/workspace state owned by the hosted endpoint; this PR does not add thread-local backend plugin selection. ## Architectural fit The stack now proves two independent runtime shapes: 1. #27184 resolves filesystem-backed skills through the executor that owns a selected root. 2. #27191 and this PR resolve a backend-hosted HTTP MCP through an extension with no executor. Together they preserve the intended separation: - selection identifies a plugin/root when explicit selection is needed; - each component's owning extension resolves its concrete access mechanism; - execution stays with the runtime required by that component; - existing skills, MCP, connector, and hook subsystems remain the downstream consumers. ## Planned follow-ups 1. **Executor stdio MCP:** selecting an executor plugin registers a manifest-declared stdio MCP server and executes it in the environment that owns the plugin. 2. **Optional backend selection:** only if CCA needs thread-local selection distinct from backend account/workspace enablement, add a concrete backend-owned capability location and surface those selected skills through the skills catalog. 3. **Connector metadata and hooks:** activate those plugin components through their existing owning subsystems, with executor hooks remaining environment-bound. 4. **Propagation and persistence:** define explicit resume, fork, subagent, refresh, and environment-removal semantics once selected roots have multiple real consumers. 5. **Local convergence:** migrate legacy local skill, MCP, connector, and hook paths behind their owning extensions one vertical at a time, then remove duplicate core managers and compatibility plumbing after parity. ## Verification Coverage in this change exercises: - extension-owned `/backend-api/ps/mcp` registration without an executor; - preservation of the legacy endpoint in hosts without the extension; - extension `Set` and `Remove` precedence over the legacy fallback; - ChatGPT-auth gating for the reserved server; - hosted MCP resource reads with and without an active thread; - connector tool invocation and MCP elicitation through the hosted route; - ignored boolean and structured forms of the removed path override; - config-lock replay compatibility for the removed feature. `cargo check -p codex-features -p codex-mcp-extension -p codex-app-server` passes. Tests and Clippy were not run locally under the current development instruction; CI provides the full validation pass. --- .../app-server/tests/suite/v2/app_list.rs | 2 +- .../app-server/tests/suite/v2/mcp_resource.rs | 2 +- .../tests/suite/v2/mcp_server_elicitation.rs | 2 +- .../tests/suite/v2/plugin_install.rs | 2 +- .../app-server/tests/suite/v2/plugin_read.rs | 2 +- codex-rs/codex-mcp/src/lib.rs | 2 +- codex-rs/codex-mcp/src/mcp/mod.rs | 83 ++++++++---------- codex-rs/codex-mcp/src/mcp/mod_tests.rs | 83 ++++-------------- codex-rs/config/src/schema.rs | 29 ++++++- codex-rs/core/config.schema.json | 60 +++++++------ codex-rs/core/src/config/config_tests.rs | 83 ------------------ codex-rs/core/src/config/mod.rs | 24 ------ codex-rs/core/src/config_lock.rs | 3 + codex-rs/core/src/mcp.rs | 23 +++-- codex-rs/core/src/session/config_lock.rs | 31 +++++-- .../core/src/tools/handlers/mcp_resource.rs | 10 ++- .../list_mcp_resource_templates.rs | 2 +- .../mcp_resource/list_mcp_resources.rs | 2 +- .../mcp_resource/read_mcp_resource.rs | 2 +- .../src/tools/handlers/mcp_resource_tests.rs | 37 ++++++++ .../ext/extension-api/src/contributors/mcp.rs | 9 -- codex-rs/ext/mcp/src/lib.rs | 11 ++- codex-rs/ext/mcp/tests/hosted_apps_mcp.rs | 85 +++++++++++++++++-- codex-rs/features/src/feature_configs.rs | 16 +--- codex-rs/features/src/lib.rs | 32 +++---- codex-rs/features/src/tests.rs | 21 +++++ codex-rs/thread-manager-sample/src/main.rs | 1 - 27 files changed, 334 insertions(+), 325 deletions(-) diff --git a/codex-rs/app-server/tests/suite/v2/app_list.rs b/codex-rs/app-server/tests/suite/v2/app_list.rs index c9615d3a27..ace43d2847 100644 --- a/codex-rs/app-server/tests/suite/v2/app_list.rs +++ b/codex-rs/app-server/tests/suite/v2/app_list.rs @@ -1516,7 +1516,7 @@ async fn start_apps_server_with_delays_and_control_inner( get(workspace_settings_response), ) .with_state(state) - .nest_service("/api/codex/apps", mcp_service); + .nest_service("/api/codex/ps/mcp", mcp_service); let handle = tokio::spawn(async move { let _ = axum::serve(listener, router).await; diff --git a/codex-rs/app-server/tests/suite/v2/mcp_resource.rs b/codex-rs/app-server/tests/suite/v2/mcp_resource.rs index d8ecb88996..2e3ffcb066 100644 --- a/codex-rs/app-server/tests/suite/v2/mcp_resource.rs +++ b/codex-rs/app-server/tests/suite/v2/mcp_resource.rs @@ -256,7 +256,7 @@ async fn start_resource_apps_mcp_server() -> Result<(String, JoinHandle<()>)> { Arc::new(LocalSessionManager::default()), StreamableHttpServerConfig::default(), ); - let router = Router::new().nest_service("/api/codex/apps", mcp_service); + let router = Router::new().nest_service("/api/codex/ps/mcp", mcp_service); let apps_server_handle = tokio::spawn(async move { let _ = axum::serve(listener, router).await; }); diff --git a/codex-rs/app-server/tests/suite/v2/mcp_server_elicitation.rs b/codex-rs/app-server/tests/suite/v2/mcp_server_elicitation.rs index 8d0387ee97..b4bd1d377a 100644 --- a/codex-rs/app-server/tests/suite/v2/mcp_server_elicitation.rs +++ b/codex-rs/app-server/tests/suite/v2/mcp_server_elicitation.rs @@ -406,7 +406,7 @@ async fn start_apps_server() -> Result<(String, JoinHandle<()>)> { get(list_directory_connectors), ) .with_state(state) - .nest_service("/api/codex/apps", mcp_service); + .nest_service("/api/codex/ps/mcp", mcp_service); let handle = tokio::spawn(async move { let _ = axum::serve(listener, router).await; diff --git a/codex-rs/app-server/tests/suite/v2/plugin_install.rs b/codex-rs/app-server/tests/suite/v2/plugin_install.rs index 5ebe4899d0..ca490ad5e8 100644 --- a/codex-rs/app-server/tests/suite/v2/plugin_install.rs +++ b/codex-rs/app-server/tests/suite/v2/plugin_install.rs @@ -1289,7 +1289,7 @@ async fn start_apps_server( get(list_directory_connectors), ) .with_state(state) - .nest_service("/api/codex/apps", mcp_service); + .nest_service("/api/codex/ps/mcp", mcp_service); let handle = tokio::spawn(async move { let _ = axum::serve(listener, router).await; diff --git a/codex-rs/app-server/tests/suite/v2/plugin_read.rs b/codex-rs/app-server/tests/suite/v2/plugin_read.rs index e675a6172a..4d8291a328 100644 --- a/codex-rs/app-server/tests/suite/v2/plugin_read.rs +++ b/codex-rs/app-server/tests/suite/v2/plugin_read.rs @@ -1914,7 +1914,7 @@ async fn start_apps_server( get(list_directory_connectors), ) .with_state(state) - .nest_service("/api/codex/apps", mcp_service); + .nest_service("/api/codex/ps/mcp", mcp_service); let handle = tokio::spawn(async move { let _ = axum::serve(listener, router).await; diff --git a/codex-rs/codex-mcp/src/lib.rs b/codex-rs/codex-mcp/src/lib.rs index a56365277a..f0c7c3c8ca 100644 --- a/codex-rs/codex-mcp/src/lib.rs +++ b/codex-rs/codex-mcp/src/lib.rs @@ -29,8 +29,8 @@ pub use mcp::configured_mcp_servers; pub use mcp::effective_mcp_servers; pub use mcp::effective_mcp_servers_from_configured; pub use mcp::host_owned_codex_apps_enabled; +pub use mcp::hosted_plugin_runtime_mcp_server_config; pub use mcp::tool_plugin_provenance; -pub use mcp::with_codex_apps_mcp; pub use mcp::McpServerStatusSnapshot; pub use mcp::McpSnapshotDetail; diff --git a/codex-rs/codex-mcp/src/mcp/mod.rs b/codex-rs/codex-mcp/src/mcp/mod.rs index adc2cf615e..e6b17b14c3 100644 --- a/codex-rs/codex-mcp/src/mcp/mod.rs +++ b/codex-rs/codex-mcp/src/mcp/mod.rs @@ -107,8 +107,6 @@ pub struct McpPermissionPromptAutoApproveContext { pub struct McpConfig { /// Base URL for ChatGPT-hosted app MCP servers, copied from the root config. pub chatgpt_base_url: String, - /// Optional path override for the host-owned apps MCP server. - pub apps_mcp_path_override: Option, /// Optional product SKU forwarded to the host-owned apps MCP server. pub apps_mcp_product_sku: Option, /// Codex home directory used for MCP OAuth state and app-tool cache files. @@ -129,22 +127,18 @@ pub struct McpConfig { pub use_legacy_landlock: bool, /// Whether the app MCP integration is enabled by config. /// - /// ChatGPT auth is checked separately at runtime before the host-owned apps - /// MCP server is added. + /// ChatGPT auth is checked separately before a materialized host-owned Apps + /// server can be used. pub apps_enabled: bool, - /// Whether to synthesize the legacy host-owned Apps MCP server. - /// - /// Hosts that install an MCP extension for this server disable the legacy - /// loader and contribute the server through the normal runtime overlay. - pub legacy_apps_mcp_loader_enabled: bool, /// Whether model-visible MCP tool namespaces should keep the legacy /// `mcp__` prefix. pub prefix_mcp_tool_names: bool, /// Client-side elicitation capabilities advertised during MCP initialization. pub client_elicitation_capability: ElicitationCapability, - /// Config-backed MCP servers keyed by server name. + /// Materialized MCP servers keyed by server name. /// - /// Runtime-only additions are merged later by [`effective_mcp_servers`]. + /// A host may add compatibility built-ins and extension overlays before + /// calling runtime entry points in this crate. pub configured_mcp_servers: HashMap, /// Winning plugin owner for plugin-provided MCP servers, keyed by server name. pub plugin_ids_by_mcp_server_name: HashMap, @@ -219,32 +213,6 @@ impl ToolPluginProvenance { } } -pub fn with_codex_apps_mcp( - mut servers: HashMap, - auth: Option<&CodexAuth>, - config: &McpConfig, -) -> HashMap { - if !config.legacy_apps_mcp_loader_enabled { - if !host_owned_codex_apps_enabled(config, auth) { - servers.remove(CODEX_APPS_MCP_SERVER_NAME); - } - return servers; - } - if host_owned_codex_apps_enabled(config, auth) { - servers.insert( - CODEX_APPS_MCP_SERVER_NAME.to_string(), - EffectiveMcpServer::configured(codex_apps_mcp_server_config( - &config.chatgpt_base_url, - config.apps_mcp_path_override.as_deref(), - config.apps_mcp_product_sku.as_deref(), - )), - ); - } else { - servers.remove(CODEX_APPS_MCP_SERVER_NAME); - } - servers -} - pub fn host_owned_codex_apps_enabled(config: &McpConfig, auth: Option<&CodexAuth>) -> bool { config.apps_enabled && auth.is_some_and(CodexAuth::uses_codex_backend) } @@ -260,16 +228,23 @@ pub fn effective_mcp_servers( effective_mcp_servers_from_configured(configured_mcp_servers(config), config, auth) } +/// Converts a materialized server map to its auth-gated runtime view. +/// +/// Compatibility built-ins and extension overlays must already be reflected in +/// `configured_servers`; this function does not synthesize missing servers. pub fn effective_mcp_servers_from_configured( configured_servers: HashMap, config: &McpConfig, auth: Option<&CodexAuth>, ) -> HashMap { - let servers = configured_servers + let mut servers = configured_servers .into_iter() .map(|(name, server)| (name, EffectiveMcpServer::configured(server))) .collect::>(); - with_codex_apps_mcp(servers, auth, config) + if !host_owned_codex_apps_enabled(config, auth) { + servers.remove(CODEX_APPS_MCP_SERVER_NAME); + } + servers } pub fn tool_plugin_provenance(config: &McpConfig) -> ToolPluginProvenance { @@ -441,7 +416,7 @@ fn normalize_codex_apps_base_url(base_url: &str) -> String { base_url } -fn codex_apps_mcp_url_for_base_url(base_url: &str, apps_mcp_path_override: Option<&str>) -> String { +fn codex_apps_mcp_url_for_base_url(base_url: &str) -> String { let base_url = normalize_codex_apps_base_url(base_url); let (base_url, default_path) = if base_url.contains("/backend-api") { (base_url, "wham/apps") @@ -450,18 +425,34 @@ fn codex_apps_mcp_url_for_base_url(base_url: &str, apps_mcp_path_override: Optio } else { (format!("{base_url}/api/codex"), "apps") }; - let path = apps_mcp_path_override - .unwrap_or(default_path) - .trim_start_matches('/'); - format!("{base_url}/{path}") + format!("{base_url}/{default_path}") } pub fn codex_apps_mcp_server_config( chatgpt_base_url: &str, - apps_mcp_path_override: Option<&str>, apps_mcp_product_sku: Option<&str>, ) -> McpServerConfig { - let url = codex_apps_mcp_url_for_base_url(chatgpt_base_url, apps_mcp_path_override); + mcp_server_config_for_url( + codex_apps_mcp_url_for_base_url(chatgpt_base_url), + apps_mcp_product_sku, + ) +} + +/// Builds the ChatGPT-hosted plugin runtime served by plugin-service. +pub fn hosted_plugin_runtime_mcp_server_config( + chatgpt_base_url: &str, + apps_mcp_product_sku: Option<&str>, +) -> McpServerConfig { + let base_url = normalize_codex_apps_base_url(chatgpt_base_url); + let base_url = if base_url.contains("/backend-api") || base_url.contains("/api/codex") { + base_url + } else { + format!("{base_url}/api/codex") + }; + mcp_server_config_for_url(format!("{base_url}/ps/mcp"), apps_mcp_product_sku) +} + +fn mcp_server_config_for_url(url: String, apps_mcp_product_sku: Option<&str>) -> McpServerConfig { let http_headers = apps_mcp_product_sku.map(|product_sku| { HashMap::from([("X-OpenAI-Product-Sku".to_string(), product_sku.to_string())]) }); diff --git a/codex-rs/codex-mcp/src/mcp/mod_tests.rs b/codex-rs/codex-mcp/src/mcp/mod_tests.rs index fb1af362ca..767f20611f 100644 --- a/codex-rs/codex-mcp/src/mcp/mod_tests.rs +++ b/codex-rs/codex-mcp/src/mcp/mod_tests.rs @@ -16,7 +16,6 @@ use std::path::PathBuf; fn test_mcp_config(codex_home: PathBuf) -> McpConfig { McpConfig { chatgpt_base_url: "https://chatgpt.com".to_string(), - apps_mcp_path_override: None, apps_mcp_product_sku: None, codex_home, mcp_oauth_credentials_store_mode: OAuthCredentialsStoreMode::default(), @@ -27,7 +26,6 @@ fn test_mcp_config(codex_home: PathBuf) -> McpConfig { codex_linux_sandbox_exe: None, use_legacy_landlock: false, apps_enabled: false, - legacy_apps_mcp_loader_enabled: true, prefix_mcp_tool_names: true, client_elicitation_capability: ElicitationCapability::default(), configured_mcp_servers: HashMap::new(), @@ -178,52 +176,27 @@ fn tool_plugin_provenance_collects_app_and_mcp_sources() { #[test] fn codex_apps_mcp_url_for_base_url_keeps_existing_paths() { assert_eq!( - codex_apps_mcp_url_for_base_url( - "https://chatgpt.com/backend-api", - /*apps_mcp_path_override*/ None, - ), + codex_apps_mcp_url_for_base_url("https://chatgpt.com/backend-api"), "https://chatgpt.com/backend-api/wham/apps" ); assert_eq!( - codex_apps_mcp_url_for_base_url( - "https://chat.openai.com", - /*apps_mcp_path_override*/ None, - ), + codex_apps_mcp_url_for_base_url("https://chat.openai.com"), "https://chat.openai.com/backend-api/wham/apps" ); assert_eq!( - codex_apps_mcp_url_for_base_url( - "http://localhost:8080/api/codex", - /*apps_mcp_path_override*/ None, - ), + codex_apps_mcp_url_for_base_url("http://localhost:8080/api/codex"), "http://localhost:8080/api/codex/apps" ); assert_eq!( - codex_apps_mcp_url_for_base_url( - "http://localhost:8080", - /*apps_mcp_path_override*/ None, - ), + codex_apps_mcp_url_for_base_url("http://localhost:8080"), "http://localhost:8080/api/codex/apps" ); } #[test] fn codex_apps_server_config_uses_legacy_codex_apps_path() { - let mut config = test_mcp_config(PathBuf::from("/tmp")); - let auth = CodexAuth::create_dummy_chatgpt_auth_for_testing(); - - let mut servers = with_codex_apps_mcp(HashMap::new(), /*auth*/ None, &config); - assert!(!servers.contains_key(CODEX_APPS_MCP_SERVER_NAME)); - - config.apps_enabled = true; - - servers = with_codex_apps_mcp(servers, Some(&auth), &config); - let server = servers - .get(CODEX_APPS_MCP_SERVER_NAME) - .expect("codex apps should be present when apps is enabled"); - let config = server - .configured_config() - .expect("codex apps should use configured transport"); + let config = + codex_apps_mcp_server_config("https://chatgpt.com", /*apps_mcp_product_sku*/ None); let url = match &config.transport { McpServerTransportConfig::StreamableHttp { url, .. } => url, _ => panic!("expected streamable http transport for codex apps"), @@ -232,42 +205,9 @@ fn codex_apps_server_config_uses_legacy_codex_apps_path() { assert_eq!(url, "https://chatgpt.com/backend-api/wham/apps"); } -#[test] -fn codex_apps_server_config_uses_configured_apps_mcp_path_override() { - let mut config = test_mcp_config(PathBuf::from("/tmp")); - config.apps_mcp_path_override = Some("/custom/mcp".to_string()); - config.apps_enabled = true; - let auth = CodexAuth::create_dummy_chatgpt_auth_for_testing(); - - let servers = with_codex_apps_mcp(HashMap::new(), Some(&auth), &config); - let server = servers - .get(CODEX_APPS_MCP_SERVER_NAME) - .expect("codex apps should be present when apps is enabled"); - let config = server - .configured_config() - .expect("codex apps should use configured transport"); - let url = match &config.transport { - McpServerTransportConfig::StreamableHttp { url, .. } => url, - _ => panic!("expected streamable http transport for codex apps"), - }; - - assert_eq!(url, "https://chatgpt.com/backend-api/custom/mcp"); -} - #[test] fn codex_apps_server_config_forwards_configured_product_sku_header() { - let mut config = test_mcp_config(PathBuf::from("/tmp")); - config.apps_mcp_product_sku = Some("tpp".to_string()); - config.apps_enabled = true; - let auth = CodexAuth::create_dummy_chatgpt_auth_for_testing(); - - let servers = with_codex_apps_mcp(HashMap::new(), Some(&auth), &config); - let server = servers - .get(CODEX_APPS_MCP_SERVER_NAME) - .expect("codex apps should be present when apps is enabled"); - let config = server - .configured_config() - .expect("codex apps should use configured transport"); + let config = codex_apps_mcp_server_config("https://chatgpt.com", Some("tpp")); match &config.transport { McpServerTransportConfig::StreamableHttp { @@ -289,7 +229,7 @@ fn codex_apps_server_config_forwards_configured_product_sku_header() { } #[tokio::test] -async fn effective_mcp_servers_preserve_user_servers_and_add_codex_apps() { +async fn effective_mcp_servers_preserve_runtime_servers() { let codex_home = tempfile::tempdir().expect("tempdir"); let mut config = test_mcp_config(codex_home.path().to_path_buf()); config.apps_enabled = true; @@ -345,6 +285,13 @@ async fn effective_mcp_servers_preserve_user_servers_and_add_codex_apps() { tools: HashMap::new(), }, ); + config.configured_mcp_servers.insert( + CODEX_APPS_MCP_SERVER_NAME.to_string(), + codex_apps_mcp_server_config( + &config.chatgpt_base_url, + config.apps_mcp_product_sku.as_deref(), + ), + ); let effective = effective_mcp_servers(&config, Some(&auth)); diff --git a/codex-rs/config/src/schema.rs b/codex-rs/config/src/schema.rs index 6b012494d9..c641f1703d 100644 --- a/codex-rs/config/src/schema.rs +++ b/codex-rs/config/src/schema.rs @@ -9,6 +9,7 @@ use schemars::schema::ObjectValidation; use schemars::schema::RootSchema; use schemars::schema::Schema; use schemars::schema::SchemaObject; +use schemars::schema::SubschemaValidation; use serde_json::Map; use serde_json::Value; use std::path::Path; @@ -46,9 +47,7 @@ pub fn features_schema(schema_gen: &mut SchemaGenerator) -> Schema { if feature.id == codex_features::Feature::AppsMcpPathOverride { validation.properties.insert( feature.key.to_string(), - schema_gen.subschema_for::>(), + removed_apps_mcp_path_override_schema(schema_gen), ); continue; } @@ -76,6 +75,30 @@ pub fn features_schema(schema_gen: &mut SchemaGenerator) -> Schema { Schema::Object(object) } +fn removed_apps_mcp_path_override_schema(schema_gen: &mut SchemaGenerator) -> Schema { + let mut config_validation = ObjectValidation::default(); + config_validation + .properties + .insert("enabled".to_string(), schema_gen.subschema_for::()); + config_validation + .properties + .insert("path".to_string(), schema_gen.subschema_for::()); + config_validation.additional_properties = Some(Box::new(Schema::Bool(false))); + + let config = Schema::Object(SchemaObject { + instance_type: Some(InstanceType::Object.into()), + object: Some(Box::new(config_validation)), + ..Default::default() + }); + Schema::Object(SchemaObject { + subschemas: Some(Box::new(SubschemaValidation { + any_of: Some(vec![schema_gen.subschema_for::(), config]), + ..Default::default() + })), + ..Default::default() + }) +} + /// Schema for the `[mcp_servers]` map using the raw input shape. pub fn mcp_servers_schema(schema_gen: &mut SchemaGenerator) -> Schema { let mut object = SchemaObject { diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.json index b30b868bc7..9ed02fdd17 100644 --- a/codex-rs/core/config.schema.json +++ b/codex-rs/core/config.schema.json @@ -226,18 +226,6 @@ }, "type": "object" }, - "AppsMcpPathOverrideConfigToml": { - "additionalProperties": false, - "properties": { - "enabled": { - "type": "boolean" - }, - "path": { - "type": "string" - } - }, - "type": "object" - }, "AskForApproval": { "description": "Determines the conditions under which the user is consulted to approve running the command proposed by Codex.", "oneOf": [ @@ -408,7 +396,23 @@ "type": "boolean" }, "apps_mcp_path_override": { - "$ref": "#/definitions/FeatureToml_for_AppsMcpPathOverrideConfigToml" + "anyOf": [ + { + "type": "boolean" + }, + { + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "path": { + "type": "string" + } + }, + "type": "object" + } + ] }, "auth_elicitation": { "type": "boolean" @@ -836,16 +840,6 @@ }, "type": "object" }, - "FeatureToml_for_AppsMcpPathOverrideConfigToml": { - "anyOf": [ - { - "type": "boolean" - }, - { - "$ref": "#/definitions/AppsMcpPathOverrideConfigToml" - } - ] - }, "FeatureToml_for_CodeModeConfigToml": { "anyOf": [ { @@ -4534,7 +4528,23 @@ "type": "boolean" }, "apps_mcp_path_override": { - "$ref": "#/definitions/FeatureToml_for_AppsMcpPathOverrideConfigToml" + "anyOf": [ + { + "type": "boolean" + }, + { + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "path": { + "type": "string" + } + }, + "type": "object" + } + ] }, "auth_elicitation": { "type": "boolean" @@ -5224,4 +5234,4 @@ }, "title": "ConfigToml", "type": "object" -} +} \ No newline at end of file diff --git a/codex-rs/core/src/config/config_tests.rs b/codex-rs/core/src/config/config_tests.rs index 3a3b490d69..c19fe76fb5 100644 --- a/codex-rs/core/src/config/config_tests.rs +++ b/codex-rs/core/src/config/config_tests.rs @@ -5500,14 +5500,9 @@ async fn to_mcp_config_preserves_apps_feature_from_config() -> std::io::Result<( .await?; let plugins_manager = PluginsManager::new(codex_home.path().to_path_buf()); - config.apps_mcp_path_override = Some("/custom/mcp".to_string()); config.apps_mcp_product_sku = Some("tpp".to_string()); let mcp_config = config.to_mcp_config(&plugins_manager).await; assert!(mcp_config.apps_enabled); - assert_eq!( - mcp_config.apps_mcp_path_override.as_deref(), - Some("/custom/mcp") - ); assert_eq!(mcp_config.apps_mcp_product_sku.as_deref(), Some("tpp")); let _ = config.features.disable(Feature::Apps); @@ -8862,84 +8857,6 @@ allow_login_shell = false Ok(()) } -#[tokio::test] -async fn config_loads_apps_mcp_path_override_from_feature_config() -> std::io::Result<()> { - let codex_home = TempDir::new()?; - let toml = r#" -model = "gpt-5.4" - -[features.apps_mcp_path_override] -path = "/custom/mcp" -"#; - let cfg: ConfigToml = - toml::from_str(toml).expect("TOML deserialization should succeed for apps MCP feature"); - - let config = Config::load_from_base_config_with_overrides( - cfg, - ConfigOverrides::default(), - codex_home.abs(), - ) - .await?; - - assert_eq!( - config.apps_mcp_path_override.as_deref(), - Some("/custom/mcp") - ); - Ok(()) -} - -#[tokio::test] -async fn config_defaults_enabled_apps_mcp_path_override_to_plugin_service() -> std::io::Result<()> { - let codex_home = TempDir::new()?; - let toml = r#" -model = "gpt-5.4" - -[features] -apps_mcp_path_override = true -"#; - let cfg: ConfigToml = - toml::from_str(toml).expect("TOML deserialization should succeed for apps MCP feature"); - - let config = Config::load_from_base_config_with_overrides( - cfg, - ConfigOverrides::default(), - codex_home.abs(), - ) - .await?; - - assert!(config.features.enabled(Feature::AppsMcpPathOverride)); - assert_eq!(config.apps_mcp_path_override.as_deref(), Some("/ps/mcp")); - Ok(()) -} - -#[tokio::test] -async fn config_preserves_explicit_apps_mcp_path_override_path() -> std::io::Result<()> { - let codex_home = TempDir::new()?; - let toml = r#" -model = "gpt-5.4" - -[features.apps_mcp_path_override] -enabled = true -path = "/custom/mcp" -"#; - let cfg: ConfigToml = - toml::from_str(toml).expect("TOML deserialization should succeed for apps MCP feature"); - - let config = Config::load_from_base_config_with_overrides( - cfg, - ConfigOverrides::default(), - codex_home.abs(), - ) - .await?; - - assert_eq!( - config.apps_mcp_path_override.as_deref(), - Some("/custom/mcp") - ); - assert!(config.features.enabled(Feature::AppsMcpPathOverride)); - Ok(()) -} - #[tokio::test] async fn config_loads_apps_mcp_product_sku_from_toml() -> std::io::Result<()> { let codex_home = TempDir::new()?; diff --git a/codex-rs/core/src/config/mod.rs b/codex-rs/core/src/config/mod.rs index 5c2a93bc45..cadb797212 100644 --- a/codex-rs/core/src/config/mod.rs +++ b/codex-rs/core/src/config/mod.rs @@ -58,7 +58,6 @@ use codex_config::types::WindowsSandboxModeToml; use codex_core_plugins::PluginsConfigInput; use codex_exec_server::ExecutorFileSystem; use codex_exec_server::LOCAL_FS; -use codex_features::AppsMcpPathOverrideConfigToml; use codex_features::CodeModeConfigToml; use codex_features::Feature; use codex_features::FeatureConfigSource; @@ -931,9 +930,6 @@ pub struct Config { /// Base URL for requests to ChatGPT (as opposed to the OpenAI API). pub chatgpt_base_url: String, - /// Optional path override for the host-owned apps MCP server. - pub apps_mcp_path_override: Option, - /// Optional product SKU forwarded to the host-owned apps MCP server. pub apps_mcp_product_sku: Option, @@ -1415,7 +1411,6 @@ impl Config { McpConfig { chatgpt_base_url: self.chatgpt_base_url.clone(), - apps_mcp_path_override: self.apps_mcp_path_override.clone(), apps_mcp_product_sku: self.apps_mcp_product_sku.clone(), codex_home: self.codex_home.to_path_buf(), mcp_oauth_credentials_store_mode: self.mcp_oauth_credentials_store_mode, @@ -1428,7 +1423,6 @@ impl Config { codex_linux_sandbox_exe: self.codex_linux_sandbox_exe.clone(), use_legacy_landlock: self.features.use_legacy_landlock(), apps_enabled: self.features.enabled(Feature::Apps), - legacy_apps_mcp_loader_enabled: true, prefix_mcp_tool_names: self.prefix_mcp_tool_names(), client_elicitation_capability: if self.features.enabled(Feature::AuthElicitation) { ElicitationCapability { @@ -2413,15 +2407,6 @@ fn multi_agent_v2_toml_config(features: Option<&FeaturesToml>) -> Option<&MultiA } } -fn apps_mcp_path_override_toml_config( - features: Option<&FeaturesToml>, -) -> Option<&AppsMcpPathOverrideConfigToml> { - match features?.apps_mcp_path_override.as_ref()? { - FeatureToml::Enabled(_) => None, - FeatureToml::Config(config) => Some(config), - } -} - fn network_proxy_toml_config(features: Option<&FeaturesToml>) -> Option<&NetworkProxyConfigToml> { match features?.network_proxy.as_ref()? { FeatureToml::Enabled(_) => None, @@ -3043,14 +3028,6 @@ impl Config { resolve_experimental_request_user_input_enabled(&cfg); let code_mode = resolve_code_mode_config(&cfg); let multi_agent_v2 = resolve_multi_agent_v2_config(&cfg); - let apps_mcp_path_override = if features.enabled(Feature::AppsMcpPathOverride) { - let base = apps_mcp_path_override_toml_config(cfg.features.as_ref()); - base.and_then(|config| config.path.as_ref()) - .cloned() - .or_else(|| Some("/ps/mcp".to_string())) - } else { - None - }; let terminal_resize_reflow = resolve_terminal_resize_reflow_config(&cfg); let agent_roles = @@ -3553,7 +3530,6 @@ impl Config { chatgpt_base_url: cfg .chatgpt_base_url .unwrap_or("https://chatgpt.com/backend-api/".to_string()), - apps_mcp_path_override, apps_mcp_product_sku: cfg.apps_mcp_product_sku.clone(), realtime_audio: cfg .audio diff --git a/codex-rs/core/src/config_lock.rs b/codex-rs/core/src/config_lock.rs index 14ee7e1105..f99ded0bf8 100644 --- a/codex-rs/core/src/config_lock.rs +++ b/codex-rs/core/src/config_lock.rs @@ -125,6 +125,9 @@ fn config_lock_for_comparison( ) -> ConfigLockfileToml { let mut lockfile = lockfile.clone(); clear_config_lock_debug_controls(&mut lockfile.config); + if let Some(features) = lockfile.config.features.as_mut() { + features.clear_removed_compatibility_entries(); + } if options.allow_codex_version_mismatch { lockfile.codex_version.clear(); } diff --git a/codex-rs/core/src/mcp.rs b/codex-rs/core/src/mcp.rs index 1b693d70fd..71571deaec 100644 --- a/codex-rs/core/src/mcp.rs +++ b/codex-rs/core/src/mcp.rs @@ -11,6 +11,7 @@ use codex_mcp::CODEX_APPS_MCP_SERVER_NAME; use codex_mcp::EffectiveMcpServer; use codex_mcp::McpConfig; use codex_mcp::ToolPluginProvenance; +use codex_mcp::codex_apps_mcp_server_config; use codex_mcp::configured_mcp_servers; use codex_mcp::effective_mcp_servers; use codex_mcp::tool_plugin_provenance as collect_tool_plugin_provenance; @@ -40,16 +41,24 @@ impl McpManager { } } - /// Returns the MCP config after applying runtime-only extension overlays. + /// Returns the MCP config after applying compatibility built-ins and + /// runtime-only extension overlays. pub async fn runtime_config(&self, config: &Config) -> McpConfig { let mut mcp_config = config.to_mcp_config(self.plugins_manager.as_ref()).await; - let contributions = self.contributions(config).await; - if contributions - .iter() - .any(|contribution| contribution.name() == CODEX_APPS_MCP_SERVER_NAME) - { - mcp_config.legacy_apps_mcp_loader_enabled = false; + if mcp_config.apps_enabled { + mcp_config.configured_mcp_servers.insert( + CODEX_APPS_MCP_SERVER_NAME.to_string(), + codex_apps_mcp_server_config( + &mcp_config.chatgpt_base_url, + mcp_config.apps_mcp_product_sku.as_deref(), + ), + ); + } else { + mcp_config + .configured_mcp_servers + .remove(CODEX_APPS_MCP_SERVER_NAME); } + let contributions = self.contributions(config).await; Self::apply_to_configured_servers(&contributions, &mut mcp_config.configured_mcp_servers); mcp_config } diff --git a/codex-rs/core/src/session/config_lock.rs b/codex-rs/core/src/session/config_lock.rs index c081a2aee6..ec2056581c 100644 --- a/codex-rs/core/src/session/config_lock.rs +++ b/codex-rs/core/src/session/config_lock.rs @@ -2,7 +2,6 @@ use anyhow::Context; use codex_config::config_toml::ConfigLockfileToml; use codex_config::config_toml::ConfigToml; use codex_config::types::MemoriesToml; -use codex_features::AppsMcpPathOverrideConfigToml; use codex_features::Feature; use codex_features::FeatureToml; use codex_features::FeaturesToml; @@ -149,10 +148,6 @@ fn save_config_resolved_fields( resolved_config_to_toml(&config.multi_agent_v2, "features.multi_agent_v2")?; multi_agent_v2.enabled = Some(config.features.enabled(Feature::MultiAgentV2)); features.multi_agent_v2 = Some(FeatureToml::Config(multi_agent_v2)); - features.apps_mcp_path_override = Some(FeatureToml::Config(AppsMcpPathOverrideConfigToml { - enabled: Some(config.features.enabled(Feature::AppsMcpPathOverride)), - path: config.apps_mcp_path_override.clone(), - })); lock_config.memories = Some(resolved_config_to_toml::( &config.memories, "memories", @@ -325,6 +320,32 @@ mod tests { assert!(message.contains("model = "), "{message}"); } + #[tokio::test] + async fn lock_validation_ignores_removed_apps_mcp_path_override() { + let sc = crate::session::tests::make_session_configuration_for_tests().await; + let actual = sc.to_config_lockfile_toml().expect("lock should serialize"); + let mut expected_value = toml::Value::try_from(&actual).expect("lock should become TOML"); + expected_value["config"]["features"] + .as_table_mut() + .expect("features should be a table") + .insert( + "apps_mcp_path_override".to_string(), + toml::Value::Table(toml::Table::from_iter([ + ("enabled".to_string(), toml::Value::Boolean(true)), + ( + "path".to_string(), + toml::Value::String("/custom/mcp".to_string()), + ), + ])), + ); + let expected: ConfigLockfileToml = expected_value + .try_into() + .expect("lock with removed input should deserialize"); + + validate_config_lock_replay(&expected, &actual, ConfigLockReplayOptions::default()) + .expect("removed compatibility input should not cause lock drift"); + } + #[tokio::test] async fn lock_validation_rejects_codex_version_mismatch_by_default() { let sc = crate::session::tests::make_session_configuration_for_tests().await; diff --git a/codex-rs/core/src/tools/handlers/mcp_resource.rs b/codex-rs/core/src/tools/handlers/mcp_resource.rs index 9357a9722d..e2f83859b6 100644 --- a/codex-rs/core/src/tools/handlers/mcp_resource.rs +++ b/codex-rs/core/src/tools/handlers/mcp_resource.rs @@ -7,6 +7,8 @@ use codex_protocol::items::McpToolCallItem; use codex_protocol::items::McpToolCallStatus; use codex_protocol::items::TurnItem; use codex_protocol::mcp::CallToolResult; +use codex_protocol::protocol::TruncationPolicy; +use codex_utils_output_truncation::truncate_text; use rmcp::model::ListResourceTemplatesResult; use rmcp::model::ListResourcesResult; use rmcp::model::ReadResourceResult; @@ -270,7 +272,10 @@ fn normalize_required_string(field: &str, value: String) -> Result(payload: T) -> Result +fn serialize_function_output( + payload: T, + truncation_policy: TruncationPolicy, +) -> Result where T: Serialize, { @@ -279,6 +284,9 @@ where "failed to serialize MCP resource response: {err}" )) })?; + // Match regular MCP tool outputs by bounding the copy persisted to the + // rollout and injected into model context. + let content = truncate_text(&content, truncation_policy * 1.2); Ok(FunctionToolOutput::from_text(content, Some(true))) } diff --git a/codex-rs/core/src/tools/handlers/mcp_resource/list_mcp_resource_templates.rs b/codex-rs/core/src/tools/handlers/mcp_resource/list_mcp_resource_templates.rs index b1e82aa8a6..c46df13b18 100644 --- a/codex-rs/core/src/tools/handlers/mcp_resource/list_mcp_resource_templates.rs +++ b/codex-rs/core/src/tools/handlers/mcp_resource/list_mcp_resource_templates.rs @@ -117,7 +117,7 @@ impl ToolExecutor for ListMcpResourceTemplatesHandler { .await; match payload_result { - Ok(payload) => match serialize_function_output(payload) { + Ok(payload) => match serialize_function_output(payload, turn.truncation_policy) { Ok(output) => { let content = function_call_output_content_items_to_text(&output.body) .unwrap_or_default(); diff --git a/codex-rs/core/src/tools/handlers/mcp_resource/list_mcp_resources.rs b/codex-rs/core/src/tools/handlers/mcp_resource/list_mcp_resources.rs index c2cdec8b15..f7fcad3422 100644 --- a/codex-rs/core/src/tools/handlers/mcp_resource/list_mcp_resources.rs +++ b/codex-rs/core/src/tools/handlers/mcp_resource/list_mcp_resources.rs @@ -115,7 +115,7 @@ impl ToolExecutor for ListMcpResourcesHandler { .await; match payload_result { - Ok(payload) => match serialize_function_output(payload) { + Ok(payload) => match serialize_function_output(payload, turn.truncation_policy) { Ok(output) => { let content = function_call_output_content_items_to_text(&output.body) .unwrap_or_default(); diff --git a/codex-rs/core/src/tools/handlers/mcp_resource/read_mcp_resource.rs b/codex-rs/core/src/tools/handlers/mcp_resource/read_mcp_resource.rs index 126c5d85e8..f059d12c84 100644 --- a/codex-rs/core/src/tools/handlers/mcp_resource/read_mcp_resource.rs +++ b/codex-rs/core/src/tools/handlers/mcp_resource/read_mcp_resource.rs @@ -93,7 +93,7 @@ impl ToolExecutor for ReadMcpResourceHandler { .await; match payload_result { - Ok(payload) => match serialize_function_output(payload) { + Ok(payload) => match serialize_function_output(payload, turn.truncation_policy) { Ok(output) => { let content = function_call_output_content_items_to_text(&output.body) .unwrap_or_default(); diff --git a/codex-rs/core/src/tools/handlers/mcp_resource_tests.rs b/codex-rs/core/src/tools/handlers/mcp_resource_tests.rs index 8a8410b0bd..b52ab1681a 100644 --- a/codex-rs/core/src/tools/handlers/mcp_resource_tests.rs +++ b/codex-rs/core/src/tools/handlers/mcp_resource_tests.rs @@ -1,6 +1,7 @@ use super::*; use pretty_assertions::assert_eq; use rmcp::model::AnnotateAble; +use rmcp::model::ResourceContents; use serde_json::json; fn resource(uri: &str, name: &str) -> Resource { @@ -123,3 +124,39 @@ fn template_with_server_serializes_server_field() { }) ); } + +#[test] +fn serialize_function_output_preserves_small_payload() { + let payload = json!({"server": "hosted", "resources": []}); + let expected = serde_json::to_string(&payload).expect("serialize payload"); + + let output = serialize_function_output(payload, TruncationPolicy::Bytes(1_024)) + .expect("serialize function output") + .into_text(); + + assert_eq!(output, expected); +} + +#[test] +fn serialize_function_output_caps_read_resource_payload() { + let truncation_policy = TruncationPolicy::Bytes(8_000); + let payload = ReadResourcePayload { + server: "hosted".to_string(), + uri: "skill://large/SKILL.md".to_string(), + result: ReadResourceResult::new(vec![ResourceContents::TextResourceContents { + uri: "skill://large/SKILL.md".to_string(), + mime_type: Some("text/markdown".to_string()), + text: "x".repeat(16_000), + meta: None, + }]), + }; + let serialized = serde_json::to_string(&payload).expect("serialize payload"); + let expected = truncate_text(&serialized, truncation_policy * 1.2); + + let output = serialize_function_output(payload, truncation_policy) + .expect("serialize bounded function output") + .into_text(); + + assert_ne!(output, serialized); + assert_eq!(output, expected); +} diff --git a/codex-rs/ext/extension-api/src/contributors/mcp.rs b/codex-rs/ext/extension-api/src/contributors/mcp.rs index a55715692b..7fdedcdda7 100644 --- a/codex-rs/ext/extension-api/src/contributors/mcp.rs +++ b/codex-rs/ext/extension-api/src/contributors/mcp.rs @@ -11,12 +11,3 @@ pub enum McpServerContribution { /// Removes a named MCP server. Remove { name: String }, } - -impl McpServerContribution { - /// Returns the stable server name owned by this contribution. - pub fn name(&self) -> &str { - match self { - Self::Set { name, .. } | Self::Remove { name } => name, - } - } -} diff --git a/codex-rs/ext/mcp/src/lib.rs b/codex-rs/ext/mcp/src/lib.rs index 4a7838ccce..8fad361a4e 100644 --- a/codex-rs/ext/mcp/src/lib.rs +++ b/codex-rs/ext/mcp/src/lib.rs @@ -3,12 +3,12 @@ use codex_extension_api::ExtensionRegistryBuilder; use codex_extension_api::McpServerContribution; use codex_extension_api::McpServerContributor; use codex_mcp::CODEX_APPS_MCP_SERVER_NAME; -use codex_mcp::codex_apps_mcp_server_config; +use codex_mcp::hosted_plugin_runtime_mcp_server_config; -struct HostedAppsMcpExtension; +struct HostedPluginRuntimeExtension; #[async_trait::async_trait] -impl McpServerContributor for HostedAppsMcpExtension { +impl McpServerContributor for HostedPluginRuntimeExtension { async fn contribute(&self, config: &Config) -> Vec { let name = CODEX_APPS_MCP_SERVER_NAME.to_string(); if !config.features.enabled(codex_features::Feature::Apps) { @@ -17,9 +17,8 @@ impl McpServerContributor for HostedAppsMcpExtension { vec![McpServerContribution::Set { name, - config: Box::new(codex_apps_mcp_server_config( + config: Box::new(hosted_plugin_runtime_mcp_server_config( &config.chatgpt_base_url, - config.apps_mcp_path_override.as_deref(), config.apps_mcp_product_sku.as_deref(), )), }] @@ -27,5 +26,5 @@ impl McpServerContributor for HostedAppsMcpExtension { } pub fn install(builder: &mut ExtensionRegistryBuilder) { - builder.mcp_server_contributor(std::sync::Arc::new(HostedAppsMcpExtension)); + builder.mcp_server_contributor(std::sync::Arc::new(HostedPluginRuntimeExtension)); } diff --git a/codex-rs/ext/mcp/tests/hosted_apps_mcp.rs b/codex-rs/ext/mcp/tests/hosted_apps_mcp.rs index 0d025df954..bbfa07c14c 100644 --- a/codex-rs/ext/mcp/tests/hosted_apps_mcp.rs +++ b/codex-rs/ext/mcp/tests/hosted_apps_mcp.rs @@ -6,20 +6,22 @@ use codex_core::config::Config; use codex_core::config::ConfigBuilder; use codex_core_plugins::PluginsManager; use codex_extension_api::ExtensionRegistryBuilder; +use codex_extension_api::McpServerContribution; +use codex_extension_api::McpServerContributor; use codex_login::CodexAuth; use codex_mcp::CODEX_APPS_MCP_SERVER_NAME; use pretty_assertions::assert_eq; +type TestResult = Result<(), Box>; + #[tokio::test] -async fn contributes_hosted_apps_mcp_without_an_executor() -> Result<(), Box> -{ +async fn contributes_hosted_plugin_runtime_without_an_executor() -> TestResult { let codex_home = tempfile::tempdir()?; let config = ConfigBuilder::default() .codex_home(codex_home.path().to_path_buf()) .fallback_cwd(Some(codex_home.path().to_path_buf())) .cli_overrides(vec![ ("features.apps".to_string(), true.into()), - ("features.apps_mcp_path_override".to_string(), true.into()), ("chatgpt_base_url".to_string(), "https://chatgpt.com".into()), ]) .build() @@ -27,15 +29,13 @@ async fn contributes_hosted_apps_mcp_without_an_executor() -> Result<(), Box Result<(), Box Result<(), Box> { +async fn legacy_fallback_overwrites_reserved_config_without_an_extension() -> TestResult { + let codex_home = tempfile::tempdir()?; + let config = ConfigBuilder::default() + .codex_home(codex_home.path().to_path_buf()) + .fallback_cwd(Some(codex_home.path().to_path_buf())) + .cli_overrides(vec![ + ("features.apps".to_string(), true.into()), + ( + "mcp_servers.codex_apps.url".to_string(), + "https://example.com/mcp".into(), + ), + ]) + .build() + .await?; + let auth = CodexAuth::create_dummy_chatgpt_auth_for_testing(); + let manager = McpManager::new(Arc::new(PluginsManager::new( + config.codex_home.to_path_buf(), + ))); + + let servers = manager.effective_servers(&config, Some(&auth)).await; + let server = servers + .get(CODEX_APPS_MCP_SERVER_NAME) + .and_then(|server| server.configured_config()) + .ok_or("legacy Apps MCP should be present")?; + let McpServerTransportConfig::StreamableHttp { url, .. } = &server.transport else { + panic!("legacy Apps MCP should use streamable HTTP"); + }; + assert_eq!(url, "https://chatgpt.com/backend-api/wham/apps"); + + Ok(()) +} + +#[tokio::test] +async fn extension_can_remove_legacy_fallback_while_apps_are_enabled() -> TestResult { + let codex_home = tempfile::tempdir()?; + let config = ConfigBuilder::default() + .codex_home(codex_home.path().to_path_buf()) + .fallback_cwd(Some(codex_home.path().to_path_buf())) + .cli_overrides(vec![("features.apps".to_string(), true.into())]) + .build() + .await?; + let auth = CodexAuth::create_dummy_chatgpt_auth_for_testing(); + let mut builder = ExtensionRegistryBuilder::new(); + builder.mcp_server_contributor(Arc::new(RemoveCodexApps)); + let manager = McpManager::new_with_extensions( + Arc::new(PluginsManager::new(config.codex_home.to_path_buf())), + Arc::new(builder.build()), + ); + + let servers = manager.effective_servers(&config, Some(&auth)).await; + + assert!(!servers.contains_key(CODEX_APPS_MCP_SERVER_NAME)); + Ok(()) +} + +#[tokio::test] +async fn hosted_apps_mcp_requires_chatgpt_auth() -> TestResult { let codex_home = tempfile::tempdir()?; let config = ConfigBuilder::default() .codex_home(codex_home.path().to_path_buf()) @@ -61,7 +117,7 @@ async fn hosted_apps_mcp_requires_chatgpt_auth() -> Result<(), Box Result<(), Box> { +async fn disabled_apps_remove_reserved_server_config() -> TestResult { let codex_home = tempfile::tempdir()?; let config = ConfigBuilder::default() .codex_home(codex_home.path().to_path_buf()) @@ -91,3 +147,14 @@ fn installed_manager(config: &Config) -> McpManager { Arc::new(builder.build()), ) } + +struct RemoveCodexApps; + +#[async_trait::async_trait] +impl McpServerContributor for RemoveCodexApps { + async fn contribute(&self, _config: &Config) -> Vec { + vec![McpServerContribution::Remove { + name: CODEX_APPS_MCP_SERVER_NAME.to_string(), + }] + } +} diff --git a/codex-rs/features/src/feature_configs.rs b/codex-rs/features/src/feature_configs.rs index b7f666c21d..f59b89e9a8 100644 --- a/codex-rs/features/src/feature_configs.rs +++ b/codex-rs/features/src/feature_configs.rs @@ -70,21 +70,11 @@ impl FeatureConfig for MultiAgentV2ConfigToml { #[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq, JsonSchema)] #[serde(deny_unknown_fields)] -pub struct AppsMcpPathOverrideConfigToml { +pub(crate) struct RemovedAppsMcpPathOverrideConfigToml { #[serde(skip_serializing_if = "Option::is_none")] - pub enabled: Option, + enabled: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub path: Option, -} - -impl FeatureConfig for AppsMcpPathOverrideConfigToml { - fn enabled(&self) -> Option { - self.enabled.or(self.path.as_ref().map(|_| true)) - } - - fn set_enabled(&mut self, enabled: bool) { - self.enabled = Some(enabled); - } + path: Option, } #[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq, JsonSchema)] diff --git a/codex-rs/features/src/lib.rs b/codex-rs/features/src/lib.rs index 8ee05aed21..1777ceef06 100644 --- a/codex-rs/features/src/lib.rs +++ b/codex-rs/features/src/lib.rs @@ -16,13 +16,13 @@ use toml::Table; mod feature_configs; mod legacy; -pub use feature_configs::AppsMcpPathOverrideConfigToml; pub use feature_configs::CodeModeConfigToml; pub use feature_configs::MultiAgentV2ConfigToml; pub use feature_configs::NetworkProxyConfigToml; pub use feature_configs::NetworkProxyDomainPermissionToml; pub use feature_configs::NetworkProxyModeToml; pub use feature_configs::NetworkProxyUnixSocketPermissionToml; +use feature_configs::RemovedAppsMcpPathOverrideConfigToml; use legacy::LegacyFeatureToggles; pub use legacy::legacy_feature_keys; @@ -143,7 +143,7 @@ pub enum Feature { Apps, /// Enable MCP apps. EnableMcpApps, - /// Use the new path for the host-owned apps MCP server. + /// Removed compatibility flag for the legacy Apps MCP path override. AppsMcpPathOverride, /// Removed compatibility flag retained as a no-op now that tool_search is always enabled. ToolSearch, @@ -443,7 +443,7 @@ impl Features { "apply_patch_freeform" => { continue; } - "tool_search" => { + "tool_search" | "apps_mcp_path_override" => { continue; } "image_detail_original" => { @@ -605,8 +605,9 @@ pub struct FeaturesToml { pub code_mode: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] pub multi_agent_v2: Option>, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub apps_mcp_path_override: Option>, + #[serde(default, rename = "apps_mcp_path_override", skip_serializing)] + #[schemars(skip)] + removed_apps_mcp_path_override: Option>, pub network_proxy: Option>, /// Boolean feature toggles keyed by canonical or legacy feature name. #[serde(flatten)] @@ -621,6 +622,13 @@ impl Features { } impl FeaturesToml { + /// Removes compatibility-only inputs that no longer affect runtime + /// behavior or belong in newly materialized config. + pub fn clear_removed_compatibility_entries(&mut self) { + self.removed_apps_mcp_path_override = None; + self.entries.remove("apps_mcp_path_override"); + } + pub fn entries(&self) -> BTreeMap { let mut entries = self.entries.clone(); if let Some(enabled) = self.code_mode.as_ref().and_then(FeatureToml::enabled) { @@ -629,13 +637,6 @@ impl FeaturesToml { if let Some(enabled) = self.multi_agent_v2.as_ref().and_then(FeatureToml::enabled) { entries.insert(Feature::MultiAgentV2.key().to_string(), enabled); } - if let Some(enabled) = self - .apps_mcp_path_override - .as_ref() - .and_then(FeatureToml::enabled) - { - entries.insert(Feature::AppsMcpPathOverride.key().to_string(), enabled); - } if let Some(enabled) = self.network_proxy.as_ref().and_then(FeatureToml::enabled) { entries.insert(Feature::NetworkProxy.key().to_string(), enabled); } @@ -643,10 +644,11 @@ impl FeaturesToml { } pub fn materialize_resolved_enabled(&mut self, features: &Features) { + self.clear_removed_compatibility_entries(); let Self { code_mode, multi_agent_v2, - apps_mcp_path_override, + removed_apps_mcp_path_override: _, network_proxy, entries, } = self; @@ -659,8 +661,6 @@ impl FeaturesToml { materialize_resolved_feature_enabled(code_mode, enabled); } else if spec.id == Feature::MultiAgentV2 { materialize_resolved_feature_enabled(multi_agent_v2, enabled); - } else if spec.id == Feature::AppsMcpPathOverride { - materialize_resolved_feature_enabled(apps_mcp_path_override, enabled); } else if spec.id == Feature::NetworkProxy { materialize_resolved_feature_enabled(network_proxy, enabled); } else { @@ -987,7 +987,7 @@ pub const FEATURES: &[FeatureSpec] = &[ FeatureSpec { id: Feature::AppsMcpPathOverride, key: "apps_mcp_path_override", - stage: Stage::UnderDevelopment, + stage: Stage::Removed, default_enabled: false, }, FeatureSpec { diff --git a/codex-rs/features/src/tests.rs b/codex-rs/features/src/tests.rs index aa5e0174ba..e230c45352 100644 --- a/codex-rs/features/src/tests.rs +++ b/codex-rs/features/src/tests.rs @@ -83,6 +83,27 @@ fn plugin_hooks_is_removed_and_disabled_by_default() { assert_eq!(feature_for_key("plugin_hooks"), Some(Feature::PluginHooks)); } +#[test] +fn removed_apps_mcp_path_override_shapes_are_ignored() { + let features = [ + toml::from_str::("apps_mcp_path_override = true") + .expect("boolean compatibility form should deserialize"), + toml::from_str::( + r#" +[apps_mcp_path_override] +enabled = true +path = "/custom/mcp" +"#, + ) + .expect("structured compatibility form should deserialize"), + ]; + + assert_eq!( + features.map(|features| features.entries()), + [BTreeMap::new(), BTreeMap::new()] + ); +} + #[test] fn code_mode_only_requires_code_mode() { let mut features = Features::with_defaults(); diff --git a/codex-rs/thread-manager-sample/src/main.rs b/codex-rs/thread-manager-sample/src/main.rs index 97bfea8280..1bea0cae8b 100644 --- a/codex-rs/thread-manager-sample/src/main.rs +++ b/codex-rs/thread-manager-sample/src/main.rs @@ -251,7 +251,6 @@ fn new_config(model: Option, arg0_paths: Arg0DispatchPaths) -> anyhow::R model_catalog: None, model_verbosity: None, chatgpt_base_url: "https://chatgpt.com/backend-api/".to_string(), - apps_mcp_path_override: None, apps_mcp_product_sku: None, realtime_audio: RealtimeAudioConfig::default(), experimental_realtime_ws_base_url: None, From ced1b8aa883f25b992f9ebe7d218f3709926f912 Mon Sep 17 00:00:00 2001 From: jif Date: Wed, 10 Jun 2026 13:06:48 +0200 Subject: [PATCH 29/47] [codex] Tag multi-agent spawn metrics with version (#27375) ## Summary - tag legacy multi-agent spawn metrics with `version=v1` - tag multi-agent v2 spawn metrics with `version=v2` ## Why `codex.multi_agent.spawn` is emitted by both runtimes, so the existing metric cannot distinguish v2 adoption from aggregate multi-agent spawning. The bounded version tag makes that breakdown directly queryable without changing the counter's success-only semantics. ## Validation - `just fmt` - `git diff --check` - Tests and Clippy were intentionally left to CI. --- codex-rs/core/src/tools/handlers/multi_agents/spawn.rs | 2 +- codex-rs/core/src/tools/handlers/multi_agents_v2/spawn.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/codex-rs/core/src/tools/handlers/multi_agents/spawn.rs b/codex-rs/core/src/tools/handlers/multi_agents/spawn.rs index a50d17f34a..3f6c97fcbc 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents/spawn.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents/spawn.rs @@ -204,7 +204,7 @@ async fn handle_spawn_agent( turn.session_telemetry.counter( "codex.multi_agent.spawn", /*inc*/ 1, - &[("role", role_tag)], + &[("role", role_tag), ("version", "v1")], ); Ok(SpawnAgentResult { diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/spawn.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/spawn.rs index fc97df05dc..34fddc60b4 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/spawn.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/spawn.rs @@ -165,7 +165,7 @@ async fn handle_spawn_agent( turn.session_telemetry.counter( "codex.multi_agent.spawn", /*inc*/ 1, - &[("role", role_tag)], + &[("role", role_tag), ("version", "v2")], ); let task_name = String::from(new_agent_path); From d2f6d23c6cb07f72d18dfd0d8ca09c807870e677 Mon Sep 17 00:00:00 2001 From: jif Date: Wed, 10 Jun 2026 13:31:09 +0100 Subject: [PATCH 30/47] Remove async-trait from extension contributors (#27383) ## Why Extension contributors are registered behind `dyn Trait` objects, so native `async fn`/RPITIT methods would make these traits non-object-safe. Spell out the boxed, `Send` future contract directly so `extension-api` no longer needs `async-trait` while retaining the existing runtime model. ## What changed - add a shared `ExtensionFuture` alias and use it for asynchronous contributor methods - migrate production and test implementations to return `Box::pin(async move { ... })` - remove `async-trait` dependencies where they are no longer used, keeping it dev-only where unrelated test executors still require it ## Behavior No behavior change is intended. Contributor futures remain boxed, `Send`, dynamically dispatched, and lazily executed; cancellation and callback ordering stay unchanged. ## Testing - `just test -p codex-extension-api` (11 passed) - affected extension crates (64 passed) - targeted `codex-core` contributor tests (14 passed) - `just fmt` - `just bazel-lock-update` - `just bazel-lock-check` A broad local `codex-core` run compiled successfully but encountered unrelated sandbox and missing test-binary fixture failures; CI will run the full checks. --- codex-rs/Cargo.lock | 3 - codex-rs/core/src/session/tests.rs | 227 ++++++----- codex-rs/core/src/session/turn_tests.rs | 27 +- .../core/src/stream_events_utils_tests.rs | 58 +-- codex-rs/core/src/thread_manager_tests.rs | 30 +- .../src/tools/handlers/extension_tools.rs | 19 +- codex-rs/ext/extension-api/Cargo.toml | 1 - .../ext/extension-api/src/contributors.rs | 125 ++++-- codex-rs/ext/extension-api/src/lib.rs | 1 + codex-rs/ext/extension-api/tests/registry.rs | 127 +++--- codex-rs/ext/goal/src/extension.rs | 383 +++++++++--------- codex-rs/ext/guardian/Cargo.toml | 1 - codex-rs/ext/guardian/src/lib.rs | 22 +- .../ext/image-generation/src/extension.rs | 15 +- codex-rs/ext/mcp/Cargo.toml | 1 - codex-rs/ext/mcp/src/lib.rs | 31 +- codex-rs/ext/mcp/tests/hosted_apps_mcp.rs | 14 +- codex-rs/ext/memories/src/extension.rs | 15 +- codex-rs/ext/skills/Cargo.toml | 2 +- codex-rs/ext/skills/src/extension.rs | 216 +++++----- codex-rs/ext/web-search/src/extension.rs | 15 +- 21 files changed, 736 insertions(+), 597 deletions(-) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 46bb21bf62..17b2696cbf 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -2921,7 +2921,6 @@ dependencies = [ name = "codex-extension-api" version = "0.0.0" dependencies = [ - "async-trait", "codex-config", "codex-context-fragments", "codex-protocol", @@ -3070,7 +3069,6 @@ dependencies = [ name = "codex-guardian" version = "0.0.0" dependencies = [ - "async-trait", "codex-core", "codex-extension-api", "codex-protocol", @@ -3255,7 +3253,6 @@ dependencies = [ name = "codex-mcp-extension" version = "0.0.0" dependencies = [ - "async-trait", "codex-config", "codex-core", "codex-core-plugins", diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs index f6166260a0..b83217293d 100644 --- a/codex-rs/core/src/session/tests.rs +++ b/codex-rs/core/src/session/tests.rs @@ -1922,26 +1922,27 @@ async fn record_token_usage_info_notifies_extension_contributors() { records: Arc>>, } - #[async_trait::async_trait] impl codex_extension_api::TokenUsageContributor for TokenUsageRecorder { - async fn on_token_usage( - &self, - session_store: &codex_extension_api::ExtensionData, - thread_store: &codex_extension_api::ExtensionData, - turn_store: &codex_extension_api::ExtensionData, - token_usage: &TokenUsageInfo, - ) { - self.records - .lock() - .expect("token usage records lock") - .push(RecordedTokenUsage { - session_level_id: session_store.level_id().to_string(), - thread_level_id: thread_store.level_id().to_string(), - turn_level_id: turn_store.level_id().to_string(), - token_usage: token_usage.clone(), - saw_session_store: session_store.get::().is_some(), - saw_thread_store: thread_store.get::().is_some(), - }); + fn on_token_usage<'a>( + &'a self, + session_store: &'a codex_extension_api::ExtensionData, + thread_store: &'a codex_extension_api::ExtensionData, + turn_store: &'a codex_extension_api::ExtensionData, + token_usage: &'a TokenUsageInfo, + ) -> codex_extension_api::ExtensionFuture<'a, ()> { + Box::pin(async move { + self.records + .lock() + .expect("token usage records lock") + .push(RecordedTokenUsage { + session_level_id: session_store.level_id().to_string(), + thread_level_id: thread_store.level_id().to_string(), + turn_level_id: turn_store.level_id().to_string(), + token_usage: token_usage.clone(), + saw_session_store: session_store.get::().is_some(), + saw_thread_store: thread_store.get::().is_some(), + }); + }) } } @@ -2040,25 +2041,32 @@ async fn turn_start_lifecycle_exposes_turn_metadata_and_token_baseline() { records: Arc>>, } - #[async_trait::async_trait] impl codex_extension_api::TurnLifecycleContributor for TurnStartRecorder { - async fn on_turn_start(&self, input: codex_extension_api::TurnStartInput<'_>) { - self.records - .lock() - .expect("turn start records lock") - .push(RecordedTurnStart { - session_level_id: input.session_store.level_id().to_string(), - thread_level_id: input.thread_store.level_id().to_string(), - turn_level_id: input.turn_store.level_id().to_string(), - turn_id: input.turn_id.to_string(), - collaboration_mode: input.collaboration_mode.clone(), - token_usage_at_turn_start: input.token_usage_at_turn_start.clone(), - saw_session_store: input - .session_store - .get::() - .is_some(), - saw_thread_store: input.thread_store.get::().is_some(), - }); + fn on_turn_start<'a>( + &'a self, + input: codex_extension_api::TurnStartInput<'a>, + ) -> codex_extension_api::ExtensionFuture<'a, ()> { + Box::pin(async move { + self.records + .lock() + .expect("turn start records lock") + .push(RecordedTurnStart { + session_level_id: input.session_store.level_id().to_string(), + thread_level_id: input.thread_store.level_id().to_string(), + turn_level_id: input.turn_store.level_id().to_string(), + turn_id: input.turn_id.to_string(), + collaboration_mode: input.collaboration_mode.clone(), + token_usage_at_turn_start: input.token_usage_at_turn_start.clone(), + saw_session_store: input + .session_store + .get::() + .is_some(), + saw_thread_store: input + .thread_store + .get::() + .is_some(), + }); + }) } } @@ -2138,24 +2146,31 @@ async fn turn_error_lifecycle_exposes_error_and_stores() { records: Arc>>, } - #[async_trait::async_trait] impl codex_extension_api::TurnLifecycleContributor for TurnErrorRecorder { - async fn on_turn_error(&self, input: codex_extension_api::TurnErrorInput<'_>) { - self.records - .lock() - .expect("turn error records lock") - .push(RecordedTurnError { - session_level_id: input.session_store.level_id().to_string(), - thread_level_id: input.thread_store.level_id().to_string(), - turn_level_id: input.turn_store.level_id().to_string(), - turn_id: input.turn_id.to_string(), - error: input.error, - saw_session_store: input - .session_store - .get::() - .is_some(), - saw_thread_store: input.thread_store.get::().is_some(), - }); + fn on_turn_error<'a>( + &'a self, + input: codex_extension_api::TurnErrorInput<'a>, + ) -> codex_extension_api::ExtensionFuture<'a, ()> { + Box::pin(async move { + self.records + .lock() + .expect("turn error records lock") + .push(RecordedTurnError { + session_level_id: input.session_store.level_id().to_string(), + thread_level_id: input.thread_store.level_id().to_string(), + turn_level_id: input.turn_store.level_id().to_string(), + turn_id: input.turn_id.to_string(), + error: input.error, + saw_session_store: input + .session_store + .get::() + .is_some(), + saw_thread_store: input + .thread_store + .get::() + .is_some(), + }); + }) } } @@ -6411,16 +6426,20 @@ async fn submission_loop_channel_close_emits_thread_stop_lifecycle() { expected_thread_id: ThreadId, } - #[async_trait::async_trait] impl codex_extension_api::ThreadLifecycleContributor for ThreadStopRecorder { - async fn on_thread_stop(&self, input: codex_extension_api::ThreadStopInput<'_>) { - assert_eq!( - self.expected_thread_id.to_string(), - input.thread_store.level_id() - ); - assert!(input.session_store.get::().is_some()); - assert!(input.thread_store.get::().is_some()); - self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + fn on_thread_stop<'a>( + &'a self, + input: codex_extension_api::ThreadStopInput<'a>, + ) -> codex_extension_api::ExtensionFuture<'a, ()> { + Box::pin(async move { + assert_eq!( + self.expected_thread_id.to_string(), + input.thread_store.level_id() + ); + assert!(input.session_store.get::().is_some()); + assert!(input.thread_store.get::().is_some()); + self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + }) } } @@ -6457,33 +6476,41 @@ async fn submission_loop_channel_close_aborts_active_turn_before_thread_stop_lif expected_turn_id: String, } - #[async_trait::async_trait] impl codex_extension_api::ThreadLifecycleContributor for LifecycleRecorder { - async fn on_thread_stop(&self, input: codex_extension_api::ThreadStopInput<'_>) { - assert_eq!( - self.expected_thread_id.to_string(), - input.thread_store.level_id() - ); - self.calls - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .push("thread_stop"); + fn on_thread_stop<'a>( + &'a self, + input: codex_extension_api::ThreadStopInput<'a>, + ) -> codex_extension_api::ExtensionFuture<'a, ()> { + Box::pin(async move { + assert_eq!( + self.expected_thread_id.to_string(), + input.thread_store.level_id() + ); + self.calls + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .push("thread_stop"); + }) } } - #[async_trait::async_trait] impl codex_extension_api::TurnLifecycleContributor for LifecycleRecorder { - async fn on_turn_abort(&self, input: codex_extension_api::TurnAbortInput<'_>) { - assert_eq!( - self.expected_thread_id.to_string(), - input.thread_store.level_id() - ); - assert_eq!(self.expected_turn_id, input.turn_store.level_id()); - assert_eq!(TurnAbortReason::Interrupted, input.reason); - self.calls - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .push("turn_abort"); + fn on_turn_abort<'a>( + &'a self, + input: codex_extension_api::TurnAbortInput<'a>, + ) -> codex_extension_api::ExtensionFuture<'a, ()> { + Box::pin(async move { + assert_eq!( + self.expected_thread_id.to_string(), + input.thread_store.level_id() + ); + assert_eq!(self.expected_turn_id, input.turn_store.level_id()); + assert_eq!(TurnAbortReason::Interrupted, input.reason); + self.calls + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .push("turn_abort"); + }) } } @@ -8698,15 +8725,19 @@ async fn task_finish_emits_thread_idle_lifecycle_after_active_turn_clears() { expected_thread_id: ThreadId, } - #[async_trait::async_trait] impl codex_extension_api::ThreadLifecycleContributor for ThreadIdleRecorder { - async fn on_thread_idle(&self, input: codex_extension_api::ThreadIdleInput<'_>) { - assert_eq!( - self.expected_thread_id.to_string(), - input.thread_store.level_id() - ); - self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); - self.idle_tx.send(()).await.expect("idle receiver open"); + fn on_thread_idle<'a>( + &'a self, + input: codex_extension_api::ThreadIdleInput<'a>, + ) -> codex_extension_api::ExtensionFuture<'a, ()> { + Box::pin(async move { + assert_eq!( + self.expected_thread_id.to_string(), + input.thread_store.level_id() + ); + self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + self.idle_tx.send(()).await.expect("idle receiver open"); + }) } } @@ -8740,10 +8771,14 @@ async fn thread_idle_lifecycle_waits_for_trigger_turn_mailbox_work() { calls: Arc, } - #[async_trait::async_trait] impl codex_extension_api::ThreadLifecycleContributor for ThreadIdleRecorder { - async fn on_thread_idle(&self, _input: codex_extension_api::ThreadIdleInput<'_>) { - self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + fn on_thread_idle<'a>( + &'a self, + _input: codex_extension_api::ThreadIdleInput<'a>, + ) -> codex_extension_api::ExtensionFuture<'a, ()> { + Box::pin(async move { + self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + }) } } diff --git a/codex-rs/core/src/session/turn_tests.rs b/codex-rs/core/src/session/turn_tests.rs index 81a43ff5a9..3511c03bda 100644 --- a/codex-rs/core/src/session/turn_tests.rs +++ b/codex-rs/core/src/session/turn_tests.rs @@ -7,20 +7,21 @@ use std::sync::Arc; struct RewriteAgentMessageContributor; -#[async_trait::async_trait] impl TurnItemContributor for RewriteAgentMessageContributor { - async fn contribute( - &self, - _thread_store: &ExtensionData, - _turn_store: &ExtensionData, - item: &mut TurnItem, - ) -> Result<(), String> { - if let TurnItem::AgentMessage(agent_message) = item { - agent_message.content = vec![AgentMessageContent::Text { - text: "plan contributed assistant text".to_string(), - }]; - } - Ok(()) + fn contribute<'a>( + &'a self, + _thread_store: &'a ExtensionData, + _turn_store: &'a ExtensionData, + item: &'a mut TurnItem, + ) -> codex_extension_api::ExtensionFuture<'a, Result<(), String>> { + Box::pin(async move { + if let TurnItem::AgentMessage(agent_message) = item { + agent_message.content = vec![AgentMessageContent::Text { + text: "plan contributed assistant text".to_string(), + }]; + } + Ok(()) + }) } } diff --git a/codex-rs/core/src/stream_events_utils_tests.rs b/codex-rs/core/src/stream_events_utils_tests.rs index 9c4be94576..c151a46fb3 100644 --- a/codex-rs/core/src/stream_events_utils_tests.rs +++ b/codex-rs/core/src/stream_events_utils_tests.rs @@ -167,41 +167,43 @@ struct TestTurnItemContributor; #[derive(Debug)] struct TurnItemContributorRan; -#[async_trait::async_trait] impl TurnItemContributor for TestTurnItemContributor { - async fn contribute( - &self, - _thread_store: &ExtensionData, - turn_store: &ExtensionData, - item: &mut TurnItem, - ) -> Result<(), String> { - turn_store.insert(TurnItemContributorRan); - if let TurnItem::AgentMessage(agent_message) = item { - agent_message.memory_citation = Some(MemoryCitation { - entries: Vec::new(), - rollout_ids: Vec::new(), - }); - } - Ok(()) + fn contribute<'a>( + &'a self, + _thread_store: &'a ExtensionData, + turn_store: &'a ExtensionData, + item: &'a mut TurnItem, + ) -> codex_extension_api::ExtensionFuture<'a, Result<(), String>> { + Box::pin(async move { + turn_store.insert(TurnItemContributorRan); + if let TurnItem::AgentMessage(agent_message) = item { + agent_message.memory_citation = Some(MemoryCitation { + entries: Vec::new(), + rollout_ids: Vec::new(), + }); + } + Ok(()) + }) } } struct RewriteAgentMessageContributor; -#[async_trait::async_trait] impl TurnItemContributor for RewriteAgentMessageContributor { - async fn contribute( - &self, - _thread_store: &ExtensionData, - _turn_store: &ExtensionData, - item: &mut TurnItem, - ) -> Result<(), String> { - if let TurnItem::AgentMessage(agent_message) = item { - agent_message.content = vec![AgentMessageContent::Text { - text: "contributed assistant text".to_string(), - }]; - } - Ok(()) + fn contribute<'a>( + &'a self, + _thread_store: &'a ExtensionData, + _turn_store: &'a ExtensionData, + item: &'a mut TurnItem, + ) -> codex_extension_api::ExtensionFuture<'a, Result<(), String>> { + Box::pin(async move { + if let TurnItem::AgentMessage(agent_message) = item { + agent_message.content = vec![AgentMessageContent::Text { + text: "contributed assistant text".to_string(), + }]; + } + Ok(()) + }) } } diff --git a/codex-rs/core/src/thread_manager_tests.rs b/codex-rs/core/src/thread_manager_tests.rs index bb0c0af314..48ca2598ab 100644 --- a/codex-rs/core/src/thread_manager_tests.rs +++ b/codex-rs/core/src/thread_manager_tests.rs @@ -394,20 +394,24 @@ async fn start_thread_seeds_extension_data_before_lifecycle_contributors_run() { observed: Arc>>, } - #[async_trait::async_trait] impl codex_extension_api::ThreadLifecycleContributor for InitialDataRecorder { - async fn on_thread_start(&self, input: codex_extension_api::ThreadStartInput<'_, Config>) { - let marker = input - .thread_store - .get::() - .expect("initial extension data should be available"); - *self - .observed - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(( - input.thread_store.level_id().to_string(), - marker.0.to_string(), - )); + fn on_thread_start<'a>( + &'a self, + input: codex_extension_api::ThreadStartInput<'a, Config>, + ) -> codex_extension_api::ExtensionFuture<'a, ()> { + Box::pin(async move { + let marker = input + .thread_store + .get::() + .expect("initial extension data should be available"); + *self + .observed + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(( + input.thread_store.level_id().to_string(), + marker.0.to_string(), + )); + }) } } diff --git a/codex-rs/core/src/tools/handlers/extension_tools.rs b/codex-rs/core/src/tools/handlers/extension_tools.rs index ecbf772e90..5e177156c6 100644 --- a/codex-rs/core/src/tools/handlers/extension_tools.rs +++ b/codex-rs/core/src/tools/handlers/extension_tools.rs @@ -388,16 +388,17 @@ mod tests { struct RecordExtensionTurnItemContributor; - #[async_trait::async_trait] impl TurnItemContributor for RecordExtensionTurnItemContributor { - async fn contribute( - &self, - _thread_store: &ExtensionData, - turn_store: &ExtensionData, - _item: &mut TurnItem, - ) -> Result<(), String> { - turn_store.insert(ExtensionTurnItemContributorRan); - Ok(()) + fn contribute<'a>( + &'a self, + _thread_store: &'a ExtensionData, + turn_store: &'a ExtensionData, + _item: &'a mut TurnItem, + ) -> codex_extension_api::ExtensionFuture<'a, Result<(), String>> { + Box::pin(async move { + turn_store.insert(ExtensionTurnItemContributorRan); + Ok(()) + }) } } diff --git a/codex-rs/ext/extension-api/Cargo.toml b/codex-rs/ext/extension-api/Cargo.toml index 2e155b430e..25ed8d0d71 100644 --- a/codex-rs/ext/extension-api/Cargo.toml +++ b/codex-rs/ext/extension-api/Cargo.toml @@ -14,7 +14,6 @@ doctest = false workspace = true [dependencies] -async-trait = { workspace = true } codex-config = { workspace = true } codex-context-fragments = { workspace = true } codex-protocol = { workspace = true } diff --git a/codex-rs/ext/extension-api/src/contributors.rs b/codex-rs/ext/extension-api/src/contributors.rs index 0403294371..e93088b5d0 100644 --- a/codex-rs/ext/extension-api/src/contributors.rs +++ b/codex-rs/ext/extension-api/src/contributors.rs @@ -1,4 +1,5 @@ use std::future::Future; +use std::pin::Pin; use std::sync::Arc; use codex_context_fragments::ContextualUserFragment; @@ -36,6 +37,9 @@ pub use turn_lifecycle::TurnErrorInput; pub use turn_lifecycle::TurnStartInput; pub use turn_lifecycle::TurnStopInput; +/// Boxed, sendable future returned by asynchronous extension contributors. +pub type ExtensionFuture<'a, T> = Pin + Send + 'a>>; + /// Extension contribution that resolves runtime MCP servers from host config. /// /// Contributors run in registration order. Later contributions for the same @@ -43,9 +47,8 @@ pub use turn_lifecycle::TurnStopInput; /// own and must apply any source-specific policy before returning a server. /// Plugin-owned servers and their provenance continue to be resolved by the /// plugin manager until that ownership moves into an extension explicitly. -#[async_trait::async_trait] pub trait McpServerContributor: Send + Sync { - async fn contribute(&self, config: &C) -> Vec; + fn contribute<'a>(&'a self, config: &'a C) -> ExtensionFuture<'a, Vec>; } /// Extension contribution that adds prompt fragments during prompt assembly. @@ -54,7 +57,7 @@ pub trait ContextContributor: Send + Sync { &'a self, session_store: &'a ExtensionData, thread_store: &'a ExtensionData, - ) -> std::pin::Pin> + Send + 'a>>; + ) -> ExtensionFuture<'a, Vec>; } /// Contributor for host-owned thread lifecycle gates. @@ -62,24 +65,43 @@ pub trait ContextContributor: Send + Sync { /// Implementations should use these callbacks to seed, rehydrate, or flush /// extension-private thread state. Heavy dependencies belong on the extension /// value created by the host, not in these inputs. -#[async_trait::async_trait] pub trait ThreadLifecycleContributor: Send + Sync { /// Called after thread-scoped extension stores are created, before later /// contributors can read from them. - async fn on_thread_start(&self, _input: ThreadStartInput<'_, C>) {} + fn on_thread_start<'a>(&'a self, input: ThreadStartInput<'a, C>) -> ExtensionFuture<'a, ()> { + Box::pin(async move { + let _self = self; + let _input = input; + }) + } /// Called after the host constructs a runtime from persisted history. - async fn on_thread_resume(&self, _input: ThreadResumeInput<'_>) {} + fn on_thread_resume<'a>(&'a self, input: ThreadResumeInput<'a>) -> ExtensionFuture<'a, ()> { + Box::pin(async move { + let _self = self; + let _input = input; + }) + } /// Called after the host has drained immediately pending thread work. /// /// Implementations may use host capabilities captured by the extension to /// submit follow-up input. The host remains responsible for deciding /// whether that input starts a turn, is queued, or is ignored. - async fn on_thread_idle(&self, _input: ThreadIdleInput<'_>) {} + fn on_thread_idle<'a>(&'a self, input: ThreadIdleInput<'a>) -> ExtensionFuture<'a, ()> { + Box::pin(async move { + let _self = self; + let _input = input; + }) + } /// Called before the host drops the thread runtime and thread-scoped store. - async fn on_thread_stop(&self, _input: ThreadStopInput<'_>) {} + fn on_thread_stop<'a>(&'a self, input: ThreadStopInput<'a>) -> ExtensionFuture<'a, ()> { + Box::pin(async move { + let _self = self; + let _input = input; + }) + } } /// Contributor for host-owned turn lifecycle gates. @@ -87,20 +109,39 @@ pub trait ThreadLifecycleContributor: Send + Sync { /// Implementations should use these callbacks to seed, observe, or clear /// extension-private turn state. The host exposes stable identifiers and /// extension stores instead of core runtime objects. -#[async_trait::async_trait] pub trait TurnLifecycleContributor: Send + Sync { /// Called after turn-scoped extension stores are created, before the task /// for the turn starts running. - async fn on_turn_start(&self, _input: TurnStartInput<'_>) {} + fn on_turn_start<'a>(&'a self, input: TurnStartInput<'a>) -> ExtensionFuture<'a, ()> { + Box::pin(async move { + let _self = self; + let _input = input; + }) + } /// Called before the host drops the completed turn runtime and turn store. - async fn on_turn_stop(&self, _input: TurnStopInput<'_>) {} + fn on_turn_stop<'a>(&'a self, input: TurnStopInput<'a>) -> ExtensionFuture<'a, ()> { + Box::pin(async move { + let _self = self; + let _input = input; + }) + } /// Called after the host aborts a running turn. - async fn on_turn_abort(&self, _input: TurnAbortInput<'_>) {} + fn on_turn_abort<'a>(&'a self, input: TurnAbortInput<'a>) -> ExtensionFuture<'a, ()> { + Box::pin(async move { + let _self = self; + let _input = input; + }) + } /// Called when the host observes an error for a running turn. - async fn on_turn_error(&self, _input: TurnErrorInput<'_>) {} + fn on_turn_error<'a>(&'a self, input: TurnErrorInput<'a>) -> ExtensionFuture<'a, ()> { + Box::pin(async move { + let _self = self; + let _input = input; + }) + } } /// Extension contribution that can add turn-local model input. @@ -109,16 +150,15 @@ pub trait TurnLifecycleContributor: Send + Sync { /// must preserve authority boundaries for external resources. Expensive or /// host-specific dependencies belong on the extension value installed by the /// host, not in this input. -#[async_trait::async_trait] pub trait TurnInputContributor: Send + Sync { /// Returns additional contextual fragments for one submitted turn. - async fn contribute( - &self, + fn contribute<'a>( + &'a self, input: TurnInputContext, - session_store: &ExtensionData, - thread_store: &ExtensionData, - turn_store: &ExtensionData, - ) -> Vec>; + session_store: &'a ExtensionData, + thread_store: &'a ExtensionData, + turn_store: &'a ExtensionData, + ) -> ExtensionFuture<'a, Vec>>; } /// Contributor for host-owned configuration changes. @@ -142,16 +182,19 @@ pub trait ConfigContributor: Send + Sync { /// Implementations should keep this callback cheap. The host calls it after /// updating cached token usage and before emitting the corresponding client /// token-count notification. -#[async_trait::async_trait] pub trait TokenUsageContributor: Send + Sync { /// Called each time the host records token usage from a model response. - async fn on_token_usage( - &self, - _session_store: &ExtensionData, - _thread_store: &ExtensionData, - _turn_store: &ExtensionData, - _token_usage: &TokenUsageInfo, - ) { + fn on_token_usage<'a>( + &'a self, + _session_store: &'a ExtensionData, + _thread_store: &'a ExtensionData, + _turn_store: &'a ExtensionData, + _token_usage: &'a TokenUsageInfo, + ) -> ExtensionFuture<'a, ()> { + Box::pin(async move { + let _self = self; + let _inputs = (_session_store, _thread_store, _turn_store, _token_usage); + }) } } @@ -183,14 +226,13 @@ pub trait ToolLifecycleContributor: Send + Sync { } /// Extension contribution that can claim rendered approval-review prompts. -#[async_trait::async_trait] pub trait ApprovalReviewContributor: Send + Sync { - async fn contribute( - &self, - session_store: &ExtensionData, - thread_store: &ExtensionData, - prompt: &str, - ) -> Option; + fn contribute<'a>( + &'a self, + session_store: &'a ExtensionData, + thread_store: &'a ExtensionData, + prompt: &'a str, + ) -> ExtensionFuture<'a, Option>; } /// Ordered post-processing contribution for one parsed turn item. @@ -198,12 +240,11 @@ pub trait ApprovalReviewContributor: Send + Sync { /// Implementations may mutate the item before it is emitted and may use the /// explicitly exposed thread- and turn-lifetime stores when they need durable /// extension-private state. -#[async_trait::async_trait] pub trait TurnItemContributor: Send + Sync { - async fn contribute( - &self, - thread_store: &ExtensionData, - turn_store: &ExtensionData, - item: &mut TurnItem, - ) -> Result<(), String>; + fn contribute<'a>( + &'a self, + thread_store: &'a ExtensionData, + turn_store: &'a ExtensionData, + item: &'a mut TurnItem, + ) -> ExtensionFuture<'a, Result<(), String>>; } diff --git a/codex-rs/ext/extension-api/src/lib.rs b/codex-rs/ext/extension-api/src/lib.rs index 8b269f31d1..dc3495a71c 100644 --- a/codex-rs/ext/extension-api/src/lib.rs +++ b/codex-rs/ext/extension-api/src/lib.rs @@ -31,6 +31,7 @@ pub use codex_tools::parse_tool_input_schema_without_compaction; pub use contributors::ApprovalReviewContributor; pub use contributors::ConfigContributor; pub use contributors::ContextContributor; +pub use contributors::ExtensionFuture; pub use contributors::McpServerContribution; pub use contributors::McpServerContributor; pub use contributors::PromptFragment; diff --git a/codex-rs/ext/extension-api/tests/registry.rs b/codex-rs/ext/extension-api/tests/registry.rs index d26f1010cb..42cb18af62 100644 --- a/codex-rs/ext/extension-api/tests/registry.rs +++ b/codex-rs/ext/extension-api/tests/registry.rs @@ -1,5 +1,3 @@ -use std::future::Future; -use std::pin::Pin; use std::sync::Arc; use std::sync::Mutex; @@ -9,6 +7,7 @@ use codex_extension_api::ContextContributor; use codex_extension_api::ContextualUserFragment; use codex_extension_api::ExtensionData; use codex_extension_api::ExtensionEventSink; +use codex_extension_api::ExtensionFuture; use codex_extension_api::ExtensionRegistryBuilder; use codex_extension_api::PromptFragment; use codex_extension_api::ThreadLifecycleContributor; @@ -37,32 +36,32 @@ impl ContextContributor for AllContributors { &'a self, _session_store: &'a ExtensionData, _thread_store: &'a ExtensionData, - ) -> Pin> + Send + 'a>> { + ) -> ExtensionFuture<'a, Vec> { Box::pin(std::future::ready(Vec::new())) } } -#[async_trait::async_trait] impl ThreadLifecycleContributor<()> for AllContributors {} -#[async_trait::async_trait] impl TurnLifecycleContributor for AllContributors {} impl ConfigContributor<()> for AllContributors {} -#[async_trait::async_trait] impl TokenUsageContributor for AllContributors {} -#[async_trait::async_trait] impl TurnInputContributor for AllContributors { - async fn contribute( - &self, - _input: TurnInputContext, - _session_store: &ExtensionData, - _thread_store: &ExtensionData, - _turn_store: &ExtensionData, - ) -> Vec> { - Vec::new() + fn contribute<'a>( + &'a self, + input: TurnInputContext, + _session_store: &'a ExtensionData, + _thread_store: &'a ExtensionData, + _turn_store: &'a ExtensionData, + ) -> ExtensionFuture<'a, Vec>> { + Box::pin(async move { + let _self = self; + let _input = input; + Vec::new() + }) } } @@ -78,27 +77,31 @@ impl ToolContributor for AllContributors { impl ToolLifecycleContributor for AllContributors {} -#[async_trait::async_trait] impl TurnItemContributor for AllContributors { - async fn contribute( - &self, - _thread_store: &ExtensionData, - _turn_store: &ExtensionData, - _item: &mut TurnItem, - ) -> Result<(), String> { - Ok(()) + fn contribute<'a>( + &'a self, + _thread_store: &'a ExtensionData, + _turn_store: &'a ExtensionData, + _item: &'a mut TurnItem, + ) -> ExtensionFuture<'a, Result<(), String>> { + Box::pin(async move { + let _self = self; + Ok(()) + }) } } -#[async_trait::async_trait] impl ApprovalReviewContributor for AllContributors { - async fn contribute( - &self, - _session_store: &ExtensionData, - _thread_store: &ExtensionData, - _prompt: &str, - ) -> Option { - Some(ReviewDecision::ApprovedForSession) + fn contribute<'a>( + &'a self, + _session_store: &'a ExtensionData, + _thread_store: &'a ExtensionData, + _prompt: &'a str, + ) -> ExtensionFuture<'a, Option> { + Box::pin(async move { + let _self = self; + Some(ReviewDecision::ApprovedForSession) + }) } } @@ -146,7 +149,7 @@ impl ContextContributor for NamedContextContributor { &'a self, _session_store: &'a ExtensionData, _thread_store: &'a ExtensionData, - ) -> Pin> + Send + 'a>> { + ) -> ExtensionFuture<'a, Vec> { Box::pin(std::future::ready(vec![PromptFragment::developer_policy( self.0, )])) @@ -158,19 +161,20 @@ struct RecordingTurnItemContributor { calls: Arc>>, } -#[async_trait::async_trait] impl TurnItemContributor for RecordingTurnItemContributor { - async fn contribute( - &self, - _thread_store: &ExtensionData, - _turn_store: &ExtensionData, - _item: &mut TurnItem, - ) -> Result<(), String> { - self.calls - .lock() - .unwrap_or_else(|error| panic!("turn item calls lock poisoned: {error}")) - .push(self.name); - Ok(()) + fn contribute<'a>( + &'a self, + _thread_store: &'a ExtensionData, + _turn_store: &'a ExtensionData, + _item: &'a mut TurnItem, + ) -> ExtensionFuture<'a, Result<(), String>> { + Box::pin(async move { + self.calls + .lock() + .unwrap_or_else(|error| panic!("turn item calls lock poisoned: {error}")) + .push(self.name); + Ok(()) + }) } } @@ -236,24 +240,25 @@ struct RecordingApprovalContributor { calls: Arc>>, } -#[async_trait::async_trait] impl ApprovalReviewContributor for RecordingApprovalContributor { - async fn contribute( - &self, - session_store: &ExtensionData, - thread_store: &ExtensionData, - prompt: &str, - ) -> Option { - self.calls - .lock() - .unwrap_or_else(|error| panic!("approval calls lock poisoned: {error}")) - .push(ApprovalCall { - contributor: self.name, - session_id: session_store.level_id().to_string(), - thread_id: thread_store.level_id().to_string(), - prompt: prompt.to_string(), - }); - self.decision.clone() + fn contribute<'a>( + &'a self, + session_store: &'a ExtensionData, + thread_store: &'a ExtensionData, + prompt: &'a str, + ) -> ExtensionFuture<'a, Option> { + Box::pin(async move { + self.calls + .lock() + .unwrap_or_else(|error| panic!("approval calls lock poisoned: {error}")) + .push(ApprovalCall { + contributor: self.name, + session_id: session_store.level_id().to_string(), + thread_id: thread_store.level_id().to_string(), + prompt: prompt.to_string(), + }); + self.decision.clone() + }) } } diff --git a/codex-rs/ext/goal/src/extension.rs b/codex-rs/ext/goal/src/extension.rs index 0696c2b868..8e5b20e7f5 100644 --- a/codex-rs/ext/goal/src/extension.rs +++ b/codex-rs/ext/goal/src/extension.rs @@ -1,12 +1,12 @@ use std::sync::Arc; use std::sync::Weak; -use async_trait::async_trait; use codex_analytics::AnalyticsEventsClient; use codex_core::ThreadManager; use codex_extension_api::ConfigContributor; use codex_extension_api::ExtensionData; use codex_extension_api::ExtensionEventSink; +use codex_extension_api::ExtensionFuture; use codex_extension_api::ExtensionRegistryBuilder; use codex_extension_api::ThreadIdleInput; use codex_extension_api::ThreadLifecycleContributor; @@ -95,76 +95,83 @@ impl GoalExtension { } } -#[async_trait] impl ThreadLifecycleContributor for GoalExtension where C: Send + Sync + 'static, { - async fn on_thread_start(&self, input: ThreadStartInput<'_, C>) { - let enabled = (self.goals_enabled)(input.config); - let tools_available_for_thread = input.persistent_thread_state_available - && !matches!( - input.session_source, - SessionSource::SubAgent(SubAgentSource::Review) - ); - input - .thread_store - .insert(GoalExtensionConfig::from_enabled(enabled)); - let accounting_state = input - .thread_store - .get_or_init::(GoalAccountingState::default); - let Ok(thread_id) = ThreadId::from_string(input.thread_store.level_id()) else { - return; - }; - let runtime = input.thread_store.get_or_init::(|| { - GoalRuntimeHandle::new( - thread_id, - Arc::clone(&self.state_dbs), - self.event_emitter.clone(), - self.metrics.clone(), - self.thread_manager.clone(), - accounting_state, - GoalRuntimeConfig { - analytics: self.analytics.clone(), - enabled, - tools_available_for_thread, - }, - ) - }); - runtime.set_enabled(enabled); - self.goal_service.register_runtime(&runtime); + fn on_thread_start<'a>(&'a self, input: ThreadStartInput<'a, C>) -> ExtensionFuture<'a, ()> { + Box::pin(async move { + let enabled = (self.goals_enabled)(input.config); + let tools_available_for_thread = input.persistent_thread_state_available + && !matches!( + input.session_source, + SessionSource::SubAgent(SubAgentSource::Review) + ); + input + .thread_store + .insert(GoalExtensionConfig::from_enabled(enabled)); + let accounting_state = input + .thread_store + .get_or_init::(GoalAccountingState::default); + let Ok(thread_id) = ThreadId::from_string(input.thread_store.level_id()) else { + return; + }; + let runtime = input.thread_store.get_or_init::(|| { + GoalRuntimeHandle::new( + thread_id, + Arc::clone(&self.state_dbs), + self.event_emitter.clone(), + self.metrics.clone(), + self.thread_manager.clone(), + accounting_state, + GoalRuntimeConfig { + analytics: self.analytics.clone(), + enabled, + tools_available_for_thread, + }, + ) + }); + runtime.set_enabled(enabled); + self.goal_service.register_runtime(&runtime); + }) } - async fn on_thread_resume(&self, input: ThreadResumeInput<'_>) { - let Some(runtime) = goal_runtime_handle(input.thread_store) else { - return; - }; + fn on_thread_resume<'a>(&'a self, input: ThreadResumeInput<'a>) -> ExtensionFuture<'a, ()> { + Box::pin(async move { + let Some(runtime) = goal_runtime_handle(input.thread_store) else { + return; + }; - if let Err(err) = runtime.restore_after_resume().await { - tracing::warn!( - "failed to restore goal runtime after thread resume for {}: {err}", - runtime.thread_id() - ); - } + if let Err(err) = runtime.restore_after_resume().await { + tracing::warn!( + "failed to restore goal runtime after thread resume for {}: {err}", + runtime.thread_id() + ); + } + }) } - async fn on_thread_idle(&self, input: ThreadIdleInput<'_>) { - let Some(runtime) = goal_runtime_handle(input.thread_store) else { - return; - }; + fn on_thread_idle<'a>(&'a self, input: ThreadIdleInput<'a>) -> ExtensionFuture<'a, ()> { + Box::pin(async move { + let Some(runtime) = goal_runtime_handle(input.thread_store) else { + return; + }; - if let Err(err) = runtime.continue_if_idle().await { - tracing::warn!( - "failed to continue active goal for idle thread {}: {err}", - runtime.thread_id() - ); - } + if let Err(err) = runtime.continue_if_idle().await { + tracing::warn!( + "failed to continue active goal for idle thread {}: {err}", + runtime.thread_id() + ); + } + }) } - async fn on_thread_stop(&self, input: ThreadStopInput<'_>) { - if let Some(runtime) = goal_runtime_handle(input.thread_store) { - self.goal_service.unregister_runtime(&runtime); - } + fn on_thread_stop<'a>(&'a self, input: ThreadStopInput<'a>) -> ExtensionFuture<'a, ()> { + Box::pin(async move { + if let Some(runtime) = goal_runtime_handle(input.thread_store) { + self.goal_service.unregister_runtime(&runtime); + } + }) } } @@ -187,153 +194,161 @@ where } } -#[async_trait] impl TurnLifecycleContributor for GoalExtension where C: Send + Sync + 'static, { - async fn on_turn_start(&self, input: TurnStartInput<'_>) { - let Some(runtime) = goal_runtime_handle(input.thread_store) else { - return; - }; - if !runtime.is_enabled() { - return; - } + fn on_turn_start<'a>(&'a self, input: TurnStartInput<'a>) -> ExtensionFuture<'a, ()> { + Box::pin(async move { + let Some(runtime) = goal_runtime_handle(input.thread_store) else { + return; + }; + if !runtime.is_enabled() { + return; + } - let accounting = runtime.accounting_state(); - accounting.start_turn( - input.turn_id, - input.collaboration_mode.mode, - input.token_usage_at_turn_start, - ); - if matches!( - input.collaboration_mode.mode, - codex_protocol::config_types::ModeKind::Plan - ) { - accounting.clear_current_turn_goal(); - return; - } - let Ok(goal) = self - .state_dbs - .thread_goals() - .get_thread_goal(runtime.thread_id()) - .await - else { - return; - }; - if let Some(goal) = goal - && matches!( - goal.status, - codex_state::ThreadGoalStatus::Active - | codex_state::ThreadGoalStatus::BudgetLimited - ) - { - accounting.mark_turn_goal_active(input.turn_id, goal.goal_id); - } + let accounting = runtime.accounting_state(); + accounting.start_turn( + input.turn_id, + input.collaboration_mode.mode, + input.token_usage_at_turn_start, + ); + if matches!( + input.collaboration_mode.mode, + codex_protocol::config_types::ModeKind::Plan + ) { + accounting.clear_current_turn_goal(); + return; + } + let Ok(goal) = self + .state_dbs + .thread_goals() + .get_thread_goal(runtime.thread_id()) + .await + else { + return; + }; + if let Some(goal) = goal + && matches!( + goal.status, + codex_state::ThreadGoalStatus::Active + | codex_state::ThreadGoalStatus::BudgetLimited + ) + { + accounting.mark_turn_goal_active(input.turn_id, goal.goal_id); + } + }) } - async fn on_turn_stop(&self, input: TurnStopInput<'_>) { - let Some(runtime) = goal_runtime_handle(input.thread_store) else { - return; - }; - if !runtime.is_enabled() { - return; - } + fn on_turn_stop<'a>(&'a self, input: TurnStopInput<'a>) -> ExtensionFuture<'a, ()> { + Box::pin(async move { + let Some(runtime) = goal_runtime_handle(input.thread_store) else { + return; + }; + if !runtime.is_enabled() { + return; + } - let turn_id = input.turn_store.level_id(); - if let Err(err) = runtime - .account_active_goal_progress( - turn_id, - &format!("{turn_id}:turn-stop"), - codex_state::GoalAccountingMode::ActiveOnly, - BudgetLimitedGoalDisposition::ClearActive, - ) - .await - { - tracing::warn!( - "failed to account active goal progress at turn stop for {turn_id}: {err}" - ); - return; - } - runtime.accounting_state().finish_turn(turn_id); + let turn_id = input.turn_store.level_id(); + if let Err(err) = runtime + .account_active_goal_progress( + turn_id, + &format!("{turn_id}:turn-stop"), + codex_state::GoalAccountingMode::ActiveOnly, + BudgetLimitedGoalDisposition::ClearActive, + ) + .await + { + tracing::warn!( + "failed to account active goal progress at turn stop for {turn_id}: {err}" + ); + return; + } + runtime.accounting_state().finish_turn(turn_id); + }) } - async fn on_turn_abort(&self, input: TurnAbortInput<'_>) { - let Some(runtime) = goal_runtime_handle(input.thread_store) else { - return; - }; - if !runtime.is_enabled() { - return; - } + fn on_turn_abort<'a>(&'a self, input: TurnAbortInput<'a>) -> ExtensionFuture<'a, ()> { + Box::pin(async move { + let Some(runtime) = goal_runtime_handle(input.thread_store) else { + return; + }; + if !runtime.is_enabled() { + return; + } - let turn_id = input.turn_store.level_id(); - if let Err(err) = runtime - .account_active_goal_progress( - turn_id, - &format!("{turn_id}:turn-abort"), - codex_state::GoalAccountingMode::ActiveOnly, - BudgetLimitedGoalDisposition::ClearActive, - ) - .await - { - tracing::warn!( - "failed to account active goal progress after turn abort for {turn_id}: {err}" - ); - return; - } - runtime.accounting_state().finish_turn(turn_id); + let turn_id = input.turn_store.level_id(); + if let Err(err) = runtime + .account_active_goal_progress( + turn_id, + &format!("{turn_id}:turn-abort"), + codex_state::GoalAccountingMode::ActiveOnly, + BudgetLimitedGoalDisposition::ClearActive, + ) + .await + { + tracing::warn!( + "failed to account active goal progress after turn abort for {turn_id}: {err}" + ); + return; + } + runtime.accounting_state().finish_turn(turn_id); + }) } - async fn on_turn_error(&self, input: TurnErrorInput<'_>) { - let Some(runtime) = goal_runtime_handle(input.thread_store) else { - return; - }; + fn on_turn_error<'a>(&'a self, input: TurnErrorInput<'a>) -> ExtensionFuture<'a, ()> { + Box::pin(async move { + let Some(runtime) = goal_runtime_handle(input.thread_store) else { + return; + }; - let reason = match input.error { - CodexErrorInfo::UsageLimitExceeded => ActiveGoalStopReason::UsageLimit, - // The turn has ended because the error was non-retryable or its - // retries were exhausted. Block the goal to prevent automatic - // continuation from looping and consuming tokens, as can happen - // with compaction errors. - _ => ActiveGoalStopReason::TurnError, - }; - if let Err(err) = runtime - .stop_active_goal_for_turn(input.turn_id, reason) - .await - { - tracing::warn!( - error = ?input.error, - "failed to stop active goal after turn error: {err}" - ); - } + let reason = match input.error { + CodexErrorInfo::UsageLimitExceeded => ActiveGoalStopReason::UsageLimit, + // The turn has ended because the error was non-retryable or its + // retries were exhausted. Block the goal to prevent automatic + // continuation from looping and consuming tokens, as can happen + // with compaction errors. + _ => ActiveGoalStopReason::TurnError, + }; + if let Err(err) = runtime + .stop_active_goal_for_turn(input.turn_id, reason) + .await + { + tracing::warn!( + error = ?input.error, + "failed to stop active goal after turn error: {err}" + ); + } + }) } } -#[async_trait] impl TokenUsageContributor for GoalExtension where C: Send + Sync + 'static, { - async fn on_token_usage( - &self, - _session_store: &ExtensionData, - thread_store: &ExtensionData, - turn_store: &ExtensionData, - token_usage: &TokenUsageInfo, - ) { - let Some(runtime) = goal_runtime_handle(thread_store) else { - return; - }; - if !runtime.is_enabled() { - return; - } + fn on_token_usage<'a>( + &'a self, + _session_store: &'a ExtensionData, + thread_store: &'a ExtensionData, + turn_store: &'a ExtensionData, + token_usage: &'a TokenUsageInfo, + ) -> ExtensionFuture<'a, ()> { + Box::pin(async move { + let Some(runtime) = goal_runtime_handle(thread_store) else { + return; + }; + if !runtime.is_enabled() { + return; + } - let Some(_recorded) = runtime - .accounting_state() - .record_token_usage(turn_store.level_id(), &token_usage.total_token_usage) - else { - return; - }; + let Some(_recorded) = runtime + .accounting_state() + .record_token_usage(turn_store.level_id(), &token_usage.total_token_usage) + else { + return; + }; + }) } } diff --git a/codex-rs/ext/guardian/Cargo.toml b/codex-rs/ext/guardian/Cargo.toml index 53e553ec1c..513254b7cf 100644 --- a/codex-rs/ext/guardian/Cargo.toml +++ b/codex-rs/ext/guardian/Cargo.toml @@ -14,7 +14,6 @@ doctest = false workspace = true [dependencies] -async-trait = { workspace = true } codex-core = { workspace = true } codex-extension-api = { workspace = true } codex-protocol = { workspace = true } diff --git a/codex-rs/ext/guardian/src/lib.rs b/codex-rs/ext/guardian/src/lib.rs index 0591887c25..a64cf4fed4 100644 --- a/codex-rs/ext/guardian/src/lib.rs +++ b/codex-rs/ext/guardian/src/lib.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use codex_core::config::Config; use codex_extension_api::AgentSpawnFuture; use codex_extension_api::AgentSpawner; +use codex_extension_api::ExtensionFuture; use codex_extension_api::ExtensionRegistryBuilder; use codex_extension_api::ThreadLifecycleContributor; use codex_extension_api::ThreadStartInput; @@ -47,18 +48,23 @@ impl GuardianThreadContext { } } -#[async_trait::async_trait] impl ThreadLifecycleContributor for GuardianExtension where S: Send + Sync, { - async fn on_thread_start(&self, input: ThreadStartInput<'_, Config>) { - let Ok(forked_from_thread_id) = ThreadId::from_string(input.thread_store.level_id()) else { - return; - }; - input.thread_store.insert(GuardianThreadContext { - forked_from_thread_id, - }); + fn on_thread_start<'a>( + &'a self, + input: ThreadStartInput<'a, Config>, + ) -> ExtensionFuture<'a, ()> { + Box::pin(async move { + let Ok(forked_from_thread_id) = ThreadId::from_string(input.thread_store.level_id()) + else { + return; + }; + input.thread_store.insert(GuardianThreadContext { + forked_from_thread_id, + }); + }) } } diff --git a/codex-rs/ext/image-generation/src/extension.rs b/codex-rs/ext/image-generation/src/extension.rs index 2c68f614f5..6a0016c037 100644 --- a/codex-rs/ext/image-generation/src/extension.rs +++ b/codex-rs/ext/image-generation/src/extension.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use codex_core::config::Config; use codex_extension_api::ConfigContributor; use codex_extension_api::ExtensionData; +use codex_extension_api::ExtensionFuture; use codex_extension_api::ExtensionRegistryBuilder; use codex_extension_api::ThreadLifecycleContributor; use codex_extension_api::ThreadStartInput; @@ -41,13 +42,17 @@ impl From<&Config> for ImageGenerationExtensionConfig { } } -#[async_trait::async_trait] impl ThreadLifecycleContributor for ImageGenerationExtension { /// Seeds image-generation availability when a thread begins. - async fn on_thread_start(&self, input: ThreadStartInput<'_, Config>) { - input - .thread_store - .insert(ImageGenerationExtensionConfig::from(input.config)); + fn on_thread_start<'a>( + &'a self, + input: ThreadStartInput<'a, Config>, + ) -> ExtensionFuture<'a, ()> { + Box::pin(async move { + input + .thread_store + .insert(ImageGenerationExtensionConfig::from(input.config)); + }) } } diff --git a/codex-rs/ext/mcp/Cargo.toml b/codex-rs/ext/mcp/Cargo.toml index 66c9ad1727..2d0e508b5e 100644 --- a/codex-rs/ext/mcp/Cargo.toml +++ b/codex-rs/ext/mcp/Cargo.toml @@ -14,7 +14,6 @@ doctest = false workspace = true [dependencies] -async-trait = { workspace = true } codex-core = { workspace = true } codex-extension-api = { workspace = true } codex-features = { workspace = true } diff --git a/codex-rs/ext/mcp/src/lib.rs b/codex-rs/ext/mcp/src/lib.rs index 8fad361a4e..8568d58150 100644 --- a/codex-rs/ext/mcp/src/lib.rs +++ b/codex-rs/ext/mcp/src/lib.rs @@ -1,4 +1,5 @@ use codex_core::config::Config; +use codex_extension_api::ExtensionFuture; use codex_extension_api::ExtensionRegistryBuilder; use codex_extension_api::McpServerContribution; use codex_extension_api::McpServerContributor; @@ -7,21 +8,25 @@ use codex_mcp::hosted_plugin_runtime_mcp_server_config; struct HostedPluginRuntimeExtension; -#[async_trait::async_trait] impl McpServerContributor for HostedPluginRuntimeExtension { - async fn contribute(&self, config: &Config) -> Vec { - let name = CODEX_APPS_MCP_SERVER_NAME.to_string(); - if !config.features.enabled(codex_features::Feature::Apps) { - return vec![McpServerContribution::Remove { name }]; - } + fn contribute<'a>( + &'a self, + config: &'a Config, + ) -> ExtensionFuture<'a, Vec> { + Box::pin(async move { + let name = CODEX_APPS_MCP_SERVER_NAME.to_string(); + if !config.features.enabled(codex_features::Feature::Apps) { + return vec![McpServerContribution::Remove { name }]; + } - vec![McpServerContribution::Set { - name, - config: Box::new(hosted_plugin_runtime_mcp_server_config( - &config.chatgpt_base_url, - config.apps_mcp_product_sku.as_deref(), - )), - }] + vec![McpServerContribution::Set { + name, + config: Box::new(hosted_plugin_runtime_mcp_server_config( + &config.chatgpt_base_url, + config.apps_mcp_product_sku.as_deref(), + )), + }] + }) } } diff --git a/codex-rs/ext/mcp/tests/hosted_apps_mcp.rs b/codex-rs/ext/mcp/tests/hosted_apps_mcp.rs index bbfa07c14c..bb401578f4 100644 --- a/codex-rs/ext/mcp/tests/hosted_apps_mcp.rs +++ b/codex-rs/ext/mcp/tests/hosted_apps_mcp.rs @@ -150,11 +150,15 @@ fn installed_manager(config: &Config) -> McpManager { struct RemoveCodexApps; -#[async_trait::async_trait] impl McpServerContributor for RemoveCodexApps { - async fn contribute(&self, _config: &Config) -> Vec { - vec![McpServerContribution::Remove { - name: CODEX_APPS_MCP_SERVER_NAME.to_string(), - }] + fn contribute<'a>( + &'a self, + _config: &'a Config, + ) -> codex_extension_api::ExtensionFuture<'a, Vec> { + Box::pin(async move { + vec![McpServerContribution::Remove { + name: CODEX_APPS_MCP_SERVER_NAME.to_string(), + }] + }) } } diff --git a/codex-rs/ext/memories/src/extension.rs b/codex-rs/ext/memories/src/extension.rs index 0be773b4c7..23c80a8779 100644 --- a/codex-rs/ext/memories/src/extension.rs +++ b/codex-rs/ext/memories/src/extension.rs @@ -4,6 +4,7 @@ use codex_core::config::Config; use codex_extension_api::ConfigContributor; use codex_extension_api::ContextContributor; use codex_extension_api::ExtensionData; +use codex_extension_api::ExtensionFuture; use codex_extension_api::ExtensionRegistryBuilder; use codex_extension_api::PromptFragment; use codex_extension_api::ThreadLifecycleContributor; @@ -69,12 +70,16 @@ impl ContextContributor for MemoriesExtension { } } -#[async_trait::async_trait] impl ThreadLifecycleContributor for MemoriesExtension { - async fn on_thread_start(&self, input: ThreadStartInput<'_, Config>) { - input - .thread_store - .insert(MemoriesExtensionConfig::from_config(input.config)); + fn on_thread_start<'a>( + &'a self, + input: ThreadStartInput<'a, Config>, + ) -> ExtensionFuture<'a, ()> { + Box::pin(async move { + input + .thread_store + .insert(MemoriesExtensionConfig::from_config(input.config)); + }) } } diff --git a/codex-rs/ext/skills/Cargo.toml b/codex-rs/ext/skills/Cargo.toml index 9ff42cfafc..c67d4b0e1b 100644 --- a/codex-rs/ext/skills/Cargo.toml +++ b/codex-rs/ext/skills/Cargo.toml @@ -14,7 +14,6 @@ doctest = false workspace = true [dependencies] -async-trait = { workspace = true } codex-core = { workspace = true } codex-core-skills = { workspace = true } codex-exec-server = { workspace = true } @@ -24,5 +23,6 @@ codex-utils-absolute-path = { workspace = true } codex-utils-string = { workspace = true } [dev-dependencies] +async-trait = { workspace = true } pretty_assertions = { workspace = true } tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/codex-rs/ext/skills/src/extension.rs b/codex-rs/ext/skills/src/extension.rs index 45d7c95cb7..e876c66b29 100644 --- a/codex-rs/ext/skills/src/extension.rs +++ b/codex-rs/ext/skills/src/extension.rs @@ -10,6 +10,7 @@ use codex_extension_api::ContextContributor; use codex_extension_api::ContextualUserFragment; use codex_extension_api::ExtensionData; use codex_extension_api::ExtensionEventSink; +use codex_extension_api::ExtensionFuture; use codex_extension_api::ExtensionRegistryBuilder; use codex_extension_api::PromptFragment; use codex_extension_api::ThreadLifecycleContributor; @@ -44,18 +45,22 @@ struct SkillsExtension { event_sink: Arc, } -#[async_trait::async_trait] impl ThreadLifecycleContributor for SkillsExtension { - async fn on_thread_start(&self, input: ThreadStartInput<'_, Config>) { - let selected_roots = input - .thread_store - .get::>() - .map(|selected_roots| selected_roots.as_ref().clone()) - .unwrap_or_default(); - input.thread_store.insert(SkillsThreadState::new( - SkillsExtensionConfig::from_config(input.config), - selected_roots, - )); + fn on_thread_start<'a>( + &'a self, + input: ThreadStartInput<'a, Config>, + ) -> ExtensionFuture<'a, ()> { + Box::pin(async move { + let selected_roots = input + .thread_store + .get::>() + .map(|selected_roots| selected_roots.as_ref().clone()) + .unwrap_or_default(); + input.thread_store.insert(SkillsThreadState::new( + SkillsExtensionConfig::from_config(input.config), + selected_roots, + )); + }) } } @@ -112,112 +117,117 @@ impl ContextContributor for SkillsExtension { } } -#[async_trait::async_trait] impl TurnInputContributor for SkillsExtension { - async fn contribute( - &self, + fn contribute<'a>( + &'a self, input: TurnInputContext, - _session_store: &ExtensionData, - thread_store: &ExtensionData, - turn_store: &ExtensionData, - ) -> Vec> { - let Some(thread_state) = thread_store.get::() else { - return Vec::new(); - }; + _session_store: &'a ExtensionData, + thread_store: &'a ExtensionData, + turn_store: &'a ExtensionData, + ) -> ExtensionFuture<'a, Vec>> { + Box::pin(async move { + let Some(thread_state) = thread_store.get::() else { + return Vec::new(); + }; - let config = thread_state.config(); - let host_loaded_skills = turn_store.get::(); - let query = SkillListQuery { - turn_id: input.turn_id.clone(), - executor_roots: thread_state.selected_roots().to_vec(), - host: host_loaded_skills.clone(), - include_host_skills: true, - include_bundled_skills: config.bundled_skills_enabled, - include_remote_skills: true, - }; - let catalog = self.providers.list_for_turn(query).await; - for warning in &catalog.warnings { - self.emit_warning(&input.turn_id, warning.clone()); - } - - let selected_entries = collect_explicit_skill_mentions(&input.user_input, &catalog); - let mut fragments: Vec> = Vec::new(); - if config.include_instructions { - let mut turn_catalog = catalog.clone(); - turn_catalog - .entries - .retain(|entry| entry.authority.kind != SkillSourceKind::Executor); - if let Some(fragment) = available_skills_fragment(&turn_catalog) { - fragments.push(Box::new(fragment)); + let config = thread_state.config(); + let host_loaded_skills = turn_store.get::(); + let query = SkillListQuery { + turn_id: input.turn_id.clone(), + executor_roots: thread_state.selected_roots().to_vec(), + host: host_loaded_skills.clone(), + include_host_skills: true, + include_bundled_skills: config.bundled_skills_enabled, + include_remote_skills: true, + }; + let catalog = self.providers.list_for_turn(query).await; + for warning in &catalog.warnings { + self.emit_warning(&input.turn_id, warning.clone()); } - } - let mut warnings = catalog.warnings.clone(); - let mut main_prompts_injected = false; - let mut injected_host_skill_prompts = InjectedHostSkillPrompts::default(); - for entry in &selected_entries { - match self - .read_main_prompt(entry, host_loaded_skills.clone()) - .await - { - Ok(read_result) => { - let (contents, truncated) = - truncate_main_prompt_contents(read_result.contents.as_str()); - if truncated { - let warning = format!( - "Skill `{}` exceeded the main prompt context limit and was truncated.", - entry.name - ); + let selected_entries = collect_explicit_skill_mentions(&input.user_input, &catalog); + let mut fragments: Vec> = Vec::new(); + if config.include_instructions { + let mut turn_catalog = catalog.clone(); + turn_catalog + .entries + .retain(|entry| entry.authority.kind != SkillSourceKind::Executor); + if let Some(fragment) = available_skills_fragment(&turn_catalog) { + fragments.push(Box::new(fragment)); + } + } + + let mut warnings = catalog.warnings.clone(); + let mut main_prompts_injected = false; + let mut injected_host_skill_prompts = InjectedHostSkillPrompts::default(); + for entry in &selected_entries { + match self + .read_main_prompt(entry, host_loaded_skills.clone()) + .await + { + Ok(read_result) => { + let (contents, truncated) = + truncate_main_prompt_contents(read_result.contents.as_str()); + if truncated { + let warning = format!( + "Skill `{}` exceeded the main prompt context limit and was truncated.", + entry.name + ); + self.emit_warning(&input.turn_id, warning.clone()); + warnings.push(warning); + } + let injection = SkillInjection { + name: truncate_utf8_to_bytes(&entry.name, MAX_SKILL_NAME_BYTES).0, + path: truncate_utf8_to_bytes( + entry.rendered_path(), + MAX_SKILL_PATH_BYTES, + ) + .0, + contents, + }; + fragments.push(Box::new(SkillInstructions::from(&injection))); + main_prompts_injected = true; + if entry.authority.kind == SkillSourceKind::Host { + injected_host_skill_prompts.insert_path(entry.main_prompt.as_str()); + } + } + Err(message) => { + let warning = format!("Failed to load skill `{}`: {message}", entry.name); self.emit_warning(&input.turn_id, warning.clone()); warnings.push(warning); } - let injection = SkillInjection { - name: truncate_utf8_to_bytes(&entry.name, MAX_SKILL_NAME_BYTES).0, - path: truncate_utf8_to_bytes(entry.rendered_path(), MAX_SKILL_PATH_BYTES).0, - contents, - }; - fragments.push(Box::new(SkillInstructions::from(&injection))); - main_prompts_injected = true; - if entry.authority.kind == SkillSourceKind::Host { - injected_host_skill_prompts.insert_path(entry.main_prompt.as_str()); + } + } + + if let Some(host_loaded_skills) = &host_loaded_skills { + for entry in selected_entries + .iter() + .filter(|entry| entry.authority.kind != SkillSourceKind::Host) + { + for host_skill in host_loaded_skills + .outcome() + .skills + .iter() + .filter(|host_skill| host_skill.name == entry.name) + { + injected_host_skill_prompts + .insert_path(host_skill.path_to_skills_md.to_string_lossy()); } } - Err(message) => { - let warning = format!("Failed to load skill `{}`: {message}", entry.name); - self.emit_warning(&input.turn_id, warning.clone()); - warnings.push(warning); - } } - } - if let Some(host_loaded_skills) = &host_loaded_skills { - for entry in selected_entries - .iter() - .filter(|entry| entry.authority.kind != SkillSourceKind::Host) - { - for host_skill in host_loaded_skills - .outcome() - .skills - .iter() - .filter(|host_skill| host_skill.name == entry.name) - { - injected_host_skill_prompts - .insert_path(host_skill.path_to_skills_md.to_string_lossy()); - } + turn_store.insert(SkillsTurnState { + catalog, + selected_entries, + warnings, + main_prompts_injected, + }); + if !injected_host_skill_prompts.is_empty() { + turn_store.insert(injected_host_skill_prompts); } - } - turn_store.insert(SkillsTurnState { - catalog, - selected_entries, - warnings, - main_prompts_injected, - }); - if !injected_host_skill_prompts.is_empty() { - turn_store.insert(injected_host_skill_prompts); - } - - fragments + fragments + }) } } diff --git a/codex-rs/ext/web-search/src/extension.rs b/codex-rs/ext/web-search/src/extension.rs index d081d4fb29..688b504dec 100644 --- a/codex-rs/ext/web-search/src/extension.rs +++ b/codex-rs/ext/web-search/src/extension.rs @@ -9,6 +9,7 @@ use codex_api::SearchSettings; use codex_core::config::Config; use codex_extension_api::ConfigContributor; use codex_extension_api::ExtensionData; +use codex_extension_api::ExtensionFuture; use codex_extension_api::ExtensionRegistryBuilder; use codex_extension_api::ThreadLifecycleContributor; use codex_extension_api::ThreadStartInput; @@ -80,12 +81,16 @@ fn search_settings(config: &Config, web_search_mode: WebSearchMode) -> SearchSet } } -#[async_trait::async_trait] impl ThreadLifecycleContributor for WebSearchExtension { - async fn on_thread_start(&self, input: ThreadStartInput<'_, Config>) { - input - .thread_store - .insert(WebSearchExtensionConfig::from(input.config)); + fn on_thread_start<'a>( + &'a self, + input: ThreadStartInput<'a, Config>, + ) -> ExtensionFuture<'a, ()> { + Box::pin(async move { + input + .thread_store + .insert(WebSearchExtensionConfig::from(input.config)); + }) } } From 41b4fabbb4fb2a5d9d956d12c706daee992a76e2 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Wed, 10 Jun 2026 08:33:21 -0700 Subject: [PATCH 31/47] Use latest-wins MCP manager replacement (#27259) ## Summary We originally addressed startup prewarming holding the read side of `RwLock` by snapshotting tool-list state. Review feedback identified the broader ownership problem: the outer synchronization should only publish or retrieve the current manager, while MCP operations rely on the manager's internal synchronization. A follow-up preserved operation retirement with a separate gate, but further review questioned whether that synchronization was actually required and whether we could support latest-wins replacement instead. This PR now stores the current MCP manager in `ArcSwap`. Each operation uses `load_full()` to obtain an owned `Arc`, then performs MCP I/O without retaining the publication mechanism. Refresh cancels obsolete startup work, constructs a replacement, and atomically publishes it. New operations see the latest manager, while operations that already loaded the previous manager retain a valid handle. Refresh happens at a turn boundary, so there should be no active user tool calls to drain. Git history supports dropping the outer `RwLock`. It was introduced in `03ffe4d595` on November 17, 2025 for non-blocking MCP startup: the session published an empty manager, startup initialized that same object while holding the write lock, and readers waited for initialization. `7cd2e84026` on February 19, 2026 removed that two-phase initialization in favor of constructing a fresh manager and swapping it in, explicitly noting that `Option` or `OnceCell` could replace the placeholder design. Hot reload later reused the existing lock to publish a replacement, but I found no indication that the lock was introduced to guarantee in-flight tool calls finish before refresh or shutdown. Terminal shutdown remains separate from refresh: it aborts startup prewarming and active tasks before shutting down the current manager, so tool calls may be interrupted and no model WebSocket work continues after shutdown. Focused regression coverage exercises pending tool-list cancellation, deferred refresh, and startup-prewarm shutdown. --- codex-rs/codex-mcp/src/connection_manager.rs | 20 ++------ .../codex-mcp/src/connection_manager_tests.rs | 42 +++++++++++++++++ codex-rs/core/src/codex_delegate_tests.rs | 40 ++-------------- codex-rs/core/src/connectors.rs | 2 +- codex-rs/core/src/mcp_tool_call.rs | 35 +++----------- codex-rs/core/src/mcp_tool_call_tests.rs | 5 +- codex-rs/core/src/session/handlers.rs | 13 ++++-- codex-rs/core/src/session/mcp.rs | 44 ++++-------------- codex-rs/core/src/session/mod.rs | 8 +--- codex-rs/core/src/session/session.rs | 4 +- codex-rs/core/src/session/tests.rs | 18 +++----- codex-rs/core/src/session/turn.rs | 19 +------- codex-rs/core/src/session/turn_context.rs | 2 +- codex-rs/core/src/session_startup_prewarm.rs | 10 +++- codex-rs/core/src/state/service.rs | 13 ++---- .../list_mcp_resource_templates.rs | 7 +-- .../mcp_resource/list_mcp_resources.rs | 7 +-- .../tools/handlers/request_plugin_install.rs | 6 +-- codex-rs/core/src/tools/router_tests.rs | 7 +-- codex-rs/core/tests/suite/rmcp_client.rs | 46 +++++++++++++++++++ 20 files changed, 156 insertions(+), 192 deletions(-) diff --git a/codex-rs/codex-mcp/src/connection_manager.rs b/codex-rs/codex-mcp/src/connection_manager.rs index 5b7b6b3e81..659d357189 100644 --- a/codex-rs/codex-mcp/src/connection_manager.rs +++ b/codex-rs/codex-mcp/src/connection_manager.rs @@ -348,22 +348,12 @@ impl McpConnectionManager { !self.clients.is_empty() } - /// Drain all MCP clients from this manager and return a future that stops - /// them and terminates their stdio server processes. - pub fn begin_shutdown(&mut self) -> impl std::future::Future + Send + 'static { - self.startup_cancellation_token.cancel(); - let clients = std::mem::take(&mut self.clients); - self.server_metadata.clear(); - async move { - for client in clients.into_values() { - client.shutdown().await; - } - } - } - /// Stop all MCP clients owned by this manager and terminate stdio server processes. - pub async fn shutdown(&mut self) { - self.begin_shutdown().await; + pub async fn shutdown(&self) { + self.startup_cancellation_token.cancel(); + for client in self.clients.values() { + client.shutdown().await; + } } pub fn server_origin(&self, server_name: &str) -> Option<&str> { diff --git a/codex-rs/codex-mcp/src/connection_manager_tests.rs b/codex-rs/codex-mcp/src/connection_manager_tests.rs index 4b90337ac2..ea405ced9a 100644 --- a/codex-rs/codex-mcp/src/connection_manager_tests.rs +++ b/codex-rs/codex-mcp/src/connection_manager_tests.rs @@ -980,6 +980,48 @@ async fn list_all_tools_blocks_while_client_is_pending_without_cached_tool_info_ assert!(timeout_result.is_err()); } +#[tokio::test] +async fn shutdown_cancels_pending_tool_listing() { + let cancel_token = CancellationToken::new(); + let cancel_token_for_startup = cancel_token.clone(); + let (started_tx, started_rx) = tokio::sync::oneshot::channel(); + let pending_client = async move { + let _ = started_tx.send(()); + cancel_token_for_startup.cancelled().await; + Err(StartupOutcomeError::Cancelled) + } + .boxed() + .shared(); + let approval_policy = Constrained::allow_any(AskForApproval::OnFailure); + let permission_profile = Constrained::allow_any(PermissionProfile::default()); + let mut manager = McpConnectionManager::new_uninitialized( + &approval_policy, + &permission_profile, + /*prefix_mcp_tool_names*/ true, + ); + manager.clients.insert( + CODEX_APPS_MCP_SERVER_NAME.to_string(), + AsyncManagedClient { + client: pending_client, + cached_tool_info_snapshot: None, + cached_server_info: None, + startup_complete: Arc::new(std::sync::atomic::AtomicBool::new(false)), + tool_plugin_provenance: Arc::new(ToolPluginProvenance::default()), + cancel_token, + }, + ); + let manager = Arc::new(manager); + let manager_for_list = Arc::clone(&manager); + let list_task = tokio::spawn(async move { manager_for_list.list_all_tools().await }); + + started_rx.await.expect("tool listing should start"); + tokio::time::timeout(Duration::from_secs(1), manager.shutdown()) + .await + .expect("shutdown should cancel speculative tool listing"); + let tools = list_task.await.expect("tool listing task should not panic"); + assert!(tools.is_empty()); +} + #[tokio::test] async fn list_all_tools_does_not_block_when_cached_tool_info_snapshot_is_empty() { let pending_client = futures::future::pending::>() diff --git a/codex-rs/core/src/codex_delegate_tests.rs b/codex-rs/core/src/codex_delegate_tests.rs index 7dabb09d33..cb7767a16c 100644 --- a/codex-rs/core/src/codex_delegate_tests.rs +++ b/codex-rs/core/src/codex_delegate_tests.rs @@ -452,7 +452,7 @@ async fn delegated_mcp_guardian_abort_returns_synthetic_decline_answer() { } #[tokio::test] -async fn delegated_mcp_user_reviewer_waits_for_metadata_lookup() { +async fn delegated_mcp_user_reviewer_returns_none_without_metadata() { let (parent_session, parent_ctx, _rx_events) = crate::session::tests::make_session_and_context_with_rx().await; let pending_mcp_invocations = Arc::new(Mutex::new(HashMap::from([( @@ -464,21 +464,6 @@ async fn delegated_mcp_user_reviewer_waits_for_metadata_lookup() { }, )]))); let cancel_token = CancellationToken::new(); - let manager = Arc::clone(&parent_session.services.mcp_connection_manager); - let (manager_locked_tx, manager_locked_rx) = std::sync::mpsc::sync_channel(0); - let (release_manager_tx, release_manager_rx) = std::sync::mpsc::sync_channel(0); - let manager_lock = tokio::task::spawn_blocking(move || { - let _manager_guard = manager.blocking_write(); - manager_locked_tx - .send(()) - .expect("manager lock receiver should remain open"); - release_manager_rx - .recv() - .expect("manager lock release sender should remain open"); - }); - manager_locked_rx - .recv_timeout(Duration::from_secs(1)) - .expect("manager write lock should be acquired"); let event = RequestUserInputEvent { call_id: "call-1".to_string(), @@ -498,24 +483,7 @@ async fn delegated_mcp_user_reviewer_waits_for_metadata_lookup() { &pending_mcp_invocations, &event, &cancel_token, - ); - tokio::pin!(response); - assert!( - timeout(Duration::from_millis(100), &mut response) - .await - .is_err(), - "manual reviewer should wait for MCP metadata" - ); - release_manager_tx - .send(()) - .expect("manager lock holder should remain open"); - manager_lock - .await - .expect("manager lock task should not panic"); - assert_eq!( - timeout(Duration::from_secs(1), response) - .await - .expect("manual reviewer should finish after MCP metadata lookup"), - None - ); + ) + .await; + assert_eq!(response, None); } diff --git a/codex-rs/core/src/connectors.rs b/codex-rs/core/src/connectors.rs index 6d6ca52d8c..d8df728554 100644 --- a/codex-rs/core/src/connectors.rs +++ b/codex-rs/core/src/connectors.rs @@ -287,7 +287,7 @@ pub async fn list_accessible_connectors_from_mcp_tools_with_mcp_manager( drop(rx_event); let cancel_token = CancellationToken::new(); - let mut mcp_connection_manager = McpConnectionManager::new( + let mcp_connection_manager = McpConnectionManager::new( &mcp_servers, config.mcp_oauth_credentials_store_mode, auth_status_entries, diff --git a/codex-rs/core/src/mcp_tool_call.rs b/codex-rs/core/src/mcp_tool_call.rs index b9a8d4ff08..e5273a6669 100644 --- a/codex-rs/core/src/mcp_tool_call.rs +++ b/codex-rs/core/src/mcp_tool_call.rs @@ -319,8 +319,7 @@ async fn handle_approved_mcp_tool_call( let server_origin = sess .services .mcp_connection_manager - .read() - .await + .load_full() .server_origin(&server) .map(str::to_string); @@ -606,8 +605,7 @@ async fn maybe_request_codex_apps_auth_elicitation( if !sess .services .mcp_connection_manager - .read() - .await + .load_full() .is_host_owned_codex_apps_server(server) { return result; @@ -669,13 +667,9 @@ async fn maybe_request_codex_apps_auth_elicitation( auth_elicitation_completed_result(&plan.auth_failure, result.meta) } -#[expect( - clippy::await_holding_invalid_type, - reason = "Codex Apps cache refresh reads through the session-owned manager guard" -)] async fn refresh_codex_apps_after_connector_auth(sess: &Session, turn_context: &TurnContext) { let mcp_tools_result = { - let manager = sess.services.mcp_connection_manager.read().await; + let manager = sess.services.mcp_connection_manager.load_full(); manager.hard_refresh_codex_apps_tools_cache().await }; @@ -694,10 +688,6 @@ async fn refresh_codex_apps_after_connector_auth(sess: &Session, turn_context: & } } -#[expect( - clippy::await_holding_invalid_type, - reason = "MCP sandbox metadata reads through the session-owned manager guard" -)] async fn augment_mcp_tool_request_meta_with_sandbox_state( sess: &Session, turn_context: &TurnContext, @@ -707,8 +697,7 @@ async fn augment_mcp_tool_request_meta_with_sandbox_state( let supports_sandbox_state_meta = sess .services .mcp_connection_manager - .read() - .await + .load_full() .server_supports_sandbox_state_meta_capability(server) .await .unwrap_or(false); @@ -757,8 +746,7 @@ async fn maybe_mark_thread_memory_mode_polluted( let pollutes_memory = sess .services .mcp_connection_manager - .read() - .await + .load_full() .server_pollutes_memory(server); if !pollutes_memory { return; @@ -1414,17 +1402,13 @@ async fn mcp_tool_approval_decision_from_guardian( } } -#[expect( - clippy::await_holding_invalid_type, - reason = "MCP approval metadata reads through the session-owned manager guard" -)] pub(crate) async fn lookup_mcp_tool_metadata( sess: &Session, turn_context: &TurnContext, server: &str, tool_name: &str, ) -> Option { - let manager = sess.services.mcp_connection_manager.read().await; + let manager = sess.services.mcp_connection_manager.load_full(); let plugin_id = manager .plugin_id_for_mcp_server_name(server) .map(str::to_string); @@ -1509,10 +1493,6 @@ fn get_mcp_app_resource_uri( }) } -#[expect( - clippy::await_holding_invalid_type, - reason = "MCP app metadata reads through the session-owned manager guard" -)] async fn lookup_mcp_app_usage_metadata( sess: &Session, server: &str, @@ -1521,8 +1501,7 @@ async fn lookup_mcp_app_usage_metadata( let tools = sess .services .mcp_connection_manager - .read() - .await + .load_full() .list_all_tools() .await; diff --git a/codex-rs/core/src/mcp_tool_call_tests.rs b/codex-rs/core/src/mcp_tool_call_tests.rs index da54fe1873..723948b7be 100644 --- a/codex-rs/core/src/mcp_tool_call_tests.rs +++ b/codex-rs/core/src/mcp_tool_call_tests.rs @@ -1280,7 +1280,10 @@ async fn install_host_owned_codex_apps_manager(session: &Session, turn_context: /*elicitation_reviewer*/ None, ) .await; - *session.services.mcp_connection_manager.write().await = manager; + session + .services + .mcp_connection_manager + .store(Arc::new(manager)); } #[tokio::test] diff --git a/codex-rs/core/src/session/handlers.rs b/codex-rs/core/src/session/handlers.rs index 10b0b27cc1..bfb87a0ff3 100644 --- a/codex-rs/core/src/session/handlers.rs +++ b/codex-rs/core/src/session/handlers.rs @@ -580,6 +580,9 @@ pub async fn set_thread_memory_mode(sess: &Arc, sub_id: String, mode: T } async fn shutdown_session_runtime(sess: &Arc) { + if let Some(startup_prewarm) = sess.take_session_startup_prewarm().await { + startup_prewarm.abort().await; + } sess.abort_all_tasks(TurnAbortReason::Interrupted).await; let _ = sess.conversation.shutdown().await; sess.services @@ -589,11 +592,11 @@ async fn shutdown_session_runtime(sess: &Arc) { if let Err(err) = sess.services.code_mode_service.shutdown().await { warn!("failed to shutdown code mode session: {err}"); } - let mcp_shutdown = { - let mut manager = sess.services.mcp_connection_manager.write().await; - manager.begin_shutdown() - }; - mcp_shutdown.await; + sess.services + .mcp_connection_manager + .load_full() + .shutdown() + .await; sess.guardian_review_session.shutdown().await; } diff --git a/codex-rs/core/src/session/mcp.rs b/codex-rs/core/src/session/mcp.rs index 7798136418..39333801c0 100644 --- a/codex-rs/core/src/session/mcp.rs +++ b/codex-rs/core/src/session/mcp.rs @@ -91,8 +91,7 @@ impl Session { if self .services .mcp_connection_manager - .read() - .await + .load_full() .elicitations_auto_deny() { return McpServerElicitationOutcome { @@ -226,16 +225,11 @@ impl Session { self.services .mcp_connection_manager - .read() - .await + .load_full() .resolve_elicitation(server_name, id, response) .await } - #[expect( - clippy::await_holding_invalid_type, - reason = "MCP resource calls are serialized through the session-owned manager guard" - )] pub async fn list_resources( &self, server: &str, @@ -243,16 +237,11 @@ impl Session { ) -> anyhow::Result { self.services .mcp_connection_manager - .read() - .await + .load_full() .list_resources(server, params) .await } - #[expect( - clippy::await_holding_invalid_type, - reason = "MCP resource calls are serialized through the session-owned manager guard" - )] pub async fn list_resource_templates( &self, server: &str, @@ -260,16 +249,11 @@ impl Session { ) -> anyhow::Result { self.services .mcp_connection_manager - .read() - .await + .load_full() .list_resource_templates(server, params) .await } - #[expect( - clippy::await_holding_invalid_type, - reason = "MCP resource calls are serialized through the session-owned manager guard" - )] pub async fn read_resource( &self, server: &str, @@ -277,16 +261,11 @@ impl Session { ) -> anyhow::Result { self.services .mcp_connection_manager - .read() - .await + .load_full() .read_resource(server, params) .await } - #[expect( - clippy::await_holding_invalid_type, - reason = "MCP tool calls are serialized through the session-owned manager guard" - )] pub async fn call_tool( &self, server: &str, @@ -296,8 +275,7 @@ impl Session { ) -> anyhow::Result { self.services .mcp_connection_manager - .read() - .await + .load_full() .call_tool(server, tool, arguments, meta) .await } @@ -366,14 +344,12 @@ impl Session { ) .await; { - let current_manager = self.services.mcp_connection_manager.read().await; + let current_manager = self.services.mcp_connection_manager.load_full(); refreshed_manager.set_elicitations_auto_deny(current_manager.elicitations_auto_deny()); } - let mut old_manager = { - let mut manager = self.services.mcp_connection_manager.write().await; - std::mem::replace(&mut *manager, refreshed_manager) - }; - old_manager.shutdown().await; + self.services + .mcp_connection_manager + .store(Arc::new(refreshed_manager)); } pub(crate) async fn refresh_mcp_servers_if_requested( diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs index 2e5d5643da..7cc091e43f 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -793,7 +793,7 @@ impl Codex { ..Default::default() }) .await?; - let mcp_connection_manager = self.session.services.mcp_connection_manager.read().await; + let mcp_connection_manager = self.session.services.mcp_connection_manager.load_full(); mcp_connection_manager.set_elicitations_auto_deny(mcp_elicitations_auto_deny); Ok(()) } @@ -2748,10 +2748,6 @@ impl Session { } } - #[expect( - clippy::await_holding_invalid_type, - reason = "MCP app context rendering reads through the session-owned manager guard" - )] pub(crate) async fn build_initial_context( &self, turn_context: &TurnContext, @@ -2844,7 +2840,7 @@ impl Session { } } if turn_context.config.include_apps_instructions && turn_context.apps_enabled() { - let mcp_connection_manager = self.services.mcp_connection_manager.read().await; + let mcp_connection_manager = self.services.mcp_connection_manager.load_full(); let accessible_and_enabled_connectors = connectors::list_accessible_and_enabled_connectors_from_manager( &mcp_connection_manager, diff --git a/codex-rs/core/src/session/session.rs b/codex-rs/core/src/session/session.rs index 98d19957c0..655454a320 100644 --- a/codex-rs/core/src/session/session.rs +++ b/codex-rs/core/src/session/session.rs @@ -981,13 +981,13 @@ impl Session { // before any MCP-related events. It is reasonable to consider // changing this to use Option or OnceCell, though the current // setup is straightforward enough and performs well. - mcp_connection_manager: Arc::new(RwLock::new( + mcp_connection_manager: arc_swap::ArcSwap::from_pointee( McpConnectionManager::new_uninitialized_with_permission_profile( &config.permissions.approval_policy, config.permissions.permission_profile(), config.prefix_mcp_tool_names(), ), - )), + ), mcp_startup_cancellation_token: Mutex::new(CancellationToken::new()), unified_exec_manager: UnifiedExecProcessManager::new( config.background_terminal_max_timeout, diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs index b83217293d..8d8208205e 100644 --- a/codex-rs/core/src/session/tests.rs +++ b/codex-rs/core/src/session/tests.rs @@ -319,8 +319,7 @@ async fn request_mcp_server_elicitation_auto_accepts_when_auto_deny_is_enabled() session .services .mcp_connection_manager - .read() - .await + .load_full() .set_elicitations_auto_deny(/*auto_deny*/ true); let requested_schema: McpElicitationSchema = serde_json::from_value(json!({ @@ -4816,13 +4815,13 @@ pub(crate) async fn make_session_and_context() -> (Session, TurnContext) { ); let services = SessionServices { - mcp_connection_manager: Arc::new(RwLock::new( + mcp_connection_manager: arc_swap::ArcSwap::from_pointee( McpConnectionManager::new_uninitialized_with_permission_profile( &config.permissions.approval_policy, config.permissions.permission_profile(), config.prefix_mcp_tool_names(), ), - )), + ), mcp_startup_cancellation_token: Mutex::new(CancellationToken::new()), unified_exec_manager: UnifiedExecProcessManager::new( config.background_terminal_max_timeout, @@ -6894,13 +6893,13 @@ where ); let services = SessionServices { - mcp_connection_manager: Arc::new(RwLock::new( + mcp_connection_manager: arc_swap::ArcSwap::from_pointee( McpConnectionManager::new_uninitialized_with_permission_profile( &config.permissions.approval_policy, config.permissions.permission_profile(), config.prefix_mcp_tool_names(), ), - )), + ), mcp_startup_cancellation_token: Mutex::new(CancellationToken::new()), unified_exec_manager: UnifiedExecProcessManager::new( config.background_terminal_max_timeout, @@ -9440,18 +9439,13 @@ async fn abort_review_task_emits_exited_then_aborted_and_records_history() { } #[tokio::test] -#[expect( - clippy::await_holding_invalid_type, - reason = "test builds a router from session-owned MCP manager state" -)] async fn fatal_tool_error_stops_turn_and_reports_error() { let (session, turn_context, _rx) = make_session_and_context_with_rx().await; let tools = { session .services .mcp_connection_manager - .read() - .await + .load_full() .list_all_tools() .await }; diff --git a/codex-rs/core/src/session/turn.rs b/codex-rs/core/src/session/turn.rs index 95d24dc03d..722957d30c 100644 --- a/codex-rs/core/src/session/turn.rs +++ b/codex-rs/core/src/session/turn.rs @@ -432,10 +432,6 @@ async fn run_hooks_and_record_inputs( blocked_input && !accepted_user_input } -#[expect( - clippy::await_holding_invalid_type, - reason = "MCP tool listing borrows the read guard across cancellation-aware await" -)] #[instrument(level = "trace", skip_all)] async fn build_skills_and_plugins( sess: &Arc, @@ -473,8 +469,7 @@ async fn build_skills_and_plugins( match sess .services .mcp_connection_manager - .read() - .await + .load_full() .list_all_tools() .or_cancel(cancellation_token) .await @@ -1078,10 +1073,6 @@ async fn run_sampling_request( } } -#[expect( - clippy::await_holding_invalid_type, - reason = "tool router construction reads through the session-owned manager guard" -)] #[instrument(level = "trace", skip_all, fields( @@ -1095,18 +1086,12 @@ pub(crate) async fn built_tools( turn_context: &TurnContext, cancellation_token: &CancellationToken, ) -> CodexResult> { - let mcp_connection_manager = sess - .services - .mcp_connection_manager - .read() - .instrument(trace_span!("read_mcp_connection_manager")) - .await; + let mcp_connection_manager = sess.services.mcp_connection_manager.load_full(); let has_mcp_servers = mcp_connection_manager.has_servers(); let all_mcp_tools = mcp_connection_manager .list_all_tools() .or_cancel(cancellation_token) .await?; - drop(mcp_connection_manager); let loaded_plugins = sess .services .plugins_manager diff --git a/codex-rs/core/src/session/turn_context.rs b/codex-rs/core/src/session/turn_context.rs index e3bd01081a..10eff3f3b4 100644 --- a/codex-rs/core/src/session/turn_context.rs +++ b/codex-rs/core/src/session/turn_context.rs @@ -724,7 +724,7 @@ impl Session { .unwrap_or_else(|| session_configuration.cwd().clone()); let per_turn_config = Self::build_per_turn_config(&session_configuration, cwd.clone()); { - let mcp_connection_manager = self.services.mcp_connection_manager.read().await; + let mcp_connection_manager = self.services.mcp_connection_manager.load_full(); mcp_connection_manager.set_approval_policy(&session_configuration.approval_policy); mcp_connection_manager .set_permission_profile(session_configuration.permission_profile()); diff --git a/codex-rs/core/src/session_startup_prewarm.rs b/codex-rs/core/src/session_startup_prewarm.rs index ba2cc42f29..f4948ae2f2 100644 --- a/codex-rs/core/src/session_startup_prewarm.rs +++ b/codex-rs/core/src/session_startup_prewarm.rs @@ -4,6 +4,7 @@ use std::time::Instant; use tokio::task::JoinHandle; use tokio_util::sync::CancellationToken; +use tokio_util::task::AbortOnDropHandle; use tracing::info; use tracing::warn; @@ -19,7 +20,7 @@ use codex_protocol::error::Result as CodexResult; use codex_protocol::models::BaseInstructions; pub(crate) struct SessionStartupPrewarmHandle { - task: JoinHandle>, + task: AbortOnDropHandle>, started_at: Instant, timeout: Duration, } @@ -40,12 +41,17 @@ impl SessionStartupPrewarmHandle { timeout: Duration, ) -> Self { Self { - task, + task: AbortOnDropHandle::new(task), started_at, timeout, } } + pub(crate) async fn abort(self) { + self.task.abort(); + let _ = self.task.await; + } + async fn resolve( self, session_telemetry: &SessionTelemetry, diff --git a/codex-rs/core/src/state/service.rs b/codex-rs/core/src/state/service.rs index 9bc4d73915..b4c499118d 100644 --- a/codex-rs/core/src/state/service.rs +++ b/codex-rs/core/src/state/service.rs @@ -35,12 +35,12 @@ use codex_thread_store::ThreadStore; use std::path::PathBuf; use tokio::runtime::Handle; use tokio::sync::Mutex; -use tokio::sync::RwLock; use tokio::sync::watch; use tokio_util::sync::CancellationToken; pub(crate) struct SessionServices { - pub(crate) mcp_connection_manager: Arc>, + /// The latest manager; callers retain an owned handle while performing MCP I/O. + pub(crate) mcp_connection_manager: ArcSwap, pub(crate) mcp_startup_cancellation_token: Mutex, pub(crate) unified_exec_manager: UnifiedExecProcessManager, #[cfg_attr(not(unix), allow(dead_code))] @@ -87,18 +87,13 @@ pub(crate) struct SessionServices { impl SessionServices { /// Installs the manager before validating required servers so startup-time elicitation can /// resolve through the session's manager while validation waits. - #[expect( - clippy::await_holding_invalid_type, - reason = "required MCP validation keeps the installed manager reachable for startup-time elicitation" - )] pub(crate) async fn install_mcp_connection_manager( &self, manager: McpConnectionManager, ) -> Result<()> { - *self.mcp_connection_manager.write().await = manager; + self.mcp_connection_manager.store(Arc::new(manager)); self.mcp_connection_manager - .read() - .await + .load_full() .validate_required_servers() .await } diff --git a/codex-rs/core/src/tools/handlers/mcp_resource/list_mcp_resource_templates.rs b/codex-rs/core/src/tools/handlers/mcp_resource/list_mcp_resource_templates.rs index c46df13b18..e96aad7552 100644 --- a/codex-rs/core/src/tools/handlers/mcp_resource/list_mcp_resource_templates.rs +++ b/codex-rs/core/src/tools/handlers/mcp_resource/list_mcp_resource_templates.rs @@ -40,10 +40,6 @@ impl ToolExecutor for ListMcpResourceTemplatesHandler { true } - #[expect( - clippy::await_holding_invalid_type, - reason = "MCP resource template listing reads through the session-owned manager guard" - )] async fn handle( &self, invocation: ToolInvocation, @@ -107,8 +103,7 @@ impl ToolExecutor for ListMcpResourceTemplatesHandler { let templates = session .services .mcp_connection_manager - .read() - .await + .load_full() .list_all_resource_templates() .await; Ok(ListResourceTemplatesPayload::from_all_servers(templates)) diff --git a/codex-rs/core/src/tools/handlers/mcp_resource/list_mcp_resources.rs b/codex-rs/core/src/tools/handlers/mcp_resource/list_mcp_resources.rs index f7fcad3422..1b052184ee 100644 --- a/codex-rs/core/src/tools/handlers/mcp_resource/list_mcp_resources.rs +++ b/codex-rs/core/src/tools/handlers/mcp_resource/list_mcp_resources.rs @@ -40,10 +40,6 @@ impl ToolExecutor for ListMcpResourcesHandler { true } - #[expect( - clippy::await_holding_invalid_type, - reason = "MCP resource listing reads through the session-owned manager guard" - )] async fn handle( &self, invocation: ToolInvocation, @@ -105,8 +101,7 @@ impl ToolExecutor for ListMcpResourcesHandler { let resources = session .services .mcp_connection_manager - .read() - .await + .load_full() .list_all_resources() .await; Ok(ListResourcesPayload::from_all_servers(resources)) diff --git a/codex-rs/core/src/tools/handlers/request_plugin_install.rs b/codex-rs/core/src/tools/handlers/request_plugin_install.rs index 3f7bd92d25..ce5bb3fb1f 100644 --- a/codex-rs/core/src/tools/handlers/request_plugin_install.rs +++ b/codex-rs/core/src/tools/handlers/request_plugin_install.rs @@ -304,10 +304,6 @@ fn is_remote_plugin_install_suggestion(plugin_id: &str) -> bool { .is_some_and(|(_, marketplace_name)| marketplace_name == REMOTE_GLOBAL_MARKETPLACE_NAME) } -#[expect( - clippy::await_holding_invalid_type, - reason = "connector cache refresh reads through the session-owned manager guard" -)] async fn refresh_missing_requested_connectors( session: &crate::session::session::Session, turn: &crate::session::turn_context::TurnContext, @@ -319,7 +315,7 @@ async fn refresh_missing_requested_connectors( return Some(Vec::new()); } - let manager = session.services.mcp_connection_manager.read().await; + let manager = session.services.mcp_connection_manager.load_full(); let mcp_tools = manager.list_all_tools().await; let accessible_connectors = connectors::with_app_enabled_state( connectors::accessible_connectors_from_mcp_tools(&mcp_tools), diff --git a/codex-rs/core/src/tools/router_tests.rs b/codex-rs/core/src/tools/router_tests.rs index cece7fc171..7c3e0157bc 100644 --- a/codex-rs/core/src/tools/router_tests.rs +++ b/codex-rs/core/src/tools/router_tests.rs @@ -95,17 +95,12 @@ fn extension_tool_test_registry() -> Arc> { } #[tokio::test] -#[expect( - clippy::await_holding_invalid_type, - reason = "test builds a router from session-owned MCP manager state" -)] async fn parallel_support_does_not_match_namespaced_local_tool_names() -> anyhow::Result<()> { let (session, turn) = make_session_and_context().await; let mcp_tools = session .services .mcp_connection_manager - .read() - .await + .load_full() .list_all_tools() .await; let router = ToolRouter::from_turn_context( diff --git a/codex-rs/core/tests/suite/rmcp_client.rs b/codex-rs/core/tests/suite/rmcp_client.rs index 5cb61fa4a1..c47aa0970a 100644 --- a/codex-rs/core/tests/suite/rmcp_client.rs +++ b/codex-rs/core/tests/suite/rmcp_client.rs @@ -558,6 +558,52 @@ async fn stdio_server_round_trip() -> anyhow::Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn shutdown_cancels_startup_prewarm_waiting_for_mcp_startup() -> anyhow::Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_websocket_server(vec![vec![vec![ + responses::ev_response_created("warm-1"), + responses::ev_completed("warm-1"), + ]]]) + .await; + let pending_mcp_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let pending_mcp_url = format!("http://{}/mcp", pending_mcp_listener.local_addr()?); + + let fixture = test_codex() + .with_config(move |config| { + insert_mcp_server( + config, + "shutdown_prewarm", + McpServerTransportConfig::StreamableHttp { + url: pending_mcp_url, + bearer_token_env_var: None, + http_headers: None, + env_http_headers: None, + }, + TestMcpServerOptions::default(), + ); + }) + .build_with_websocket_server(&server) + .await?; + + let (_pending_mcp_connection, _) = + tokio::time::timeout(Duration::from_secs(5), pending_mcp_listener.accept()) + .await + .context("startup prewarm should start the MCP connection")??; + tokio::time::timeout(Duration::from_secs(2), fixture.codex.shutdown_and_wait()) + .await + .context("shutdown should not wait for startup prewarm MCP startup")??; + tokio::time::sleep(Duration::from_millis(100)).await; + assert!( + server.connections().is_empty(), + "startup prewarm should not send a websocket request after shutdown" + ); + + server.shutdown().await; + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] #[serial(mcp_cwd)] async fn stdio_server_uses_configured_cwd_before_runtime_fallback() -> anyhow::Result<()> { From 30ddb3325e380d8e32d39324bd9acf931e5217bb Mon Sep 17 00:00:00 2001 From: pakrym-oai Date: Wed, 10 Jun 2026 08:47:16 -0700 Subject: [PATCH 32/47] [codex] Store compact window id in rollout (#27264) ## Why Compaction window identity is part of session history, not model-client transport state. Persisting it with the compacted rollout item lets resumed threads continue from the reconstructed window without keeping mutable window state on `ModelClient`. ## What changed - Added `window_id` to `CompactedItem` and stamp it when `replace_compacted_history` installs compacted history. - Moved auto-compact window id ownership into `AutoCompactWindow` / `SessionState`; `ModelClient` now receives the request window id from callers instead of storing it. - Returned `window_id` from rollout reconstruction for resume. Reconstruction uses the newest surviving compacted item's stored `window_id` when present, and falls back to the legacy compacted-item count when it is absent. - Kept fork startup at the fresh default window id and updated direct model-client tests to pass explicit test window ids. ## Validation - `cargo check -p codex-core --tests` --- .../src/protocol/thread_history.rs | 1 + codex-rs/core/src/agent/control_tests.rs | 1 + codex-rs/core/src/client.rs | 66 +++++++++---------- codex-rs/core/src/client_tests.rs | 6 +- codex-rs/core/src/compact.rs | 6 +- codex-rs/core/src/compact_remote.rs | 4 +- codex-rs/core/src/compact_remote_v2.rs | 6 +- codex-rs/core/src/session/mod.rs | 40 +++++++---- .../src/session/rollout_reconstruction.rs | 22 +++++++ .../session/rollout_reconstruction_tests.rs | 8 +++ codex-rs/core/src/session/session.rs | 14 ---- codex-rs/core/src/session/tests.rs | 11 ++++ codex-rs/core/src/session/turn.rs | 12 ++-- codex-rs/core/src/session_startup_prewarm.rs | 3 +- .../core/src/state/auto_compact_window.rs | 39 ++++++----- codex-rs/core/src/state/session.rs | 16 +++-- codex-rs/core/src/state/session_tests.rs | 4 +- codex-rs/core/tests/responses_headers.rs | 7 +- codex-rs/core/tests/suite/client.rs | 3 + .../core/tests/suite/client_websockets.rs | 23 +++++-- codex-rs/memories/write/src/runtime.rs | 2 + codex-rs/protocol/src/protocol.rs | 2 + codex-rs/rollout/src/metadata_tests.rs | 1 + .../thread-store/src/thread_metadata_sync.rs | 1 + 24 files changed, 192 insertions(+), 106 deletions(-) diff --git a/codex-rs/app-server-protocol/src/protocol/thread_history.rs b/codex-rs/app-server-protocol/src/protocol/thread_history.rs index 075b3e043e..1305ec00fb 100644 --- a/codex-rs/app-server-protocol/src/protocol/thread_history.rs +++ b/codex-rs/app-server-protocol/src/protocol/thread_history.rs @@ -2939,6 +2939,7 @@ mod tests { RolloutItem::Compacted(CompactedItem { message: String::new(), replacement_history: None, + window_id: None, }), RolloutItem::EventMsg(EventMsg::TurnComplete(TurnCompleteEvent { turn_id: "turn-compact".into(), diff --git a/codex-rs/core/src/agent/control_tests.rs b/codex-rs/core/src/agent/control_tests.rs index c626dda4a5..772da10c30 100644 --- a/codex-rs/core/src/agent/control_tests.rs +++ b/codex-rs/core/src/agent/control_tests.rs @@ -1097,6 +1097,7 @@ async fn spawn_agent_fork_strips_parent_usage_hints_from_compacted_history() { RolloutItem::Compacted(CompactedItem { message: String::new(), replacement_history: Some(replacement_history), + window_id: None, }), RolloutItem::TurnContext(turn_context.to_turn_context_item()), RolloutItem::ResponseItem(spawn_agent_call(&parent_spawn_call_id)), diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index 02d231c962..87381fca7b 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -28,7 +28,6 @@ use std::sync::Arc; use std::sync::Mutex as StdMutex; use std::sync::OnceLock; use std::sync::atomic::AtomicBool; -use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; use codex_api::ApiError; @@ -172,7 +171,6 @@ pub(crate) struct CompactConversationRequestSettings { struct ModelClientState { session_id: SessionId, thread_id: ThreadId, - window_generation: AtomicU64, installation_id: String, provider: SharedModelProvider, auth_env_telemetry: AuthEnvTelemetry, @@ -345,7 +343,6 @@ impl ModelClient { state: Arc::new(ModelClientState { session_id, thread_id, - window_generation: AtomicU64::new(0), installation_id, provider: model_provider, auth_env_telemetry, @@ -394,24 +391,6 @@ impl ModelClient { self.state.provider.auth_manager() } - pub(crate) fn set_window_generation(&self, window_generation: u64) { - self.state - .window_generation - .store(window_generation, Ordering::Relaxed); - self.store_cached_websocket_session(WebsocketSession::default()); - } - - pub(crate) fn advance_window_generation(&self) { - self.state.window_generation.fetch_add(1, Ordering::Relaxed); - self.store_cached_websocket_session(WebsocketSession::default()); - } - - pub(crate) fn current_window_id(&self) -> String { - let thread_id = self.state.thread_id; - let window_generation = self.state.window_generation.load(Ordering::Relaxed); - format!("{thread_id}:{window_generation}") - } - fn take_cached_websocket_session(&self) -> WebsocketSession { let mut cached_websocket_session = self .state @@ -457,6 +436,7 @@ impl ModelClient { /// /// The model selection and telemetry context are passed explicitly to keep `ModelClient` /// session-scoped. + #[allow(clippy::too_many_arguments)] pub(crate) async fn compact_conversation_history( &self, prompt: &Prompt, @@ -464,6 +444,7 @@ impl ModelClient { settings: CompactConversationRequestSettings, session_telemetry: &SessionTelemetry, compaction_trace: &CompactionTraceContext, + window_id: &str, turn_metadata_header: Option<&str>, ) -> Result> { if prompt.input.is_empty() { @@ -488,6 +469,7 @@ impl ModelClient { settings.effort, settings.summary, settings.service_tier, + window_id, )?; let ResponsesApiRequest { model, @@ -522,7 +504,7 @@ impl ModelClient { /*turn_state*/ None, parse_turn_metadata_header(turn_metadata_header).as_ref(), )); - extra_headers.extend(self.build_responses_identity_headers()); + extra_headers.extend(self.build_responses_identity_headers(Some(window_id))); extra_headers.extend(build_session_headers( Some(self.state.session_id.to_string()), Some(self.state.thread_id.to_string()), @@ -644,14 +626,16 @@ impl ModelClient { extra_headers } - fn build_responses_identity_headers(&self) -> ApiHeaderMap { + fn build_responses_identity_headers(&self, window_id: Option<&str>) -> ApiHeaderMap { let mut extra_headers = self.build_subagent_headers(); if let Some(parent_thread_id) = parent_thread_id_header_value(self.state.parent_thread_id) && let Ok(val) = HeaderValue::from_str(&parent_thread_id) { extra_headers.insert(X_CODEX_PARENT_THREAD_ID_HEADER, val); } - if let Ok(val) = HeaderValue::from_str(&self.current_window_id()) { + if let Some(window_id) = window_id + && let Ok(val) = HeaderValue::from_str(window_id) + { extra_headers.insert(X_CODEX_WINDOW_ID_HEADER, val); } extra_headers @@ -659,6 +643,7 @@ impl ModelClient { fn build_ws_client_metadata( &self, + window_id: &str, turn_metadata_header: Option<&str>, use_responses_lite: bool, ) -> HashMap { @@ -667,10 +652,7 @@ impl ModelClient { X_CODEX_INSTALLATION_ID_HEADER.to_string(), self.state.installation_id.clone(), ); - client_metadata.insert( - X_CODEX_WINDOW_ID_HEADER.to_string(), - self.current_window_id(), - ); + client_metadata.insert(X_CODEX_WINDOW_ID_HEADER.to_string(), window_id.to_string()); if let Some(subagent) = subagent_header_value(&self.state.session_source) { client_metadata.insert(X_OPENAI_SUBAGENT_HEADER.to_string(), subagent); } @@ -752,6 +734,7 @@ impl ModelClient { } } + #[allow(clippy::too_many_arguments)] fn build_responses_request( &self, provider: &codex_api::Provider, @@ -760,6 +743,7 @@ impl ModelClient { effort: Option, summary: ReasoningSummaryConfig, service_tier: Option, + window_id: &str, ) -> Result { let instructions = &prompt.base_instructions.text; let input = prompt.get_formatted_input(); @@ -807,10 +791,7 @@ impl ModelClient { X_CODEX_INSTALLATION_ID_HEADER.to_string(), self.state.installation_id.clone(), ), - ( - X_CODEX_WINDOW_ID_HEADER.to_string(), - self.current_window_id(), - ), + (X_CODEX_WINDOW_ID_HEADER.to_string(), window_id.to_string()), ])), }; Ok(request) @@ -955,7 +936,7 @@ impl ModelClient { headers.insert("x-client-request-id", header_value); } headers.extend(build_session_headers(Some(session_id), Some(thread_id))); - headers.extend(self.build_responses_identity_headers()); + headers.extend(self.build_responses_identity_headers(/*window_id*/ None)); if let Some(header_value) = self.generate_attestation_header_for().await { headers.insert(X_OAI_ATTESTATION_HEADER, header_value); } @@ -998,6 +979,7 @@ impl ModelClientSession { /// regardless of transport choice. async fn build_responses_options( &self, + window_id: &str, turn_metadata_header: Option<&str>, compression: Compression, use_responses_lite: bool, @@ -1015,7 +997,10 @@ impl ModelClientSession { Some(&self.turn_state), turn_metadata_header.as_ref(), ); - headers.extend(self.client.build_responses_identity_headers()); + headers.extend( + self.client + .build_responses_identity_headers(Some(window_id)), + ); if let Some(header_value) = self.client.generate_attestation_header_for().await { headers.insert(X_OAI_ATTESTATION_HEADER, header_value); } @@ -1115,7 +1100,6 @@ impl ModelClientSession { pub async fn preconnect_websocket( &mut self, session_telemetry: &SessionTelemetry, - _model_info: &ModelInfo, ) -> std::result::Result<(), ApiError> { if !self.client.responses_websocket_enabled() { return Ok(()); @@ -1257,6 +1241,7 @@ impl ModelClientSession { )] async fn stream_responses_api( &self, + window_id: &str, prompt: &Prompt, model_info: &ModelInfo, session_telemetry: &SessionTelemetry, @@ -1288,6 +1273,7 @@ impl ModelClientSession { let compression = self.responses_request_compression(client_setup.auth.as_ref()); let mut options = self .build_responses_options( + window_id, turn_metadata_header, compression, model_info.use_responses_lite, @@ -1301,6 +1287,7 @@ impl ModelClientSession { effort.clone(), summary, service_tier.clone(), + window_id, )?; let inference_trace_attempt = inference_trace.start_attempt(); inference_trace_attempt.add_request_headers(&mut options.extra_headers); @@ -1374,6 +1361,7 @@ impl ModelClientSession { )] async fn stream_responses_websocket( &mut self, + window_id: &str, prompt: &Prompt, model_info: &ModelInfo, session_telemetry: &SessionTelemetry, @@ -1402,6 +1390,7 @@ impl ModelClientSession { let options = self .build_responses_options( + window_id, turn_metadata_header, compression, model_info.use_responses_lite, @@ -1414,10 +1403,12 @@ impl ModelClientSession { effort.clone(), summary, service_tier.clone(), + window_id, )?; let mut ws_payload = ResponseCreateWsRequest { client_metadata: response_create_client_metadata( Some(self.client.build_ws_client_metadata( + window_id, turn_metadata_header, model_info.use_responses_lite, )), @@ -1553,6 +1544,7 @@ impl ModelClientSession { #[allow(clippy::too_many_arguments)] pub async fn prewarm_websocket( &mut self, + window_id: &str, prompt: &Prompt, model_info: &ModelInfo, session_telemetry: &SessionTelemetry, @@ -1571,6 +1563,7 @@ impl ModelClientSession { let disabled_trace = InferenceTraceContext::disabled(); match self .stream_responses_websocket( + window_id, prompt, model_info, session_telemetry, @@ -1614,6 +1607,7 @@ impl ModelClientSession { /// branches. pub async fn stream( &mut self, + window_id: &str, prompt: &Prompt, model_info: &ModelInfo, session_telemetry: &SessionTelemetry, @@ -1630,6 +1624,7 @@ impl ModelClientSession { let request_trace = current_span_w3c_trace_context(); match self .stream_responses_websocket( + window_id, prompt, model_info, session_telemetry, @@ -1651,6 +1646,7 @@ impl ModelClientSession { } self.stream_responses_api( + window_id, prompt, model_info, session_telemetry, diff --git a/codex-rs/core/src/client_tests.rs b/codex-rs/core/src/client_tests.rs index 2ebffe3922..d93bd8bf7a 100644 --- a/codex-rs/core/src/client_tests.rs +++ b/codex-rs/core/src/client_tests.rs @@ -291,13 +291,13 @@ fn build_ws_client_metadata_includes_window_lineage_and_turn_metadata() { Some(parent_thread_id), ); - client.advance_window_generation(); - + let thread_id = client.state.thread_id; + let window_id = format!("{thread_id}:1"); let client_metadata = client.build_ws_client_metadata( + &window_id, Some(r#"{"turn_id":"turn-123"}"#), /*use_responses_lite*/ false, ); - let thread_id = client.state.thread_id; assert_eq!( client_metadata, std::collections::HashMap::from([ diff --git a/codex-rs/core/src/compact.rs b/codex-rs/core/src/compact.rs index e2573d6d6d..96feddf138 100644 --- a/codex-rs/core/src/compact.rs +++ b/codex-rs/core/src/compact.rs @@ -228,7 +228,7 @@ async fn run_compact_task_inner_impl( personality: turn_context.personality, ..Default::default() }; - let window_id = sess.services.model_client.current_window_id(); + let window_id = sess.current_window_id().await; let turn_metadata_header = turn_context .turn_metadata_state .current_header_value_for_compaction(&window_id, compaction_metadata); @@ -236,6 +236,7 @@ async fn run_compact_task_inner_impl( &sess, turn_context.as_ref(), &mut client_session, + &window_id, turn_metadata_header.as_deref(), &prompt, ) @@ -309,6 +310,7 @@ async fn run_compact_task_inner_impl( let compacted_item = CompactedItem { message: summary_text.clone(), replacement_history: Some(new_history.clone()), + window_id: None, }; sess.replace_compacted_history(new_history, reference_context_item, compacted_item) .await; @@ -579,11 +581,13 @@ async fn drain_to_completed( sess: &Session, turn_context: &TurnContext, client_session: &mut ModelClientSession, + window_id: &str, turn_metadata_header: Option<&str>, prompt: &Prompt, ) -> CodexResult<()> { let mut stream = client_session .stream( + window_id, prompt, &turn_context.model_info, &turn_context.session_telemetry, diff --git a/codex-rs/core/src/compact_remote.rs b/codex-rs/core/src/compact_remote.rs index bd852b4e79..767fdf7b01 100644 --- a/codex-rs/core/src/compact_remote.rs +++ b/codex-rs/core/src/compact_remote.rs @@ -224,7 +224,7 @@ async fn run_remote_compact_task_inner_impl( output_schema: None, output_schema_strict: true, }; - let window_id = sess.services.model_client.current_window_id(); + let window_id = sess.current_window_id().await; let turn_metadata_header = turn_context .turn_metadata_state .current_header_value_for_compaction(&window_id, compaction_metadata); @@ -245,6 +245,7 @@ async fn run_remote_compact_task_inner_impl( }, &turn_context.session_telemetry, &compaction_trace, + &window_id, turn_metadata_header.as_deref(), ) .await?; @@ -263,6 +264,7 @@ async fn run_remote_compact_task_inner_impl( let compacted_item = CompactedItem { message: String::new(), replacement_history: Some(new_history.clone()), + window_id: None, }; // Install is the semantic boundary where the compact endpoint's output becomes live // thread history. Keep it distinct from the later inference request so the reducer can diff --git a/codex-rs/core/src/compact_remote_v2.rs b/codex-rs/core/src/compact_remote_v2.rs index a80d4dd56f..9a14dcd963 100644 --- a/codex-rs/core/src/compact_remote_v2.rs +++ b/codex-rs/core/src/compact_remote_v2.rs @@ -240,7 +240,7 @@ async fn run_remote_compact_task_inner_impl( output_schema_strict: true, }; - let window_id = sess.services.model_client.current_window_id(); + let window_id = sess.current_window_id().await; let turn_metadata_header = turn_context .turn_metadata_state .current_header_value_for_compaction(&window_id, compaction_metadata); @@ -264,6 +264,7 @@ async fn run_remote_compact_task_inner_impl( turn_context, client_session, &prompt, + &window_id, turn_metadata_header.as_deref(), ) .await; @@ -299,6 +300,7 @@ async fn run_remote_compact_task_inner_impl( let compacted_item = CompactedItem { message: String::new(), replacement_history: Some(new_history.clone()), + window_id: None, }; compaction_trace.record_installed(&CompactionCheckpointTracePayload { input_history: &trace_input_history, @@ -323,6 +325,7 @@ async fn run_remote_compaction_request_v2( turn_context: &TurnContext, client_session: &mut ModelClientSession, prompt: &Prompt, + window_id: &str, turn_metadata_header: Option<&str>, ) -> CodexResult { let max_retries = turn_context @@ -334,6 +337,7 @@ async fn run_remote_compaction_request_v2( loop { let result = match client_session .stream( + window_id, prompt, &turn_context.model_info, &turn_context.session_telemetry, diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs index 7cc091e43f..ddb31a9a52 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -1299,15 +1299,20 @@ impl Session { turn_context: &TurnContext, rollout_items: &[RolloutItem], ) -> Option { - let reconstructed_rollout = self + let rollout_reconstruction::RolloutReconstruction { + history, + previous_turn_settings, + reference_context_item, + window_id, + } = self .reconstruct_history_from_rollout(turn_context, rollout_items) .await; - let previous_turn_settings = reconstructed_rollout.previous_turn_settings.clone(); - self.replace_history( - reconstructed_rollout.history, - reconstructed_rollout.reference_context_item, - ) - .await; + { + let mut state = self.state.lock().await; + state.replace_history(history, reference_context_item); + state.set_auto_compact_window_id(window_id); + state.set_previous_turn_settings(previous_turn_settings.clone()); + } let prefix_tokens = if matches!( turn_context.config.model_auto_compact_token_limit_scope, AutoCompactTokenLimitScope::BodyAfterPrefix @@ -1322,8 +1327,6 @@ impl Session { self.set_auto_compact_window_estimated_prefill_for_scope(turn_context, prefix_tokens) .await; } - self.set_previous_turn_settings(previous_turn_settings.clone()) - .await; previous_turn_settings } @@ -2655,6 +2658,7 @@ impl Session { .await; } + #[cfg(test)] pub(crate) async fn replace_history( &self, items: Vec, @@ -2668,14 +2672,15 @@ impl Session { &self, items: Vec, reference_context_item: Option, - compacted_item: CompactedItem, + mut compacted_item: CompactedItem, ) { { let mut state = self.state.lock().await; state.replace_history(items, reference_context_item.clone()); - state.start_next_auto_compact_window(); } + compacted_item.window_id = Some(self.advance_auto_compact_window_id().await); + self.persist_rollout_items(&[RolloutItem::Compacted(compacted_item)]) .await; if let Some(turn_context_item) = reference_context_item { @@ -2686,7 +2691,6 @@ impl Session { let mut state = self.state.lock().await; state.queue_pending_session_start_source(codex_hooks::SessionStartSource::Compact); } - self.services.model_client.advance_window_generation(); } async fn persist_rollout_response_items(&self, items: &[ResponseItem]) { @@ -2989,6 +2993,18 @@ impl Session { state.clone_history() } + pub(crate) async fn current_window_id(&self) -> String { + let state = self.state.lock().await; + let thread_id = self.thread_id; + let window_id = state.auto_compact_window_id(); + format!("{thread_id}:{window_id}") + } + + async fn advance_auto_compact_window_id(&self) -> u64 { + let mut state = self.state.lock().await; + state.advance_auto_compact_window_id() + } + pub(crate) async fn reference_context_item(&self) -> Option { let state = self.state.lock().await; state.reference_context_item() diff --git a/codex-rs/core/src/session/rollout_reconstruction.rs b/codex-rs/core/src/session/rollout_reconstruction.rs index a4c042af0c..2d0327bf40 100644 --- a/codex-rs/core/src/session/rollout_reconstruction.rs +++ b/codex-rs/core/src/session/rollout_reconstruction.rs @@ -8,6 +8,7 @@ pub(super) struct RolloutReconstruction { pub(super) history: Vec, pub(super) previous_turn_settings: Option, pub(super) reference_context_item: Option, + pub(super) window_id: u64, } #[derive(Debug, Default)] @@ -33,6 +34,7 @@ struct ActiveReplaySegment<'a> { previous_turn_settings: Option, reference_context_item: TurnReferenceContextItem, base_replacement_history: Option<&'a [ResponseItem]>, + window_id: Option, } fn turn_ids_are_compatible(active_turn_id: Option<&str>, item_turn_id: Option<&str>) -> bool { @@ -45,6 +47,7 @@ fn finalize_active_segment<'a>( base_replacement_history: &mut Option<&'a [ResponseItem]>, previous_turn_settings: &mut Option, reference_context_item: &mut TurnReferenceContextItem, + window_id: &mut Option, pending_rollback_turns: &mut usize, ) { // Thread rollback drops the newest surviving real user-message boundaries. In replay, that @@ -65,6 +68,10 @@ fn finalize_active_segment<'a>( *base_replacement_history = Some(segment_base_replacement_history); } + if window_id.is_none() { + *window_id = active_segment.window_id; + } + // `previous_turn_settings` come from the newest surviving user turn that established them. if previous_turn_settings.is_none() && active_segment.counts_as_user_turn { *previous_turn_settings = active_segment.previous_turn_settings; @@ -97,6 +104,7 @@ impl Session { let mut base_replacement_history: Option<&[ResponseItem]> = None; let mut previous_turn_settings = None; let mut reference_context_item = TurnReferenceContextItem::NeverSet; + let mut window_id = None; // Rollback is "drop the newest N user turns". While scanning in reverse, that becomes // "skip the next N user-turn segments we finalize". let mut pending_rollback_turns = 0usize; @@ -112,6 +120,9 @@ impl Session { RolloutItem::Compacted(compacted) => { let active_segment = active_segment.get_or_insert_with(ActiveReplaySegment::default); + if active_segment.window_id.is_none() { + active_segment.window_id = compacted.window_id; + } // Looking backward, compaction clears any older baseline unless a newer // `TurnContextItem` in this same segment has already re-established it. if matches!( @@ -198,6 +209,7 @@ impl Session { &mut base_replacement_history, &mut previous_turn_settings, &mut reference_context_item, + &mut window_id, &mut pending_rollback_turns, ); } @@ -227,10 +239,19 @@ impl Session { &mut base_replacement_history, &mut previous_turn_settings, &mut reference_context_item, + &mut window_id, &mut pending_rollback_turns, ); } + let fallback_window_id = u64::try_from( + rollout_items + .iter() + .filter(|item| matches!(item, RolloutItem::Compacted(_))) + .count(), + ) + .unwrap_or(u64::MAX); + let mut history = ContextManager::new(); let mut saw_legacy_compaction_without_replacement_history = false; if let Some(base_replacement_history) = base_replacement_history { @@ -296,6 +317,7 @@ impl Session { history: history.raw_items().to_vec(), previous_turn_settings, reference_context_item, + window_id: window_id.unwrap_or(fallback_window_id), } } } diff --git a/codex-rs/core/src/session/rollout_reconstruction_tests.rs b/codex-rs/core/src/session/rollout_reconstruction_tests.rs index 9ee70dada3..c84ee7f5f3 100644 --- a/codex-rs/core/src/session/rollout_reconstruction_tests.rs +++ b/codex-rs/core/src/session/rollout_reconstruction_tests.rs @@ -791,6 +791,7 @@ async fn record_initial_history_resumed_rollback_drops_incomplete_user_turn_comp RolloutItem::Compacted(CompactedItem { message: String::new(), replacement_history: Some(Vec::new()), + window_id: None, }), RolloutItem::EventMsg(EventMsg::ThreadRolledBack( codex_protocol::protocol::ThreadRolledBackEvent { num_turns: 1 }, @@ -846,6 +847,7 @@ async fn record_initial_history_resumed_does_not_seed_reference_context_item_aft RolloutItem::Compacted(CompactedItem { message: String::new(), replacement_history: Some(Vec::new()), + window_id: None, }), ]; @@ -871,6 +873,7 @@ async fn reconstruct_history_legacy_compaction_without_replacement_history_does_ RolloutItem::Compacted(CompactedItem { message: "legacy summary".to_string(), replacement_history: None, + window_id: None, }), ]; @@ -902,6 +905,7 @@ async fn reconstruct_history_legacy_compaction_without_replacement_history_clear RolloutItem::Compacted(CompactedItem { message: "legacy summary".to_string(), replacement_history: None, + window_id: None, }), RolloutItem::EventMsg(EventMsg::TurnStarted( codex_protocol::protocol::TurnStartedEvent { @@ -994,6 +998,7 @@ async fn record_initial_history_resumed_turn_context_after_compaction_reestablis RolloutItem::Compacted(CompactedItem { message: String::new(), replacement_history: Some(Vec::new()), + window_id: None, }), RolloutItem::TurnContext(previous_context_item), RolloutItem::EventMsg(EventMsg::TurnComplete( @@ -1140,6 +1145,7 @@ async fn record_initial_history_resumed_aborted_turn_without_id_clears_active_tu RolloutItem::Compacted(CompactedItem { message: String::new(), replacement_history: Some(Vec::new()), + window_id: None, }), ]; @@ -1369,6 +1375,7 @@ async fn record_initial_history_resumed_trailing_incomplete_turn_compaction_clea RolloutItem::Compacted(CompactedItem { message: String::new(), replacement_history: Some(Vec::new()), + window_id: None, }), ]; @@ -1529,6 +1536,7 @@ async fn record_initial_history_resumed_replaced_incomplete_compacted_turn_clear RolloutItem::Compacted(CompactedItem { message: String::new(), replacement_history: Some(Vec::new()), + window_id: None, }), // A newer TurnStarted replaces the incomplete compacted turn without a matching // completion/abort for the old one. diff --git a/codex-rs/core/src/session/session.rs b/codex-rs/core/src/session/session.rs index 655454a320..e42cf202e8 100644 --- a/codex-rs/core/src/session/session.rs +++ b/codex-rs/core/src/session/session.rs @@ -515,17 +515,6 @@ impl Session { } InitialHistory::Resumed(resumed_history) => resumed_history.conversation_id, }; - let window_generation = match &initial_history { - InitialHistory::Resumed(resumed_history) => u64::try_from( - resumed_history - .history - .iter() - .filter(|item| matches!(item, RolloutItem::Compacted(_))) - .count(), - ) - .unwrap_or(u64::MAX), - InitialHistory::New | InitialHistory::Cleared | InitialHistory::Forked(_) => 0, - }; // Kick off independent async setup tasks in parallel to reduce startup latency. // // - initialize thread persistence with new or resumed session info @@ -1047,9 +1036,6 @@ impl Session { code_mode_service: crate::tools::code_mode::CodeModeService::new(), environment_manager, }; - services - .model_client - .set_window_generation(window_generation); let (out_of_band_elicitation_paused, _out_of_band_elicitation_paused_rx) = watch::channel(false); diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs index 8d8208205e..29de7ca662 100644 --- a/codex-rs/core/src/session/tests.rs +++ b/codex-rs/core/src/session/tests.rs @@ -1585,6 +1585,7 @@ async fn reconstruct_history_matches_live_compactions() { .await; assert_eq!(expected, reconstructed.history); + assert_eq!(2, reconstructed.window_id); } #[tokio::test] @@ -1612,6 +1613,7 @@ async fn reconstruct_history_uses_replacement_history_verbatim() { let rollout_items = vec![RolloutItem::Compacted(CompactedItem { message: String::new(), replacement_history: Some(replacement_history.clone()), + window_id: Some(42), })]; let reconstructed = session @@ -1619,6 +1621,7 @@ async fn reconstruct_history_uses_replacement_history_verbatim() { .await; assert_eq!(reconstructed.history, replacement_history); + assert_eq!(42, reconstructed.window_id); } #[tokio::test] @@ -2919,6 +2922,7 @@ async fn thread_rollback_restores_cleared_reference_context_item_after_compactio RolloutItem::Compacted(CompactedItem { message: "summary after compaction".to_string(), replacement_history: Some(compacted_history.clone()), + window_id: Some(7), }), RolloutItem::EventMsg(EventMsg::TurnComplete(TurnCompleteEvent { turn_id: compact_turn_id, @@ -2965,6 +2969,10 @@ async fn thread_rollback_restores_cleared_reference_context_item_after_compactio Some(first_context_item), ) .await; + { + let mut state = sess.state.lock().await; + state.set_auto_compact_window_id(/*window_id*/ 99); + } handlers::thread_rollback(&sess, "sub-1".to_string(), /*num_turns*/ 1).await; let rollback_event = wait_for_thread_rolled_back(&rx).await; @@ -2972,6 +2980,7 @@ async fn thread_rollback_restores_cleared_reference_context_item_after_compactio assert_eq!(sess.clone_history().await.raw_items(), compacted_history); assert!(sess.reference_context_item().await.is_none()); + assert!(sess.current_window_id().await.ends_with(":7")); } #[tokio::test] @@ -9581,6 +9590,7 @@ async fn sample_rollout( rollout_items.push(RolloutItem::Compacted(CompactedItem { message: summary1.to_string(), replacement_history: None, + window_id: None, })); let user2 = ResponseItem::Message { @@ -9621,6 +9631,7 @@ async fn sample_rollout( rollout_items.push(RolloutItem::Compacted(CompactedItem { message: summary2.to_string(), replacement_history: None, + window_id: None, })); let user3 = ResponseItem::Message { diff --git a/codex-rs/core/src/session/turn.rs b/codex-rs/core/src/session/turn.rs index 722957d30c..f65ada6112 100644 --- a/codex-rs/core/src/session/turn.rs +++ b/codex-rs/core/src/session/turn.rs @@ -221,7 +221,7 @@ pub(crate) async fn run_turn( .instrument(trace_span!("run_turn.prepare_sampling_request_input")) .await; - let window_id = sess.services.model_client.current_window_id(); + let window_id = sess.current_window_id().await; let turn_metadata_header = turn_context .turn_metadata_state .current_header_value_for_model_request(&window_id); @@ -231,6 +231,7 @@ pub(crate) async fn run_turn( Arc::clone(&turn_extension_data), Arc::clone(&turn_diff_tracker), &mut client_session, + &window_id, turn_metadata_header.as_deref(), sampling_request_input.clone(), cancellation_token.child_token(), @@ -264,7 +265,6 @@ pub(crate) async fn run_turn( estimated_token_count = ?estimated_token_count, auto_compact_scope_limit = token_status.auto_compact_scope_limit, auto_compact_limit_scope = ?turn_context.config.model_auto_compact_token_limit_scope, - auto_compact_window_ordinal = ?token_status.auto_compact_window_ordinal, auto_compact_window_prefill_tokens = ?token_status.auto_compact_window_prefill_tokens, full_context_window_limit = ?token_status.full_context_window_limit, full_context_window_limit_reached = token_status.full_context_window_limit_reached, @@ -702,7 +702,6 @@ struct AutoCompactTokenStatus { auto_compact_scope_tokens: i64, auto_compact_scope_limit: i64, full_context_window_limit: Option, - auto_compact_window_ordinal: Option, auto_compact_window_prefill_tokens: Option, full_context_window_limit_reached: bool, token_limit_reached: bool, @@ -713,7 +712,6 @@ async fn auto_compact_token_status( turn_context: &TurnContext, ) -> AutoCompactTokenStatus { let active_context_tokens = sess.get_total_token_usage().await; - let mut auto_compact_window_ordinal = None; let mut auto_compact_window_prefill_tokens = None; let (auto_compact_scope_tokens, auto_compact_scope_limit, full_context_window_limit) = match turn_context.config.model_auto_compact_token_limit_scope { @@ -727,7 +725,6 @@ async fn auto_compact_token_status( ), AutoCompactTokenLimitScope::BodyAfterPrefix => { let window = sess.auto_compact_window_snapshot().await; - auto_compact_window_ordinal = Some(window.ordinal); auto_compact_window_prefill_tokens = window.prefill_input_tokens; let baseline = window.prefill_input_tokens.unwrap_or(active_context_tokens); ( @@ -753,7 +750,6 @@ async fn auto_compact_token_status( auto_compact_scope_tokens, auto_compact_scope_limit, full_context_window_limit, - auto_compact_window_ordinal, auto_compact_window_prefill_tokens, full_context_window_limit_reached, token_limit_reached, @@ -988,6 +984,7 @@ async fn run_sampling_request( turn_store: Arc, turn_diff_tracker: SharedTurnDiffTracker, client_session: &mut ModelClientSession, + window_id: &str, turn_metadata_header: Option<&str>, input: Vec, cancellation_token: CancellationToken, @@ -1031,6 +1028,7 @@ async fn run_sampling_request( Arc::clone(&turn_context), Arc::clone(&turn_store), client_session, + window_id, turn_metadata_header, Arc::clone(&turn_diff_tracker), &prompt, @@ -1760,6 +1758,7 @@ async fn try_run_sampling_request( turn_context: Arc, turn_store: Arc, client_session: &mut ModelClientSession, + window_id: &str, turn_metadata_header: Option<&str>, turn_diff_tracker: SharedTurnDiffTracker, prompt: &Prompt, @@ -1781,6 +1780,7 @@ async fn try_run_sampling_request( let sampling_timing_guard = turn_context.turn_timing_state.begin_sampling(); let mut stream = client_session .stream( + window_id, prompt, &turn_context.model_info, &turn_context.session_telemetry, diff --git a/codex-rs/core/src/session_startup_prewarm.rs b/codex-rs/core/src/session_startup_prewarm.rs index f4948ae2f2..fda4982545 100644 --- a/codex-rs/core/src/session_startup_prewarm.rs +++ b/codex-rs/core/src/session_startup_prewarm.rs @@ -266,7 +266,7 @@ async fn schedule_startup_prewarm_inner( build_prompt_started_at.elapsed(), /*status*/ None, ); - let window_id = session.services.model_client.current_window_id(); + let window_id = session.current_window_id().await; let startup_turn_metadata_header = startup_turn_context .turn_metadata_state .current_header_value_for_prewarm(&window_id); @@ -274,6 +274,7 @@ async fn schedule_startup_prewarm_inner( let websocket_warmup_started_at = Instant::now(); client_session .prewarm_websocket( + &window_id, &startup_prompt, &startup_turn_context.model_info, &startup_turn_context.session_telemetry, diff --git a/codex-rs/core/src/state/auto_compact_window.rs b/codex-rs/core/src/state/auto_compact_window.rs index d1deb3c288..f6ebd325f3 100644 --- a/codex-rs/core/src/state/auto_compact_window.rs +++ b/codex-rs/core/src/state/auto_compact_window.rs @@ -2,7 +2,6 @@ use codex_protocol::protocol::TokenUsage; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) struct AutoCompactWindowSnapshot { - pub(crate) ordinal: u64, pub(crate) prefill_input_tokens: Option, } @@ -14,7 +13,7 @@ enum AutoCompactWindowPrefill { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(super) struct AutoCompactWindow { - ordinal: u64, + window_id: u64, /// Absolute input-token baseline for the current compaction window. /// /// `body_after_prefix` subtracts this from later active-context usage. It is @@ -26,7 +25,7 @@ pub(super) struct AutoCompactWindow { impl AutoCompactWindow { pub(super) fn new() -> Self { Self { - ordinal: 1, + window_id: 0, prefill_input_tokens: None, } } @@ -35,9 +34,17 @@ impl AutoCompactWindow { self.prefill_input_tokens = None; } - pub(super) fn start_next(&mut self) { - self.ordinal = self.ordinal.saturating_add(1); - self.clear_prefill(); + pub(super) fn window_id(&self) -> u64 { + self.window_id + } + + pub(super) fn set_window_id(&mut self, window_id: u64) { + self.window_id = window_id; + } + + pub(super) fn advance_window_id(&mut self) -> u64 { + self.window_id = self.window_id.saturating_add(1); + self.window_id } /// Records the request-input side of the first server usage sample. The @@ -74,7 +81,6 @@ impl AutoCompactWindow { None => None, }; AutoCompactWindowSnapshot { - ordinal: self.ordinal, prefill_input_tokens, } } @@ -89,10 +95,15 @@ mod tests { fn tracks_prefill_and_window_boundaries() { let mut window = AutoCompactWindow::new(); + assert_eq!(window.window_id(), 0); + window.set_window_id(/*window_id*/ 3); + assert_eq!(window.window_id(), 3); + assert_eq!(window.advance_window_id(), 4); + assert_eq!(window.window_id(), 4); + assert_eq!( window.snapshot(), AutoCompactWindowSnapshot { - ordinal: 1, prefill_input_tokens: None, } ); @@ -101,7 +112,6 @@ mod tests { assert_eq!( window.snapshot(), AutoCompactWindowSnapshot { - ordinal: 1, prefill_input_tokens: Some(150), } ); @@ -114,7 +124,6 @@ mod tests { assert_eq!( window.snapshot(), AutoCompactWindowSnapshot { - ordinal: 1, prefill_input_tokens: Some(120), } ); @@ -128,18 +137,8 @@ mod tests { assert_eq!( window.snapshot(), AutoCompactWindowSnapshot { - ordinal: 1, prefill_input_tokens: Some(120), } ); - - window.start_next(); - assert_eq!( - window.snapshot(), - AutoCompactWindowSnapshot { - ordinal: 2, - prefill_input_tokens: None, - } - ); } } diff --git a/codex-rs/core/src/state/session.rs b/codex-rs/core/src/state/session.rs index 024f1904cc..ced1428a31 100644 --- a/codex-rs/core/src/state/session.rs +++ b/codex-rs/core/src/state/session.rs @@ -140,14 +140,22 @@ impl SessionState { self.auto_compact_window.set_estimated_prefill(tokens); } - pub(crate) fn start_next_auto_compact_window(&mut self) { - self.auto_compact_window.start_next(); - } - pub(crate) fn auto_compact_window_snapshot(&self) -> AutoCompactWindowSnapshot { self.auto_compact_window.snapshot() } + pub(crate) fn auto_compact_window_id(&self) -> u64 { + self.auto_compact_window.window_id() + } + + pub(crate) fn set_auto_compact_window_id(&mut self, window_id: u64) { + self.auto_compact_window.set_window_id(window_id); + } + + pub(crate) fn advance_auto_compact_window_id(&mut self) -> u64 { + self.auto_compact_window.advance_window_id() + } + pub(crate) fn token_info(&self) -> Option { self.history.token_info() } diff --git a/codex-rs/core/src/state/session_tests.rs b/codex-rs/core/src/state/session_tests.rs index 5f0ae085e7..0fbb92b958 100644 --- a/codex-rs/core/src/state/session_tests.rs +++ b/codex-rs/core/src/state/session_tests.rs @@ -65,18 +65,16 @@ async fn set_rate_limits_defaults_limit_id_to_codex_when_missing() { } #[tokio::test] -async fn replace_history_clears_auto_compact_window_prefill_without_advancing() { +async fn replace_history_clears_auto_compact_window_prefill() { let session_configuration = make_session_configuration_for_tests().await; let mut state = SessionState::new(session_configuration); - state.start_next_auto_compact_window(); state.set_auto_compact_window_estimated_prefill(/*tokens*/ 100); state.replace_history(Vec::new(), /*reference_context_item*/ None); assert_eq!( state.auto_compact_window_snapshot(), AutoCompactWindowSnapshot { - ordinal: 2, prefill_input_tokens: None, } ); diff --git a/codex-rs/core/tests/responses_headers.rs b/codex-rs/core/tests/responses_headers.rs index 1ba93aec10..af87762c04 100644 --- a/codex-rs/core/tests/responses_headers.rs +++ b/codex-rs/core/tests/responses_headers.rs @@ -85,6 +85,7 @@ async fn responses_stream_includes_subagent_header_on_review() { let session_source = SessionSource::SubAgent(SubAgentSource::Review); let model_info = codex_core::test_support::construct_model_info_offline(model.as_str(), &config); + let expected_window_id = format!("{thread_id}:0"); let session_telemetry = SessionTelemetry::new( thread_id, model.as_str(), @@ -126,6 +127,7 @@ async fn responses_stream_includes_subagent_header_on_review() { let mut stream = client_session .stream( + &expected_window_id, &prompt, &model_info, &session_telemetry, @@ -144,7 +146,6 @@ async fn responses_stream_includes_subagent_header_on_review() { } let request = request_recorder.single_request(); - let expected_window_id = format!("{thread_id}:0"); assert_eq!( request.header("x-openai-subagent").as_deref(), Some("review") @@ -217,6 +218,7 @@ async fn responses_stream_includes_subagent_header_on_other() { let session_source = SessionSource::SubAgent(SubAgentSource::Other("my-task".to_string())); let model_info = codex_core::test_support::construct_model_info_offline(model.as_str(), &config); + let window_id = format!("{thread_id}:0"); let session_telemetry = SessionTelemetry::new( thread_id, @@ -259,6 +261,7 @@ async fn responses_stream_includes_subagent_header_on_other() { let mut stream = client_session .stream( + &window_id, &prompt, &model_info, &session_telemetry, @@ -336,6 +339,7 @@ async fn responses_respects_model_info_overrides_from_config() { SessionSource::SubAgent(SubAgentSource::Other("override-check".to_string())); let model_info = codex_core::test_support::construct_model_info_offline(model.as_str(), &config); + let window_id = format!("{thread_id}:0"); let session_telemetry = SessionTelemetry::new( thread_id, model.as_str(), @@ -377,6 +381,7 @@ async fn responses_respects_model_info_overrides_from_config() { let mut stream = client_session .stream( + &window_id, &prompt, &model_info, &session_telemetry, diff --git a/codex-rs/core/tests/suite/client.rs b/codex-rs/core/tests/suite/client.rs index 9a6f104d0c..a522e041c2 100644 --- a/codex-rs/core/tests/suite/client.rs +++ b/codex-rs/core/tests/suite/client.rs @@ -90,6 +90,7 @@ use wiremock::matchers::path; use wiremock::matchers::query_param; const INSTALLATION_ID_FILENAME: &str = "installation_id"; +const TEST_WINDOW_ID: &str = "test-thread:0"; #[expect(clippy::unwrap_used)] fn assert_message_role(request_body: &serde_json::Value, role: &str) { @@ -924,6 +925,7 @@ async fn send_provider_auth_request(server: &MockServer, auth: ModelProviderAuth let mut stream = client_session .stream( + TEST_WINDOW_ID, &prompt, &model_info, &session_telemetry, @@ -2467,6 +2469,7 @@ async fn azure_responses_request_includes_store_and_reasoning_ids() { let mut stream = client_session .stream( + TEST_WINDOW_ID, &prompt, &model_info, &session_telemetry, diff --git a/codex-rs/core/tests/suite/client_websockets.rs b/codex-rs/core/tests/suite/client_websockets.rs index 4dc674ec30..8b135b6b75 100755 --- a/codex-rs/core/tests/suite/client_websockets.rs +++ b/codex-rs/core/tests/suite/client_websockets.rs @@ -65,6 +65,7 @@ const X_CLIENT_REQUEST_ID_HEADER: &str = "x-client-request-id"; const WS_REQUEST_HEADER_RESPONSES_LITE_CLIENT_METADATA_KEY: &str = "ws_request_header_x_openai_internal_codex_responses_lite"; const TEST_INSTALLATION_ID: &str = "11111111-1111-4111-8111-111111111111"; +const TEST_WINDOW_ID: &str = "test-thread:0"; const X_CODEX_WS_STREAM_REQUEST_START_MS_CLIENT_METADATA_KEY: &str = "x-codex-ws-stream-request-start-ms"; @@ -274,7 +275,7 @@ async fn responses_websocket_preconnect_does_not_replace_turn_trace_payload() { let harness = websocket_harness(&server).await; let mut client_session = harness.client.new_session(); client_session - .preconnect_websocket(&harness.session_telemetry, &harness.model_info) + .preconnect_websocket(&harness.session_telemetry) .await .expect("websocket preconnect failed"); let prompt = prompt_with_input(vec![message_item("hello")]); @@ -310,7 +311,7 @@ async fn responses_websocket_preconnect_reuses_connection() { let harness = websocket_harness(&server).await; let mut client_session = harness.client.new_session(); client_session - .preconnect_websocket(&harness.session_telemetry, &harness.model_info) + .preconnect_websocket(&harness.session_telemetry) .await .expect("websocket preconnect failed"); let prompt = prompt_with_input(vec![message_item("hello")]); @@ -321,6 +322,7 @@ async fn responses_websocket_preconnect_reuses_connection() { server.single_handshake().header(USER_AGENT_HEADER), Some(codex_login::default_client::get_codex_user_agent()) ); + assert_eq!(server.single_handshake().header("x-codex-window-id"), None); let connection = server.single_connection(); assert_eq!(connection.len(), 1); @@ -342,6 +344,7 @@ async fn responses_websocket_request_prewarm_reuses_connection() { let prompt = prompt_with_input(vec![message_item("hello")]); client_session .prewarm_websocket( + TEST_WINDOW_ID, &prompt, &harness.model_info, &harness.session_telemetry, @@ -396,6 +399,7 @@ async fn responses_websocket_request_prewarm_traces_logical_request() { client_session .prewarm_websocket( + TEST_WINDOW_ID, &prompt, &harness.model_info, &harness.session_telemetry, @@ -441,6 +445,7 @@ async fn responses_websocket_request_prewarm_traces_logical_request() { let mut stream = client_session .stream( + TEST_WINDOW_ID, &prompt, &harness.model_info, &harness.session_telemetry, @@ -606,12 +611,13 @@ async fn responses_websocket_preconnect_is_reused_even_with_header_changes() { let harness = websocket_harness(&server).await; let mut client_session = harness.client.new_session(); client_session - .preconnect_websocket(&harness.session_telemetry, &harness.model_info) + .preconnect_websocket(&harness.session_telemetry) .await .expect("websocket preconnect failed"); let prompt = prompt_with_input(vec![message_item("hello")]); let mut stream = client_session .stream( + TEST_WINDOW_ID, &prompt, &harness.model_info, &harness.session_telemetry, @@ -651,6 +657,7 @@ async fn responses_websocket_request_prewarm_is_reused_even_with_header_changes( let prompt = prompt_with_input(vec![message_item("hello")]); client_session .prewarm_websocket( + TEST_WINDOW_ID, &prompt, &harness.model_info, &harness.session_telemetry, @@ -663,6 +670,7 @@ async fn responses_websocket_request_prewarm_is_reused_even_with_header_changes( .expect("websocket prewarm failed"); let mut stream = client_session .stream( + TEST_WINDOW_ID, &prompt, &harness.model_info, &harness.session_telemetry, @@ -717,6 +725,7 @@ async fn responses_websocket_prewarm_uses_v2_when_provider_supports_websockets() let prompt = prompt_with_input(vec![message_item("hello")]); client_session .prewarm_websocket( + TEST_WINDOW_ID, &prompt, &harness.model_info, &harness.session_telemetry, @@ -772,7 +781,7 @@ async fn responses_websocket_preconnect_runs_when_only_v2_feature_enabled() { let harness = websocket_harness_with_options(&server, /*runtime_metrics_enabled*/ true).await; let mut client_session = harness.client.new_session(); client_session - .preconnect_websocket(&harness.session_telemetry, &harness.model_info) + .preconnect_websocket(&harness.session_telemetry) .await .expect("websocket preconnect failed"); @@ -1066,6 +1075,7 @@ async fn responses_websocket_emits_reasoning_included_event() { let mut stream = client_session .stream( + TEST_WINDOW_ID, &prompt, &harness.model_info, &harness.session_telemetry, @@ -1140,6 +1150,7 @@ async fn responses_websocket_emits_rate_limit_events() { let mut stream = client_session .stream( + TEST_WINDOW_ID, &prompt, &harness.model_info, &harness.session_telemetry, @@ -1794,6 +1805,7 @@ async fn responses_websocket_v2_after_error_uses_full_create_without_previous_re let mut second_stream = session .stream( + TEST_WINDOW_ID, &prompt_two, &harness.model_info, &harness.session_telemetry, @@ -1882,6 +1894,7 @@ async fn responses_websocket_v2_surfaces_terminal_error_without_close_handshake( let mut second_stream = session .stream( + TEST_WINDOW_ID, &prompt_two, &harness.model_info, &harness.session_telemetry, @@ -2116,6 +2129,7 @@ async fn stream_until_complete_with_model_info( ) { let mut stream = client_session .stream( + TEST_WINDOW_ID, prompt, model_info, &harness.session_telemetry, @@ -2183,6 +2197,7 @@ async fn stream_until_complete_with_request_metadata( ) { let mut stream = client_session .stream( + TEST_WINDOW_ID, prompt, &harness.model_info, &harness.session_telemetry, diff --git a/codex-rs/memories/write/src/runtime.rs b/codex-rs/memories/write/src/runtime.rs index d2b1cc36b2..0b6718f01e 100644 --- a/codex-rs/memories/write/src/runtime.rs +++ b/codex-rs/memories/write/src/runtime.rs @@ -243,8 +243,10 @@ impl MemoryStartupContext { ); let mut client_session = model_client.new_session(); + let window_id = format!("{}:0", self.thread_id); let mut stream = client_session .stream( + &window_id, prompt, &context.model_info, &context.session_telemetry, diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index e93801d29f..01e664c734 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -2892,6 +2892,8 @@ pub struct CompactedItem { pub message: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub replacement_history: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub window_id: Option, } impl From for ResponseItem { diff --git a/codex-rs/rollout/src/metadata_tests.rs b/codex-rs/rollout/src/metadata_tests.rs index b53c958ae8..03d251e1b3 100644 --- a/codex-rs/rollout/src/metadata_tests.rs +++ b/codex-rs/rollout/src/metadata_tests.rs @@ -153,6 +153,7 @@ fn builder_from_items_falls_back_to_filename() { let items = vec![RolloutItem::Compacted(CompactedItem { message: "noop".to_string(), replacement_history: None, + window_id: None, })]; let builder = builder_from_items(items.as_slice(), path.as_path()).expect("builder"); diff --git a/codex-rs/thread-store/src/thread_metadata_sync.rs b/codex-rs/thread-store/src/thread_metadata_sync.rs index 3db827e3a2..3a379a4815 100644 --- a/codex-rs/thread-store/src/thread_metadata_sync.rs +++ b/codex-rs/thread-store/src/thread_metadata_sync.rs @@ -474,6 +474,7 @@ mod tests { let item = RolloutItem::Compacted(CompactedItem { message: "compacted".to_string(), replacement_history: None, + window_id: None, }); let first = sync From 0428e20a0b8a1153453832c616dc96133aa4b127 Mon Sep 17 00:00:00 2001 From: "Adam Perry @ OpenAI" Date: Wed, 10 Jun 2026 08:51:17 -0700 Subject: [PATCH 33/47] [codex] add io PathUri native conversion APIs (#27280) ## Why Discovered some rough edges in the API while making use of it more widely within exec-server. It would be a lot more convenient for existing users of `AbsolutePathBuf` if `PathUri` conversion methods returned `std::io::Result`s. ## What * `PathUri::to_native_path()` -> `PathUri::to_abs_path()` * `PathUri::from_file_path()` -> `PathUri::from_abs_path()` --- codex-rs/utils/path-uri/src/lib.rs | 50 ++++++++++++++++++---------- codex-rs/utils/path-uri/src/tests.rs | 41 ++++++++++++++++++----- 2 files changed, 65 insertions(+), 26 deletions(-) diff --git a/codex-rs/utils/path-uri/src/lib.rs b/codex-rs/utils/path-uri/src/lib.rs index a788fdc9b4..444af1ea28 100644 --- a/codex-rs/utils/path-uri/src/lib.rs +++ b/codex-rs/utils/path-uri/src/lib.rs @@ -9,6 +9,8 @@ use serde::Deserializer; use serde::Serialize; use serde::Serializer; use std::fmt; +use std::io; +use std::path::Path; use std::str::FromStr; use thiserror::Error; use ts_rs::TS; @@ -25,7 +27,7 @@ pub const FILE_SCHEME: &str = "file"; /// /// `file:` paths retain their URI spelling so they can be parsed independently /// of the current host. In particular, `/C:/src` remains ambiguous between a -/// Windows drive path and a valid POSIX path until [`Self::to_native_path`] +/// Windows drive path and a valid POSIX path until [`Self::to_abs_path`] /// applies the current host's rules. A local POSIX `file:` URI can also retain /// percent-encoded non-UTF-8 bytes for lossless native round trips. /// @@ -55,15 +57,27 @@ impl PathUri { /// Converts an absolute path on the current host to a `file:` URI. /// - /// On Unix, [`AbsolutePathBuf`]'s absolute-path invariant is sufficient for - /// `url` to represent the path. On Windows, conversion can still fail for - /// absolute paths whose prefix has no `file:` URI representation, including - /// `\\.\` device paths and generic `\\?\` verbatim namespaces. Those cases - /// return [`PathUriParseError::InvalidFileUriPath`]. - pub fn from_file_path(path: &AbsolutePathBuf) -> Result { - let url = Url::from_file_path(path.as_path()) - .map_err(|()| PathUriParseError::InvalidFileUriPath)?; - Self::try_from(url) + /// On Windows, paths without a URI representation, including `\\.\` device + /// paths and generic `\\?\` verbatim namespaces, are reported as invalid + /// input. + pub fn from_abs_path(path: &AbsolutePathBuf) -> io::Result { + let url = Url::from_file_path(path.as_path()).map_err(|()| { + io::Error::new( + io::ErrorKind::InvalidInput, + PathUriParseError::InvalidFileUriPath, + ) + })?; + Self::try_from(url).map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err)) + } + + /// Converts a path on the current host to a `file:` URI. + /// + /// Relative paths and paths without a URI representation are reported as + /// invalid input. + pub fn from_path(path: impl AsRef) -> io::Result { + let path = AbsolutePathBuf::from_absolute_path_checked(path) + .map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err))?; + Self::from_abs_path(&path) } /// Returns the percent-encoded URI path. @@ -149,13 +163,15 @@ impl PathUri { /// on POSIX or a POSIX root on Windows. Because a `file:` URI does not record /// its source operating system, callers should only use this method when the /// URI is known to identify a path on the current host. - pub fn to_native_path(&self) -> Result { - let path = self - .0 - .to_file_path() - .map_err(|()| PathUriParseError::InvalidFileUriPath)?; + pub fn to_abs_path(&self) -> io::Result { + let path = self.0.to_file_path().map_err(|()| { + io::Error::new( + io::ErrorKind::InvalidInput, + PathUriParseError::InvalidFileUriPath, + ) + })?; AbsolutePathBuf::from_absolute_path_checked(path) - .map_err(|_| PathUriParseError::InvalidFileUriPath) + .map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err)) } /// Returns a clone of the canonical URL. @@ -216,7 +232,7 @@ impl<'de> Deserialize<'de> for PathUri { .map_or_else(|| path_error.to_string(), |error| error.to_string()), ) })?; - Self::from_file_path(&path).map_err(serde::de::Error::custom) + Self::from_abs_path(&path).map_err(serde::de::Error::custom) } } diff --git a/codex-rs/utils/path-uri/src/tests.rs b/codex-rs/utils/path-uri/src/tests.rs index 6a237a8c1e..4fb2ba5b45 100644 --- a/codex-rs/utils/path-uri/src/tests.rs +++ b/codex-rs/utils/path-uri/src/tests.rs @@ -12,7 +12,7 @@ fn file_uri_round_trips_an_absolute_path() { .expect("current directory") .join("a path/file.rs"); - let uri = PathUri::from_file_path(&path).expect("path should convert to a file URI"); + let uri = PathUri::from_abs_path(&path).expect("path should convert to a file URI"); let uri_string = uri.to_string(); assert!(uri_string.starts_with("file:")); @@ -22,12 +22,26 @@ fn file_uri_round_trips_an_absolute_path() { uri ); assert_eq!( - uri.to_native_path() + uri.to_abs_path() .expect("local file URI should convert to a native path"), path ); } +#[test] +fn non_native_uri_io_conversion_is_invalid_input() { + #[cfg(unix)] + let uri = PathUri::parse("file://server/share/file.txt").expect("valid file URI"); + #[cfg(windows)] + let uri = PathUri::parse("file:///usr/local/file.txt").expect("valid file URI"); + + let error = uri + .to_abs_path() + .expect_err("URI should not be host-native"); + + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); +} + #[test] fn file_uri_parses_a_windows_path_on_any_host() { let uri = PathUri::parse("file:///C:/Users/Alice%20Smith/src/main.rs") @@ -52,8 +66,10 @@ fn file_uri_rejects_windows_prefixes_without_a_uri_representation() { .expect("Windows namespace path should be absolute"); assert_eq!( - PathUri::from_file_path(&path), - Err(PathUriParseError::InvalidFileUriPath), + PathUri::from_abs_path(&path) + .expect_err("Windows namespace path should not convert") + .kind(), + io::ErrorKind::InvalidInput, "converting {native_path}" ); } @@ -85,9 +101,9 @@ fn file_uri_accepts_non_utf8_posix_paths() { let path = PathBuf::from(std::ffi::OsString::from_vec(b"/tmp/non-utf8-\xff".to_vec())); let path = AbsolutePathBuf::from_absolute_path_checked(path).expect("absolute POSIX path"); - let uri = PathUri::from_file_path(&path).expect("non-UTF-8 path should convert to a file URI"); + let uri = PathUri::from_abs_path(&path).expect("non-UTF-8 path should convert to a file URI"); assert_eq!( - uri.to_native_path() + uri.to_abs_path() .expect("URI should convert to native path"), path ); @@ -111,10 +127,10 @@ fn file_uri_round_trips_literal_percent_characters() { fn file_uri_round_trips_windows_unc_paths() { let path = AbsolutePathBuf::from_absolute_path_checked(r"\\server\share\src\main.rs") .expect("absolute UNC path"); - let uri = PathUri::from_file_path(&path).expect("UNC path should convert to a file URI"); + let uri = PathUri::from_abs_path(&path).expect("UNC path should convert to a file URI"); assert_eq!(uri.encoded_path(), "/share/src/main.rs"); - assert_eq!(uri.to_native_path().expect("UNC URI should convert"), path); + assert_eq!(uri.to_abs_path().expect("UNC URI should convert"), path); } #[test] @@ -184,10 +200,17 @@ fn path_uri_deserializes_legacy_absolute_paths() { assert_eq!( uri, - PathUri::from_file_path(&path).expect("expected file URI") + PathUri::from_abs_path(&path).expect("expected file URI") ); } +#[test] +fn path_uri_rejects_relative_native_paths() { + let error = PathUri::from_path("src/lib.rs").expect_err("relative path should be rejected"); + + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); +} + #[test] fn path_uri_rejects_legacy_relative_paths_with_absolute_path_guard() { let base = AbsolutePathBuf::current_dir().expect("current directory"); From 24aee3eb5ee90dd211ef183d65fec774dc7049c9 Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Wed, 10 Jun 2026 09:18:19 -0700 Subject: [PATCH 34/47] [codex] link Windows releases with LLD (#27315) Windows x64 release builds spend about 36.5 of 48 minutes in final LLVM code generation and MSVC linking. Use the existing target-aware MSVC setup action to select LLD for release builds; the Windows ARM64 archive path already exercises the action and its LLD wrapper. In https://github.com/openai/codex/actions/runs/27242495616, macOS becomes the critical path after roughly four minutes of Windows improvement, so this is expected to reduce total workflow time by about four minutes. --- .github/workflows/rust-release-windows.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/rust-release-windows.yml b/.github/workflows/rust-release-windows.yml index e7574d85d2..49e627c1be 100644 --- a/.github/workflows/rust-release-windows.yml +++ b/.github/workflows/rust-release-windows.yml @@ -89,6 +89,11 @@ jobs: with: targets: ${{ matrix.target }} + - name: Configure LLVM linker + uses: ./.github/actions/setup-msvc-env + with: + target: ${{ matrix.target }} + - name: Cargo build (Windows binaries) shell: bash run: | From c365b8a4abfe0436ca85978fd186bd89d55a8807 Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Wed, 10 Jun 2026 09:28:12 -0700 Subject: [PATCH 35/47] Reduce archive rollout lookup CPU (#27276) ## Why Archiving a thread can spike app-server CPU when the state DB does not have a usable rollout path. The archive path falls back to locating the rollout by thread id; because rollout filenames already contain the UUID, the cheap fallback should find the file directly before invoking broader file search. ## What Changed - In `codex-rs/rollout/src/list.rs`, try the exact rollout filename lookup before `codex-file-search`. - Keep fuzzy search as the final legacy fallback when no filename match is found. - Preserve the legacy fallback when the filename scan hits a traversal error, so an inaccessible stale subtree does not block lookup elsewhere. ## Verification - `just test -p codex-rollout` - `just test -p codex-thread-store` - `just test -p codex-app-server thread_archive` --- codex-rs/rollout/src/list.rs | 70 ++++++++++++++++++++++++------------ 1 file changed, 47 insertions(+), 23 deletions(-) diff --git a/codex-rs/rollout/src/list.rs b/codex-rs/rollout/src/list.rs index fb740bb1df..af3785a975 100644 --- a/codex-rs/rollout/src/list.rs +++ b/codex-rs/rollout/src/list.rs @@ -1348,32 +1348,56 @@ async fn find_thread_path_by_id_str_in_subdir( if !root.exists() { return Ok(unverified_db_path); } - // This is safe because we know the values are valid. - #[allow(clippy::unwrap_used)] - let limit = NonZero::new(1).unwrap(); - let options = file_search::FileSearchOptions { - limit, - compute_indices: false, - respect_gitignore: false, - ..Default::default() + let (filename_match, filename_scan_error) = match find_rollout_path_by_id_from_filenames( + root.as_path(), + id_str, + ) + .await + { + Ok(path) => (path, None), + Err(err) => { + tracing::warn!( + "rollout filename lookup failed during find_thread_path_by_id_str_in_subdir: {err}" + ); + (None, Some(err)) + } }; - let results = file_search::run( - id_str, - vec![root.clone()], - options, - /*cancel_flag*/ None, - ) - .map_err(|e| io::Error::other(format!("file search failed: {e}")))?; + let found = match filename_match { + Some(path) => Some(path), + None => { + // This is safe because we know the values are valid. + #[allow(clippy::unwrap_used)] + let limit = NonZero::new(1).unwrap(); + let options = file_search::FileSearchOptions { + limit, + compute_indices: false, + respect_gitignore: false, + ..Default::default() + }; - let found = match results - .matches - .into_iter() - .map(|m| m.full_path()) - .find_map(compression::RolloutFile::from_path) - { - Some(rollout_file) => Some(rollout_file.into_path()), - None => find_rollout_path_by_id_from_filenames(root.as_path(), id_str).await?, + let results = file_search::run( + id_str, + vec![root.clone()], + options, + /*cancel_flag*/ None, + ) + .map_err(|e| io::Error::other(format!("file search failed: {e}")))?; + + let found = results + .matches + .into_iter() + .map(|m| m.full_path()) + .find_map(compression::RolloutFile::from_path) + .map(compression::RolloutFile::into_path); + + if found.is_none() + && let Some(err) = filename_scan_error + { + return Err(err); + } + found + } }; if let Some(found_path) = found.as_ref() { tracing::debug!("state db missing rollout path for thread {id_str}"); From db531b4a6c07dd8a4cb624f25b8d6abbd28ccf96 Mon Sep 17 00:00:00 2001 From: "Adam Perry @ OpenAI" Date: Wed, 10 Jun 2026 09:40:41 -0700 Subject: [PATCH 36/47] [codex] Outline ToolExecutor handler bodies (#27299) ## Why We're now [discouraging use of `async_trait`](https://github.com/openai/codex/pull/20242). Removing use of `async_trait` from `ToolExecutor` yields a `codex_core` debug test build speedup of ~78% (from 227.5s to 50.3s) on my machine. For ease of reviewing, this is a prefactor to extract trait method implementations to inherent methods. This will prevent changing indentation from creating a huge diff. ## What Outlined existing `ToolExecutor::handle` bodies into inherent async `handle_call` methods across core and extension tool handlers. The trait methods still use `async_trait` and now delegate to `self.handle_call(...).await`; handler behavior is unchanged. --- .../src/tools/code_mode/execute_handler.rs | 9 +++++++++ .../core/src/tools/code_mode/wait_handler.rs | 9 +++++++++ .../agent_jobs/report_agent_job_result.rs | 9 +++++++++ .../handlers/agent_jobs/spawn_agents_on_csv.rs | 9 +++++++++ .../core/src/tools/handlers/apply_patch.rs | 9 +++++++++ codex-rs/core/src/tools/handlers/dynamic.rs | 9 +++++++++ .../core/src/tools/handlers/extension_tools.rs | 18 ++++++++++++++++++ .../list_available_plugins_to_install.rs | 9 +++++++++ codex-rs/core/src/tools/handlers/mcp.rs | 9 +++++++++ .../list_mcp_resource_templates.rs | 9 +++++++++ .../mcp_resource/list_mcp_resources.rs | 9 +++++++++ .../handlers/mcp_resource/read_mcp_resource.rs | 9 +++++++++ .../tools/handlers/multi_agents/send_input.rs | 9 +++++++++ .../src/tools/handlers/multi_agents/wait.rs | 9 +++++++++ .../handlers/multi_agents_v2/followup_task.rs | 9 +++++++++ .../handlers/multi_agents_v2/list_agents.rs | 9 +++++++++ .../handlers/multi_agents_v2/send_message.rs | 9 +++++++++ .../src/tools/handlers/multi_agents_v2/wait.rs | 9 +++++++++ codex-rs/core/src/tools/handlers/plan.rs | 9 +++++++++ .../src/tools/handlers/request_permissions.rs | 9 +++++++++ .../tools/handlers/request_plugin_install.rs | 9 +++++++++ .../src/tools/handlers/request_user_input.rs | 9 +++++++++ .../src/tools/handlers/shell/shell_command.rs | 9 +++++++++ codex-rs/core/src/tools/handlers/test_sync.rs | 9 +++++++++ .../core/src/tools/handlers/tool_search.rs | 9 +++++++++ .../handlers/unified_exec/exec_command.rs | 9 +++++++++ .../tools/handlers/unified_exec/write_stdin.rs | 9 +++++++++ codex-rs/core/src/tools/handlers/view_image.rs | 9 +++++++++ codex-rs/core/src/tools/parallel.rs | 9 +++++++++ codex-rs/core/src/tools/registry_tests.rs | 8 ++++++++ codex-rs/core/src/tools/router_tests.rs | 9 +++++++++ codex-rs/ext/image-generation/src/tool.rs | 6 ++++++ codex-rs/ext/memories/src/tools/ad_hoc_note.rs | 13 +++++++++++++ codex-rs/ext/memories/src/tools/list.rs | 13 +++++++++++++ codex-rs/ext/memories/src/tools/read.rs | 13 +++++++++++++ codex-rs/ext/memories/src/tools/search.rs | 13 +++++++++++++ codex-rs/ext/web-search/src/tool.rs | 6 ++++++ 37 files changed, 351 insertions(+) diff --git a/codex-rs/core/src/tools/code_mode/execute_handler.rs b/codex-rs/core/src/tools/code_mode/execute_handler.rs index 1d69c8f4cc..0bdb8b20a6 100644 --- a/codex-rs/core/src/tools/code_mode/execute_handler.rs +++ b/codex-rs/core/src/tools/code_mode/execute_handler.rs @@ -104,6 +104,15 @@ impl ToolExecutor for CodeModeExecuteHandler { async fn handle( &self, invocation: ToolInvocation, + ) -> Result, FunctionCallError> { + self.handle_call(invocation).await + } +} + +impl CodeModeExecuteHandler { + async fn handle_call( + &self, + invocation: ToolInvocation, ) -> Result, FunctionCallError> { let ToolInvocation { session, diff --git a/codex-rs/core/src/tools/code_mode/wait_handler.rs b/codex-rs/core/src/tools/code_mode/wait_handler.rs index 74cc314a7b..c1928955f4 100644 --- a/codex-rs/core/src/tools/code_mode/wait_handler.rs +++ b/codex-rs/core/src/tools/code_mode/wait_handler.rs @@ -57,6 +57,15 @@ impl ToolExecutor for CodeModeWaitHandler { async fn handle( &self, invocation: ToolInvocation, + ) -> Result, FunctionCallError> { + self.handle_call(invocation).await + } +} + +impl CodeModeWaitHandler { + async fn handle_call( + &self, + invocation: ToolInvocation, ) -> Result, FunctionCallError> { let ToolInvocation { session, diff --git a/codex-rs/core/src/tools/handlers/agent_jobs/report_agent_job_result.rs b/codex-rs/core/src/tools/handlers/agent_jobs/report_agent_job_result.rs index c7adb27610..5f2a615329 100644 --- a/codex-rs/core/src/tools/handlers/agent_jobs/report_agent_job_result.rs +++ b/codex-rs/core/src/tools/handlers/agent_jobs/report_agent_job_result.rs @@ -26,6 +26,15 @@ impl ToolExecutor for ReportAgentJobResultHandler { async fn handle( &self, invocation: ToolInvocation, + ) -> Result, FunctionCallError> { + self.handle_call(invocation).await + } +} + +impl ReportAgentJobResultHandler { + async fn handle_call( + &self, + invocation: ToolInvocation, ) -> Result, FunctionCallError> { let ToolInvocation { session, payload, .. diff --git a/codex-rs/core/src/tools/handlers/agent_jobs/spawn_agents_on_csv.rs b/codex-rs/core/src/tools/handlers/agent_jobs/spawn_agents_on_csv.rs index 26438ca7ba..16e1b1f72c 100644 --- a/codex-rs/core/src/tools/handlers/agent_jobs/spawn_agents_on_csv.rs +++ b/codex-rs/core/src/tools/handlers/agent_jobs/spawn_agents_on_csv.rs @@ -27,6 +27,15 @@ impl ToolExecutor for SpawnAgentsOnCsvHandler { async fn handle( &self, invocation: ToolInvocation, + ) -> Result, FunctionCallError> { + self.handle_call(invocation).await + } +} + +impl SpawnAgentsOnCsvHandler { + async fn handle_call( + &self, + invocation: ToolInvocation, ) -> Result, FunctionCallError> { let ToolInvocation { session, diff --git a/codex-rs/core/src/tools/handlers/apply_patch.rs b/codex-rs/core/src/tools/handlers/apply_patch.rs index 66fb1d3080..6a322ed6ae 100644 --- a/codex-rs/core/src/tools/handlers/apply_patch.rs +++ b/codex-rs/core/src/tools/handlers/apply_patch.rs @@ -318,6 +318,15 @@ impl ToolExecutor for ApplyPatchHandler { async fn handle( &self, invocation: ToolInvocation, + ) -> Result, FunctionCallError> { + self.handle_call(invocation).await + } +} + +impl ApplyPatchHandler { + async fn handle_call( + &self, + invocation: ToolInvocation, ) -> Result, FunctionCallError> { let ToolInvocation { session, diff --git a/codex-rs/core/src/tools/handlers/dynamic.rs b/codex-rs/core/src/tools/handlers/dynamic.rs index a74f44fd6d..e0151cf189 100644 --- a/codex-rs/core/src/tools/handlers/dynamic.rs +++ b/codex-rs/core/src/tools/handlers/dynamic.rs @@ -89,6 +89,15 @@ impl ToolExecutor for DynamicToolHandler { async fn handle( &self, invocation: ToolInvocation, + ) -> Result, FunctionCallError> { + self.handle_call(invocation).await + } +} + +impl DynamicToolHandler { + async fn handle_call( + &self, + invocation: ToolInvocation, ) -> Result, FunctionCallError> { let ToolInvocation { session, diff --git a/codex-rs/core/src/tools/handlers/extension_tools.rs b/codex-rs/core/src/tools/handlers/extension_tools.rs index 5e177156c6..a94c764a9f 100644 --- a/codex-rs/core/src/tools/handlers/extension_tools.rs +++ b/codex-rs/core/src/tools/handlers/extension_tools.rs @@ -221,6 +221,15 @@ mod tests { async fn handle( &self, call: codex_tools::ToolCall, + ) -> Result, codex_tools::FunctionCallError> { + self.handle_call(call).await + } + } + + impl CapturingExtensionExecutor { + async fn handle_call( + &self, + call: codex_tools::ToolCall, ) -> Result, codex_tools::FunctionCallError> { let item = ExtensionTurnItem::WebSearch(WebSearchItem { id: call.call_id.clone(), @@ -452,6 +461,15 @@ mod tests { async fn handle( &self, call: codex_tools::ToolCall, + ) -> Result, codex_tools::FunctionCallError> { + self.handle_call(call).await + } + } + + impl ImageGenerationExtensionExecutor { + async fn handle_call( + &self, + call: codex_tools::ToolCall, ) -> Result, codex_tools::FunctionCallError> { call.turn_item_emitter .emit_started(ExtensionTurnItem::ImageGeneration( diff --git a/codex-rs/core/src/tools/handlers/list_available_plugins_to_install.rs b/codex-rs/core/src/tools/handlers/list_available_plugins_to_install.rs index d18a1ca36e..2b782e4552 100644 --- a/codex-rs/core/src/tools/handlers/list_available_plugins_to_install.rs +++ b/codex-rs/core/src/tools/handlers/list_available_plugins_to_install.rs @@ -71,6 +71,15 @@ impl ToolExecutor for ListAvailablePluginsToInstallHandler { async fn handle( &self, invocation: ToolInvocation, + ) -> Result, FunctionCallError> { + self.handle_call(invocation).await + } +} + +impl ListAvailablePluginsToInstallHandler { + async fn handle_call( + &self, + invocation: ToolInvocation, ) -> Result, FunctionCallError> { let ToolInvocation { payload, .. } = invocation; match payload { diff --git a/codex-rs/core/src/tools/handlers/mcp.rs b/codex-rs/core/src/tools/handlers/mcp.rs index 77ff3c14a3..d60cdfdd0c 100644 --- a/codex-rs/core/src/tools/handlers/mcp.rs +++ b/codex-rs/core/src/tools/handlers/mcp.rs @@ -116,6 +116,15 @@ impl ToolExecutor for McpHandler { async fn handle( &self, invocation: ToolInvocation, + ) -> Result, FunctionCallError> { + self.handle_call(invocation).await + } +} + +impl McpHandler { + async fn handle_call( + &self, + invocation: ToolInvocation, ) -> Result, FunctionCallError> { let ToolInvocation { session, diff --git a/codex-rs/core/src/tools/handlers/mcp_resource/list_mcp_resource_templates.rs b/codex-rs/core/src/tools/handlers/mcp_resource/list_mcp_resource_templates.rs index e96aad7552..8974d678fd 100644 --- a/codex-rs/core/src/tools/handlers/mcp_resource/list_mcp_resource_templates.rs +++ b/codex-rs/core/src/tools/handlers/mcp_resource/list_mcp_resource_templates.rs @@ -43,6 +43,15 @@ impl ToolExecutor for ListMcpResourceTemplatesHandler { async fn handle( &self, invocation: ToolInvocation, + ) -> Result, FunctionCallError> { + self.handle_call(invocation).await + } +} + +impl ListMcpResourceTemplatesHandler { + async fn handle_call( + &self, + invocation: ToolInvocation, ) -> Result, FunctionCallError> { let ToolInvocation { session, diff --git a/codex-rs/core/src/tools/handlers/mcp_resource/list_mcp_resources.rs b/codex-rs/core/src/tools/handlers/mcp_resource/list_mcp_resources.rs index 1b052184ee..f3a7b43c28 100644 --- a/codex-rs/core/src/tools/handlers/mcp_resource/list_mcp_resources.rs +++ b/codex-rs/core/src/tools/handlers/mcp_resource/list_mcp_resources.rs @@ -43,6 +43,15 @@ impl ToolExecutor for ListMcpResourcesHandler { async fn handle( &self, invocation: ToolInvocation, + ) -> Result, FunctionCallError> { + self.handle_call(invocation).await + } +} + +impl ListMcpResourcesHandler { + async fn handle_call( + &self, + invocation: ToolInvocation, ) -> Result, FunctionCallError> { let ToolInvocation { session, diff --git a/codex-rs/core/src/tools/handlers/mcp_resource/read_mcp_resource.rs b/codex-rs/core/src/tools/handlers/mcp_resource/read_mcp_resource.rs index f059d12c84..9861e5d22d 100644 --- a/codex-rs/core/src/tools/handlers/mcp_resource/read_mcp_resource.rs +++ b/codex-rs/core/src/tools/handlers/mcp_resource/read_mcp_resource.rs @@ -43,6 +43,15 @@ impl ToolExecutor for ReadMcpResourceHandler { async fn handle( &self, invocation: ToolInvocation, + ) -> Result, FunctionCallError> { + self.handle_call(invocation).await + } +} + +impl ReadMcpResourceHandler { + async fn handle_call( + &self, + invocation: ToolInvocation, ) -> Result, FunctionCallError> { let ToolInvocation { session, diff --git a/codex-rs/core/src/tools/handlers/multi_agents/send_input.rs b/codex-rs/core/src/tools/handlers/multi_agents/send_input.rs index 6e28d0a1d9..b811faf91a 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents/send_input.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents/send_input.rs @@ -26,6 +26,15 @@ impl ToolExecutor for Handler { async fn handle( &self, invocation: ToolInvocation, + ) -> Result, FunctionCallError> { + self.handle_call(invocation).await + } +} + +impl Handler { + async fn handle_call( + &self, + invocation: ToolInvocation, ) -> Result, FunctionCallError> { let ToolInvocation { session, diff --git a/codex-rs/core/src/tools/handlers/multi_agents/wait.rs b/codex-rs/core/src/tools/handlers/multi_agents/wait.rs index 525ee7f2ed..1f6b7db8f1 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents/wait.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents/wait.rs @@ -47,6 +47,15 @@ impl ToolExecutor for Handler { async fn handle( &self, invocation: ToolInvocation, + ) -> Result, FunctionCallError> { + self.handle_call(invocation).await + } +} + +impl Handler { + async fn handle_call( + &self, + invocation: ToolInvocation, ) -> Result, FunctionCallError> { let ToolInvocation { session, diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/followup_task.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/followup_task.rs index 2fe09f5f21..66a39cc79d 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/followup_task.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/followup_task.rs @@ -20,6 +20,15 @@ impl ToolExecutor for Handler { async fn handle( &self, invocation: ToolInvocation, + ) -> Result, FunctionCallError> { + self.handle_call(invocation).await + } +} + +impl Handler { + async fn handle_call( + &self, + invocation: ToolInvocation, ) -> Result, FunctionCallError> { let arguments = function_arguments(invocation.payload.clone())?; let args: FollowupTaskArgs = parse_arguments(&arguments)?; diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/list_agents.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/list_agents.rs index 46b29bdf10..18421278fe 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/list_agents.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/list_agents.rs @@ -18,6 +18,15 @@ impl ToolExecutor for Handler { async fn handle( &self, invocation: ToolInvocation, + ) -> Result, FunctionCallError> { + self.handle_call(invocation).await + } +} + +impl Handler { + async fn handle_call( + &self, + invocation: ToolInvocation, ) -> Result, FunctionCallError> { let ToolInvocation { session, diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/send_message.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/send_message.rs index a8e3605840..2fd636a85c 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/send_message.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/send_message.rs @@ -20,6 +20,15 @@ impl ToolExecutor for Handler { async fn handle( &self, invocation: ToolInvocation, + ) -> Result, FunctionCallError> { + self.handle_call(invocation).await + } +} + +impl Handler { + async fn handle_call( + &self, + invocation: ToolInvocation, ) -> Result, FunctionCallError> { let arguments = function_arguments(invocation.payload.clone())?; let args: SendMessageArgs = parse_arguments(&arguments)?; diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/wait.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/wait.rs index 5e37725b77..9d671b1989 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/wait.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/wait.rs @@ -32,6 +32,15 @@ impl ToolExecutor for Handler { async fn handle( &self, invocation: ToolInvocation, + ) -> Result, FunctionCallError> { + self.handle_call(invocation).await + } +} + +impl Handler { + async fn handle_call( + &self, + invocation: ToolInvocation, ) -> Result, FunctionCallError> { let ToolInvocation { session, diff --git a/codex-rs/core/src/tools/handlers/plan.rs b/codex-rs/core/src/tools/handlers/plan.rs index bd1060b3d7..ff18a32813 100644 --- a/codex-rs/core/src/tools/handlers/plan.rs +++ b/codex-rs/core/src/tools/handlers/plan.rs @@ -58,6 +58,15 @@ impl ToolExecutor for PlanHandler { async fn handle( &self, invocation: ToolInvocation, + ) -> Result, FunctionCallError> { + self.handle_call(invocation).await + } +} + +impl PlanHandler { + async fn handle_call( + &self, + invocation: ToolInvocation, ) -> Result, FunctionCallError> { let ToolInvocation { session, diff --git a/codex-rs/core/src/tools/handlers/request_permissions.rs b/codex-rs/core/src/tools/handlers/request_permissions.rs index a20ff2b665..7c24f15697 100644 --- a/codex-rs/core/src/tools/handlers/request_permissions.rs +++ b/codex-rs/core/src/tools/handlers/request_permissions.rs @@ -38,6 +38,15 @@ impl ToolExecutor for RequestPermissionsHandler { async fn handle( &self, invocation: ToolInvocation, + ) -> Result, FunctionCallError> { + self.handle_call(invocation).await + } +} + +impl RequestPermissionsHandler { + async fn handle_call( + &self, + invocation: ToolInvocation, ) -> Result, FunctionCallError> { let ToolInvocation { session, diff --git a/codex-rs/core/src/tools/handlers/request_plugin_install.rs b/codex-rs/core/src/tools/handlers/request_plugin_install.rs index ce5bb3fb1f..64625aaac0 100644 --- a/codex-rs/core/src/tools/handlers/request_plugin_install.rs +++ b/codex-rs/core/src/tools/handlers/request_plugin_install.rs @@ -65,6 +65,15 @@ impl ToolExecutor for RequestPluginInstallHandler { async fn handle( &self, invocation: ToolInvocation, + ) -> Result, FunctionCallError> { + self.handle_call(invocation).await + } +} + +impl RequestPluginInstallHandler { + async fn handle_call( + &self, + invocation: ToolInvocation, ) -> Result, FunctionCallError> { let ToolInvocation { payload, diff --git a/codex-rs/core/src/tools/handlers/request_user_input.rs b/codex-rs/core/src/tools/handlers/request_user_input.rs index 0aa4b4f6f8..3f3adf0ba3 100644 --- a/codex-rs/core/src/tools/handlers/request_user_input.rs +++ b/codex-rs/core/src/tools/handlers/request_user_input.rs @@ -33,6 +33,15 @@ impl ToolExecutor for RequestUserInputHandler { async fn handle( &self, invocation: ToolInvocation, + ) -> Result, FunctionCallError> { + self.handle_call(invocation).await + } +} + +impl RequestUserInputHandler { + async fn handle_call( + &self, + invocation: ToolInvocation, ) -> Result, FunctionCallError> { let ToolInvocation { session, diff --git a/codex-rs/core/src/tools/handlers/shell/shell_command.rs b/codex-rs/core/src/tools/handlers/shell/shell_command.rs index 7c492b3646..e9fcf82046 100644 --- a/codex-rs/core/src/tools/handlers/shell/shell_command.rs +++ b/codex-rs/core/src/tools/handlers/shell/shell_command.rs @@ -144,6 +144,15 @@ impl ToolExecutor for ShellCommandHandler { async fn handle( &self, invocation: ToolInvocation, + ) -> Result, FunctionCallError> { + self.handle_call(invocation).await + } +} + +impl ShellCommandHandler { + async fn handle_call( + &self, + invocation: ToolInvocation, ) -> Result, FunctionCallError> { let ToolInvocation { session, diff --git a/codex-rs/core/src/tools/handlers/test_sync.rs b/codex-rs/core/src/tools/handlers/test_sync.rs index 0e5fe894ad..802859c916 100644 --- a/codex-rs/core/src/tools/handlers/test_sync.rs +++ b/codex-rs/core/src/tools/handlers/test_sync.rs @@ -74,6 +74,15 @@ impl ToolExecutor for TestSyncHandler { async fn handle( &self, invocation: ToolInvocation, + ) -> Result, FunctionCallError> { + self.handle_call(invocation).await + } +} + +impl TestSyncHandler { + async fn handle_call( + &self, + invocation: ToolInvocation, ) -> Result, FunctionCallError> { let ToolInvocation { payload, .. } = invocation; diff --git a/codex-rs/core/src/tools/handlers/tool_search.rs b/codex-rs/core/src/tools/handlers/tool_search.rs index 2de1e20dd4..a4781e75e2 100644 --- a/codex-rs/core/src/tools/handlers/tool_search.rs +++ b/codex-rs/core/src/tools/handlers/tool_search.rs @@ -70,6 +70,15 @@ impl ToolExecutor for ToolSearchHandler { async fn handle( &self, invocation: ToolInvocation, + ) -> Result, FunctionCallError> { + self.handle_call(invocation).await + } +} + +impl ToolSearchHandler { + async fn handle_call( + &self, + invocation: ToolInvocation, ) -> Result, FunctionCallError> { let ToolInvocation { payload, .. } = invocation; diff --git a/codex-rs/core/src/tools/handlers/unified_exec/exec_command.rs b/codex-rs/core/src/tools/handlers/unified_exec/exec_command.rs index ba69f0d039..9fe3b343b5 100644 --- a/codex-rs/core/src/tools/handlers/unified_exec/exec_command.rs +++ b/codex-rs/core/src/tools/handlers/unified_exec/exec_command.rs @@ -95,6 +95,15 @@ impl ToolExecutor for ExecCommandHandler { async fn handle( &self, invocation: ToolInvocation, + ) -> Result, FunctionCallError> { + self.handle_call(invocation).await + } +} + +impl ExecCommandHandler { + async fn handle_call( + &self, + invocation: ToolInvocation, ) -> Result, FunctionCallError> { let ToolInvocation { session, diff --git a/codex-rs/core/src/tools/handlers/unified_exec/write_stdin.rs b/codex-rs/core/src/tools/handlers/unified_exec/write_stdin.rs index a639ea0065..9b319d5433 100644 --- a/codex-rs/core/src/tools/handlers/unified_exec/write_stdin.rs +++ b/codex-rs/core/src/tools/handlers/unified_exec/write_stdin.rs @@ -44,6 +44,15 @@ impl ToolExecutor for WriteStdinHandler { async fn handle( &self, invocation: ToolInvocation, + ) -> Result, FunctionCallError> { + self.handle_call(invocation).await + } +} + +impl WriteStdinHandler { + async fn handle_call( + &self, + invocation: ToolInvocation, ) -> Result, FunctionCallError> { let ToolInvocation { session, diff --git a/codex-rs/core/src/tools/handlers/view_image.rs b/codex-rs/core/src/tools/handlers/view_image.rs index 5a67db1a66..eb761359a7 100644 --- a/codex-rs/core/src/tools/handlers/view_image.rs +++ b/codex-rs/core/src/tools/handlers/view_image.rs @@ -81,6 +81,15 @@ impl ToolExecutor for ViewImageHandler { async fn handle( &self, invocation: ToolInvocation, + ) -> Result, FunctionCallError> { + self.handle_call(invocation).await + } +} + +impl ViewImageHandler { + async fn handle_call( + &self, + invocation: ToolInvocation, ) -> Result, FunctionCallError> { if !invocation .turn diff --git a/codex-rs/core/src/tools/parallel.rs b/codex-rs/core/src/tools/parallel.rs index 7db1ec9643..a6f630a342 100644 --- a/codex-rs/core/src/tools/parallel.rs +++ b/codex-rs/core/src/tools/parallel.rs @@ -314,6 +314,15 @@ mod tests { async fn handle( &self, invocation: ToolInvocation, + ) -> Result, FunctionCallError> { + self.handle_call(invocation).await + } + } + + impl CancellationCleanupHandler { + async fn handle_call( + &self, + invocation: ToolInvocation, ) -> Result, FunctionCallError> { let started = self .started diff --git a/codex-rs/core/src/tools/registry_tests.rs b/codex-rs/core/src/tools/registry_tests.rs index f2566251c5..cc7f39e2a3 100644 --- a/codex-rs/core/src/tools/registry_tests.rs +++ b/codex-rs/core/src/tools/registry_tests.rs @@ -51,6 +51,14 @@ impl ToolExecutor for LifecycleTestHandler { async fn handle( &self, _invocation: ToolInvocation, + ) -> Result, FunctionCallError> { + self.handle_call().await + } +} + +impl LifecycleTestHandler { + async fn handle_call( + &self, ) -> Result, FunctionCallError> { match self.result.clone() { LifecycleTestResult::Ok { success } => Ok(Box::new( diff --git a/codex-rs/core/src/tools/router_tests.rs b/codex-rs/core/src/tools/router_tests.rs index 7c3e0157bc..a9a52ca463 100644 --- a/codex-rs/core/src/tools/router_tests.rs +++ b/codex-rs/core/src/tools/router_tests.rs @@ -76,6 +76,15 @@ impl ToolExecutor for ExtensionEchoExecutor { async fn handle( &self, call: ExtensionToolCall, + ) -> Result, codex_tools::FunctionCallError> { + self.handle_call(call).await + } +} + +impl ExtensionEchoExecutor { + async fn handle_call( + &self, + call: ExtensionToolCall, ) -> Result, codex_tools::FunctionCallError> { let arguments: serde_json::Value = serde_json::from_str(call.function_arguments()?).expect("test arguments should parse"); diff --git a/codex-rs/ext/image-generation/src/tool.rs b/codex-rs/ext/image-generation/src/tool.rs index 69b99bdbc4..70880b4661 100644 --- a/codex-rs/ext/image-generation/src/tool.rs +++ b/codex-rs/ext/image-generation/src/tool.rs @@ -97,6 +97,12 @@ impl ToolExecutor for ImageGenerationTool { /// Executes the selected image operation and returns the completed image result. async fn handle(&self, call: ToolCall) -> Result, FunctionCallError> { + self.handle_call(call).await + } +} + +impl ImageGenerationTool { + async fn handle_call(&self, call: ToolCall) -> Result, FunctionCallError> { let args = parse_args(&call)?; let request = request_for_args(&args, call.conversation_history.items())?; call.turn_item_emitter diff --git a/codex-rs/ext/memories/src/tools/ad_hoc_note.rs b/codex-rs/ext/memories/src/tools/ad_hoc_note.rs index a6712a4022..3495d57d32 100644 --- a/codex-rs/ext/memories/src/tools/ad_hoc_note.rs +++ b/codex-rs/ext/memories/src/tools/ad_hoc_note.rs @@ -61,6 +61,19 @@ where &self, call: ToolCall, ) -> Result, codex_extension_api::FunctionCallError> + { + self.handle_call(call).await + } +} + +impl AddAdHocNoteTool +where + B: MemoriesBackend, +{ + async fn handle_call( + &self, + call: ToolCall, + ) -> Result, codex_extension_api::FunctionCallError> { let backend = self.backend.clone(); let args: AddAdHocNoteArgs = parse_args(&call)?; diff --git a/codex-rs/ext/memories/src/tools/list.rs b/codex-rs/ext/memories/src/tools/list.rs index 301c7cab71..d3b23b1ca2 100644 --- a/codex-rs/ext/memories/src/tools/list.rs +++ b/codex-rs/ext/memories/src/tools/list.rs @@ -59,6 +59,19 @@ where &self, call: ToolCall, ) -> Result, codex_extension_api::FunctionCallError> + { + self.handle_call(call).await + } +} + +impl ListTool +where + B: MemoriesBackend, +{ + async fn handle_call( + &self, + call: ToolCall, + ) -> Result, codex_extension_api::FunctionCallError> { let backend = self.backend.clone(); let args: ListArgs = parse_args(&call)?; diff --git a/codex-rs/ext/memories/src/tools/read.rs b/codex-rs/ext/memories/src/tools/read.rs index 33ede3c600..e408a17cad 100644 --- a/codex-rs/ext/memories/src/tools/read.rs +++ b/codex-rs/ext/memories/src/tools/read.rs @@ -58,6 +58,19 @@ where &self, call: ToolCall, ) -> Result, codex_extension_api::FunctionCallError> + { + self.handle_call(call).await + } +} + +impl ReadTool +where + B: MemoriesBackend, +{ + async fn handle_call( + &self, + call: ToolCall, + ) -> Result, codex_extension_api::FunctionCallError> { let backend = self.backend.clone(); let args: ReadArgs = parse_args(&call)?; diff --git a/codex-rs/ext/memories/src/tools/search.rs b/codex-rs/ext/memories/src/tools/search.rs index 1d05072703..cc8ba201a6 100644 --- a/codex-rs/ext/memories/src/tools/search.rs +++ b/codex-rs/ext/memories/src/tools/search.rs @@ -67,6 +67,19 @@ where &self, call: ToolCall, ) -> Result, codex_extension_api::FunctionCallError> + { + self.handle_call(call).await + } +} + +impl SearchTool +where + B: MemoriesBackend, +{ + async fn handle_call( + &self, + call: ToolCall, + ) -> Result, codex_extension_api::FunctionCallError> { let backend = self.backend.clone(); let args: SearchArgs = parse_args(&call)?; diff --git a/codex-rs/ext/web-search/src/tool.rs b/codex-rs/ext/web-search/src/tool.rs index 35f126dd6b..1de65edf86 100644 --- a/codex-rs/ext/web-search/src/tool.rs +++ b/codex-rs/ext/web-search/src/tool.rs @@ -75,6 +75,12 @@ impl ToolExecutor for WebSearchTool { } async fn handle(&self, call: ToolCall) -> Result, FunctionCallError> { + self.handle_call(call).await + } +} + +impl WebSearchTool { + async fn handle_call(&self, call: ToolCall) -> Result, FunctionCallError> { let commands = parse_commands(&call)?; let command_action = command_action(&commands); let provider = self From 2ef007dc1ad60f772a3e0ac6560d61f783783b9d Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Wed, 10 Jun 2026 11:52:17 -0500 Subject: [PATCH 37/47] Index visible thread list ordering (#27391) ## Summary - add partial SQLite indexes for visible thread lists ordered by creation or update time - match the `archived` and non-empty `preview` filters used by `thread/list` - add query-plan coverage for both supported sort orders ## Query performance Benchmarked the production query shape on a snapshot of my database with ~10k threads before and after applying these indexes. The query selected the full thread projection with `archived = 0`, `preview <> ''`, the `openai` provider filter, and a page size of 201. Results are the mean of 30 runs after 5 warmups: | Query | Before | After | Speedup | | --- | ---: | ---: | ---: | | First page, `created_at_ms DESC` | 132.3 ms | 15.1 ms | 8.78x | | First page, `updated_at_ms DESC` | 123.6 ms | 15.5 ms | 7.99x | | Cursor page near row 4,000, `created_at_ms DESC` | 51.8 ms | 16.8 ms | 3.07x | | Cursor page near row 4,000, `updated_at_ms DESC` | 52.4 ms | 17.1 ms | 3.06x | Before this change, SQLite used `idx_threads_archived`, filtered the candidate rows, and built a temporary B-tree for the requested ordering. With the partial indexes, SQLite reads matching visible rows directly in timestamp order and stops at the page limit. `EXPLAIN QUERY PLAN` no longer reports `USE TEMP B-TREE FOR ORDER BY`. The result rows were identical before and after. The two partial indexes occupy approximately 168 KiB combined on this snapshot. ## Performance under contention I noticed this issue on a database with high-contention and tried to use simulated contention to validate the performance in that context. A synthetic SQLite benchmark ran five concurrent readers, matching the state database pool size, and fetched 101 rows per query. Results are the median of three runs on fresh copies of the same database snapshot: | Query | Before | After | | --- | ---: | ---: | | `created_at_ms` mean latency under saturation | 328 ms | 12 ms | | `created_at_ms` throughput | 16 queries/s | 412 queries/s | | `updated_at_ms` mean latency under saturation | 336 ms | 14 ms | | `updated_at_ms` throughput | 15 queries/s | 357 queries/s | For a burst of 100 queries queued through five connections, p95 completion time fell from 6.90 seconds to 226 ms for `created_at_ms`, and from 6.31 seconds to 473 ms for `updated_at_ms`. ## Validation - `just test -p codex-state` (135 tests passed) - query-plan regression covers created-at and updated-at ordering, requires the corresponding index, and rejects `TEMP B-TREE` - `just fmt` --- .../0036_threads_visible_sort_indexes.sql | 7 + codex-rs/state/src/runtime/threads.rs | 181 ++++++++++++++++-- 2 files changed, 176 insertions(+), 12 deletions(-) create mode 100644 codex-rs/state/migrations/0036_threads_visible_sort_indexes.sql diff --git a/codex-rs/state/migrations/0036_threads_visible_sort_indexes.sql b/codex-rs/state/migrations/0036_threads_visible_sort_indexes.sql new file mode 100644 index 0000000000..33a83a9d7b --- /dev/null +++ b/codex-rs/state/migrations/0036_threads_visible_sort_indexes.sql @@ -0,0 +1,7 @@ +CREATE INDEX idx_threads_visible_created_at_ms + ON threads(archived, created_at_ms DESC) + WHERE preview <> ''; + +CREATE INDEX idx_threads_visible_updated_at_ms + ON threads(archived, updated_at_ms DESC) + WHERE preview <> ''; diff --git a/codex-rs/state/src/runtime/threads.rs b/codex-rs/state/src/runtime/threads.rs index 2c00500ca3..1ccfab5e12 100644 --- a/codex-rs/state/src/runtime/threads.rs +++ b/codex-rs/state/src/runtime/threads.rs @@ -384,6 +384,7 @@ ON CONFLICT(child_thread_id) DO NOTHING &mut builder, crate::SortKey::UpdatedAt, SortDirection::Desc, + OrderByIndex::Enabled, /*limit*/ 1, ); @@ -399,14 +400,9 @@ ON CONFLICT(child_thread_id) DO NOTHING filters: ThreadFilterOptions<'_>, ) -> anyhow::Result { let limit = page_size.saturating_add(1); - let sort_key = filters.sort_key; - let sort_direction = filters.sort_direction; let mut builder = QueryBuilder::::new(""); - push_thread_select_columns(&mut builder); - builder.push(" FROM threads"); - push_thread_filters(&mut builder, filters); - push_thread_order_and_limit(&mut builder, sort_key, sort_direction, limit); + push_list_threads_query(&mut builder, filters, limit); let rows = builder.build().fetch_all(self.pool.as_ref()).await?; let mut items = rows @@ -418,7 +414,7 @@ ON CONFLICT(child_thread_id) DO NOTHING items.pop(); items .last() - .and_then(|item| anchor_from_item(item, sort_key)) + .and_then(|item| anchor_from_item(item, filters.sort_key)) } else { None }; @@ -453,7 +449,13 @@ ON CONFLICT(child_thread_id) DO NOTHING search_term: None, }, ); - push_thread_order_and_limit(&mut builder, sort_key, SortDirection::Desc, limit); + push_thread_order_and_limit( + &mut builder, + sort_key, + SortDirection::Desc, + OrderByIndex::Enabled, + limit, + ); let rows = builder.build().fetch_all(self.pool.as_ref()).await?; rows.into_iter() @@ -917,6 +919,29 @@ fn one_thread_id_from_rows( } } +fn push_list_threads_query( + builder: &mut QueryBuilder, + filters: ThreadFilterOptions<'_>, + limit: usize, +) { + push_thread_select_columns(builder); + builder.push(" FROM threads"); + push_thread_filters(builder, filters); + let order_by_index = match filters.cwd_filters { + // Multi-cwd listing is supported but at the time of writing has no current use in production. + // Preserve its query plan so the global timestamp index does not regress cwd filtering into a scan. + Some(cwd_filters) if cwd_filters.len() > 1 => OrderByIndex::Disabled, + Some(_) | None => OrderByIndex::Enabled, + }; + push_thread_order_and_limit( + builder, + filters.sort_key, + filters.sort_direction, + order_by_index, + limit, + ); +} + pub(super) fn push_thread_select_columns(builder: &mut QueryBuilder) { builder.push( r#" @@ -1057,10 +1082,21 @@ pub(super) fn push_thread_filters<'a>( } } +/// Controls whether SQLite may use the ordered column to satisfy `ORDER BY` from an index. +/// +/// Disabling it adds a unary `+` to the ordered column. This preserves the sort semantics while +/// preventing a timestamp-only index from winning over a more selective filtering index. +#[derive(Clone, Copy)] +pub(super) enum OrderByIndex { + Enabled, + Disabled, +} + pub(super) fn push_thread_order_and_limit( builder: &mut QueryBuilder, sort_key: SortKey, sort_direction: SortDirection, + order_by_index: OrderByIndex, limit: usize, ) { let order_column = match sort_key { @@ -1072,6 +1108,12 @@ pub(super) fn push_thread_order_and_limit( SortDirection::Desc => "DESC", }; builder.push(" ORDER BY "); + match order_by_index { + OrderByIndex::Enabled => {} + OrderByIndex::Disabled => { + builder.push("+"); + } + } builder.push(order_column); builder.push(" "); builder.push(order_direction); @@ -1255,9 +1297,9 @@ mod tests { } let cwd_filters = vec![first_cwd, second_cwd]; - let page = runtime + let first_page = runtime .list_threads( - /*page_size*/ 10, + /*page_size*/ 1, ThreadFilterOptions { archived_only: false, allowed_sources: &[], @@ -1272,8 +1314,44 @@ mod tests { .await .expect("list should succeed"); - let ids = page.items.iter().map(|item| item.id).collect::>(); - assert_eq!(ids, vec![second_id, first_id]); + let ids = first_page + .items + .iter() + .map(|item| item.id) + .collect::>(); + assert_eq!(ids, vec![second_id]); + assert_eq!( + first_page.next_anchor, + Some(Anchor { + ts: DateTime::::from_timestamp_millis(1_700_000_300_000) + .expect("valid timestamp"), + }) + ); + + let second_page = runtime + .list_threads( + /*page_size*/ 1, + ThreadFilterOptions { + archived_only: false, + allowed_sources: &[], + model_providers: None, + cwd_filters: Some(cwd_filters.as_slice()), + anchor: first_page.next_anchor.as_ref(), + sort_key: SortKey::UpdatedAt, + sort_direction: SortDirection::Desc, + search_term: None, + }, + ) + .await + .expect("second page should succeed"); + + let ids = second_page + .items + .iter() + .map(|item| item.id) + .collect::>(); + assert_eq!(ids, vec![first_id]); + assert_eq!(second_page.next_anchor, None); let page = runtime .list_threads( @@ -1295,6 +1373,85 @@ mod tests { assert_eq!(page.items, Vec::new()); } + #[tokio::test] + async fn list_threads_uses_indexes_matching_cwd_filters() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home, "test-provider".to_string()) + .await + .expect("state db should initialize"); + + let model_providers = ["test-provider".to_string()]; + let cwd_filters = [ + PathBuf::from("/workspace/one"), + PathBuf::from("/workspace/two"), + ]; + let anchor = Anchor { + ts: DateTime::::from_timestamp(1_700_000_000, 0).expect("valid timestamp"), + }; + for (sort_key, visible_index, cwd_index) in [ + ( + SortKey::CreatedAt, + "idx_threads_visible_created_at_ms", + "idx_threads_archived_cwd_created_at_ms", + ), + ( + SortKey::UpdatedAt, + "idx_threads_visible_updated_at_ms", + "idx_threads_archived_cwd_updated_at_ms", + ), + ] { + for (cwd_filters, anchor, expected_index, expect_temp_sort) in [ + (None, None, visible_index, false), + (Some(&cwd_filters[..1]), None, cwd_index, false), + ( + Some(&cwd_filters[..]), + None, + "idx_threads_archived_cwd_", + true, + ), + (Some(&cwd_filters[..]), Some(&anchor), cwd_index, true), + ] { + let mut builder = QueryBuilder::::new("EXPLAIN QUERY PLAN "); + push_list_threads_query( + &mut builder, + ThreadFilterOptions { + archived_only: false, + allowed_sources: &[], + model_providers: Some(&model_providers), + cwd_filters, + anchor, + sort_key, + sort_direction: SortDirection::Desc, + search_term: None, + }, + /*limit*/ 201, + ); + let plan_details = builder + .build() + .fetch_all(runtime.pool.as_ref()) + .await + .expect("query plan should load") + .into_iter() + .map(|row| row.get::("detail")) + .collect::>(); + + assert!( + plan_details + .iter() + .any(|detail| detail.contains(expected_index)), + "query plan did not use {expected_index}: {plan_details:?}" + ); + assert_eq!( + plan_details + .iter() + .any(|detail| detail.contains("TEMP B-TREE")), + expect_temp_sort, + "unexpected sorting plan: {plan_details:?}" + ); + } + } + } + #[tokio::test] async fn apply_rollout_items_restores_memory_mode_from_session_meta() { let codex_home = unique_temp_dir(); From d3abd8774ed2caa13db253ee232791e5a2d2d8c5 Mon Sep 17 00:00:00 2001 From: jif Date: Wed, 10 Jun 2026 18:23:42 +0100 Subject: [PATCH 38/47] Fix compressed rollout search path matching (#27407) ## Why `thread/search` found content inside compressed rollouts but could drop the result when joining it with SQLite-backed thread metadata. Search returned the physical `.jsonl.zst` path while SQLite retained the logical `.jsonl` path, so exact path matching failed. ## What changed - Key rollout search matches by their canonical logical `.jsonl` path, independent of the on-disk representation. - Canonicalize thread-list paths before joining them with content-search matches. - Update compressed-rollout coverage to assert the logical-path contract. ## Validation - Ran `just fmt`. - Ran `git diff --check`. - Tests and Clippy were intentionally left to CI. --- codex-rs/rollout/src/compression_tests.rs | 5 ++--- codex-rs/rollout/src/search.rs | 11 +++++++++-- codex-rs/thread-store/src/local/search_threads.rs | 3 ++- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/codex-rs/rollout/src/compression_tests.rs b/codex-rs/rollout/src/compression_tests.rs index eb81112965..682361ea9d 100644 --- a/codex-rs/rollout/src/compression_tests.rs +++ b/codex-rs/rollout/src/compression_tests.rs @@ -106,14 +106,13 @@ async fn append_rollout_item_materializes_compressed_rollout() -> anyhow::Result } #[tokio::test] -async fn search_rollout_matches_returns_compressed_snippet() -> anyhow::Result<()> { +async fn search_rollout_matches_uses_logical_path_for_compressed_rollout() -> anyhow::Result<()> { let home = TempDir::new()?; let uuid = Uuid::from_u128(15); let thread_id = ThreadId::from_string(&uuid.to_string())?; let rollout_path = rollout_path(home.path(), "2025-01-03T12-00-00", uuid); write_rollout(&rollout_path, thread_id, "targeted search term")?; compress_now(&rollout_path)?; - let compressed_path = compressed_rollout_path(&rollout_path); let matches = search_rollout_matches( std::path::Path::new("missing-rg-for-test"), @@ -124,7 +123,7 @@ async fn search_rollout_matches_returns_compressed_snippet() -> anyhow::Result<( .await?; assert_eq!( - matches.get(compressed_path.as_path()), + matches.get(rollout_path.as_path()), Some(&Some("targeted search term".to_string())) ); Ok(()) diff --git a/codex-rs/rollout/src/search.rs b/codex-rs/rollout/src/search.rs index 705d24e7b4..622fac8b0f 100644 --- a/codex-rs/rollout/src/search.rs +++ b/codex-rs/rollout/src/search.rs @@ -21,6 +21,7 @@ use super::compression; const MATCH_CONTEXT_BEFORE_CHARS: usize = 48; const MATCH_CONTEXT_AFTER_CHARS: usize = 96; +/// Search matches keyed by the canonical `.jsonl` path for each rollout. pub type RolloutSearchMatches = HashMap>; pub async fn search_rollout_paths( @@ -145,7 +146,10 @@ async fn scan_rollout_matches( if let Some(snippet) = first_rollout_content_match_snippet(rollout_file.path(), search_term).await? { - matches.insert(rollout_file.into_path(), Some(snippet)); + matches.insert( + compression::plain_rollout_path(rollout_file.path()), + Some(snippet), + ); } continue; } @@ -217,7 +221,10 @@ async fn scan_compressed_rollout_matches( if let Some(snippet) = first_rollout_content_match_snippet(rollout_file.path(), search_term).await? { - matches.insert(rollout_file.into_path(), Some(snippet)); + matches.insert( + compression::plain_rollout_path(rollout_file.path()), + Some(snippet), + ); } } } diff --git a/codex-rs/thread-store/src/local/search_threads.rs b/codex-rs/thread-store/src/local/search_threads.rs index 20f5da8243..c9394c3962 100644 --- a/codex-rs/thread-store/src/local/search_threads.rs +++ b/codex-rs/thread-store/src/local/search_threads.rs @@ -109,7 +109,8 @@ pub(super) async fn search_threads( ) .await?; for item in page.items { - let Some(snippet) = (match remaining_rollouts.remove(item.path.as_path()) { + let logical_path = codex_rollout::plain_rollout_path(item.path.as_path()); + let Some(snippet) = (match remaining_rollouts.remove(logical_path.as_path()) { Some(Some(snippet)) => Some(snippet), Some(None) => first_rollout_content_match_snippet(item.path.as_path(), search_term) .await From 2704ecea9a1d52ece2429da4ed5775000b59164d Mon Sep 17 00:00:00 2001 From: "Adam Perry @ OpenAI" Date: Wed, 10 Jun 2026 10:26:53 -0700 Subject: [PATCH 39/47] [codex] Remove async_trait from ToolExecutor (#27304) ## Why We're now [discouraging use of `async_trait`](https://github.com/openai/codex/pull/20242). Removing use of `async_trait` from `ToolExecutor` yields a `codex_core` debug test build speedup of ~78% (from 227.5s to 50.3s) on my machine. Stacked on #27299, this PR applies the trait change after the handler bodies have been outlined. ## What Changed `ToolExecutor::handle` to return an explicit boxed `ToolExecutorFuture` instead of using `async_trait`. Updated ToolExecutor implementors to return `Box::pin(...)`, reexported the future alias through `codex-tools` and `codex-extension-api`, and removed `codex-tools` direct `async-trait` dependency. --- codex-rs/Cargo.lock | 5 -- .../src/tools/code_mode/execute_handler.rs | 8 +-- .../core/src/tools/code_mode/wait_handler.rs | 8 +-- .../agent_jobs/report_agent_job_result.rs | 8 +-- .../agent_jobs/spawn_agents_on_csv.rs | 8 +-- .../core/src/tools/handlers/apply_patch.rs | 8 +-- codex-rs/core/src/tools/handlers/dynamic.rs | 8 +-- .../src/tools/handlers/extension_tools.rs | 55 +++++++------------ .../list_available_plugins_to_install.rs | 8 +-- codex-rs/core/src/tools/handlers/mcp.rs | 8 +-- .../list_mcp_resource_templates.rs | 8 +-- .../mcp_resource/list_mcp_resources.rs | 8 +-- .../mcp_resource/read_mcp_resource.rs | 8 +-- .../handlers/multi_agents/close_agent.rs | 8 +-- .../handlers/multi_agents/resume_agent.rs | 8 +-- .../tools/handlers/multi_agents/send_input.rs | 8 +-- .../src/tools/handlers/multi_agents/spawn.rs | 8 +-- .../src/tools/handlers/multi_agents/wait.rs | 8 +-- .../handlers/multi_agents_v2/followup_task.rs | 8 +-- .../multi_agents_v2/interrupt_agent.rs | 14 ++--- .../handlers/multi_agents_v2/list_agents.rs | 8 +-- .../handlers/multi_agents_v2/send_message.rs | 8 +-- .../tools/handlers/multi_agents_v2/spawn.rs | 8 +-- .../tools/handlers/multi_agents_v2/wait.rs | 8 +-- codex-rs/core/src/tools/handlers/plan.rs | 8 +-- .../src/tools/handlers/request_permissions.rs | 8 +-- .../tools/handlers/request_plugin_install.rs | 8 +-- .../src/tools/handlers/request_user_input.rs | 8 +-- .../src/tools/handlers/shell/shell_command.rs | 8 +-- codex-rs/core/src/tools/handlers/test_sync.rs | 8 +-- .../core/src/tools/handlers/tool_search.rs | 8 +-- .../handlers/unified_exec/exec_command.rs | 8 +-- .../handlers/unified_exec/write_stdin.rs | 8 +-- .../core/src/tools/handlers/view_image.rs | 8 +-- codex-rs/core/src/tools/parallel.rs | 26 ++++----- codex-rs/core/src/tools/registry.rs | 8 +-- codex-rs/core/src/tools/registry_tests.rs | 28 +++++----- codex-rs/core/src/tools/router_tests.rs | 10 +--- codex-rs/core/src/tools/spec_plan.rs | 9 +-- codex-rs/core/src/tools/spec_plan_tests.rs | 18 ++---- .../src/tools/tool_dispatch_trace_tests.rs | 16 +++--- codex-rs/ext/extension-api/src/lib.rs | 1 + codex-rs/ext/goal/Cargo.toml | 1 - codex-rs/ext/goal/src/tool.rs | 16 +++--- codex-rs/ext/image-generation/Cargo.toml | 1 - codex-rs/ext/image-generation/src/tool.rs | 5 +- codex-rs/ext/memories/Cargo.toml | 1 - .../ext/memories/src/tools/ad_hoc_note.rs | 9 +-- codex-rs/ext/memories/src/tools/list.rs | 9 +-- codex-rs/ext/memories/src/tools/read.rs | 9 +-- codex-rs/ext/memories/src/tools/search.rs | 9 +-- codex-rs/ext/web-search/Cargo.toml | 1 - codex-rs/ext/web-search/src/tool.rs | 5 +- codex-rs/tools/Cargo.toml | 1 - codex-rs/tools/src/lib.rs | 1 + codex-rs/tools/src/tool_executor.rs | 12 ++-- 56 files changed, 161 insertions(+), 357 deletions(-) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 17b2696cbf..51a6cd08cc 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -3047,7 +3047,6 @@ name = "codex-goal-extension" version = "0.0.0" dependencies = [ "anyhow", - "async-trait", "chrono", "codex-analytics", "codex-core", @@ -3101,7 +3100,6 @@ dependencies = [ name = "codex-image-generation-extension" version = "0.0.0" dependencies = [ - "async-trait", "codex-api", "codex-core", "codex-extension-api", @@ -3301,7 +3299,6 @@ dependencies = [ name = "codex-memories-extension" version = "0.0.0" dependencies = [ - "async-trait", "codex-core", "codex-extension-api", "codex-features", @@ -3916,7 +3913,6 @@ dependencies = [ name = "codex-tools" version = "0.0.0" dependencies = [ - "async-trait", "codex-app-server-protocol", "codex-code-mode", "codex-features", @@ -4297,7 +4293,6 @@ dependencies = [ name = "codex-web-search-extension" version = "0.0.0" dependencies = [ - "async-trait", "codex-api", "codex-core", "codex-extension-api", diff --git a/codex-rs/core/src/tools/code_mode/execute_handler.rs b/codex-rs/core/src/tools/code_mode/execute_handler.rs index 0bdb8b20a6..2e46c1bd6b 100644 --- a/codex-rs/core/src/tools/code_mode/execute_handler.rs +++ b/codex-rs/core/src/tools/code_mode/execute_handler.rs @@ -91,7 +91,6 @@ impl CodeModeExecuteHandler { } } -#[async_trait::async_trait] impl ToolExecutor for CodeModeExecuteHandler { fn tool_name(&self) -> ToolName { ToolName::plain(PUBLIC_TOOL_NAME) @@ -101,11 +100,8 @@ impl ToolExecutor for CodeModeExecuteHandler { self.spec.clone() } - async fn handle( - &self, - invocation: ToolInvocation, - ) -> Result, FunctionCallError> { - self.handle_call(invocation).await + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(self.handle_call(invocation)) } } diff --git a/codex-rs/core/src/tools/code_mode/wait_handler.rs b/codex-rs/core/src/tools/code_mode/wait_handler.rs index c1928955f4..953391a573 100644 --- a/codex-rs/core/src/tools/code_mode/wait_handler.rs +++ b/codex-rs/core/src/tools/code_mode/wait_handler.rs @@ -44,7 +44,6 @@ where }) } -#[async_trait::async_trait] impl ToolExecutor for CodeModeWaitHandler { fn tool_name(&self) -> ToolName { ToolName::plain(WAIT_TOOL_NAME) @@ -54,11 +53,8 @@ impl ToolExecutor for CodeModeWaitHandler { create_wait_tool() } - async fn handle( - &self, - invocation: ToolInvocation, - ) -> Result, FunctionCallError> { - self.handle_call(invocation).await + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(self.handle_call(invocation)) } } diff --git a/codex-rs/core/src/tools/handlers/agent_jobs/report_agent_job_result.rs b/codex-rs/core/src/tools/handlers/agent_jobs/report_agent_job_result.rs index 5f2a615329..579cc25ffb 100644 --- a/codex-rs/core/src/tools/handlers/agent_jobs/report_agent_job_result.rs +++ b/codex-rs/core/src/tools/handlers/agent_jobs/report_agent_job_result.rs @@ -13,7 +13,6 @@ use super::*; pub struct ReportAgentJobResultHandler; -#[async_trait::async_trait] impl ToolExecutor for ReportAgentJobResultHandler { fn tool_name(&self) -> ToolName { ToolName::plain("report_agent_job_result") @@ -23,11 +22,8 @@ impl ToolExecutor for ReportAgentJobResultHandler { create_report_agent_job_result_tool() } - async fn handle( - &self, - invocation: ToolInvocation, - ) -> Result, FunctionCallError> { - self.handle_call(invocation).await + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(self.handle_call(invocation)) } } diff --git a/codex-rs/core/src/tools/handlers/agent_jobs/spawn_agents_on_csv.rs b/codex-rs/core/src/tools/handlers/agent_jobs/spawn_agents_on_csv.rs index 16e1b1f72c..6da517b9cc 100644 --- a/codex-rs/core/src/tools/handlers/agent_jobs/spawn_agents_on_csv.rs +++ b/codex-rs/core/src/tools/handlers/agent_jobs/spawn_agents_on_csv.rs @@ -14,7 +14,6 @@ use super::*; pub struct SpawnAgentsOnCsvHandler; -#[async_trait::async_trait] impl ToolExecutor for SpawnAgentsOnCsvHandler { fn tool_name(&self) -> ToolName { ToolName::plain("spawn_agents_on_csv") @@ -24,11 +23,8 @@ impl ToolExecutor for SpawnAgentsOnCsvHandler { create_spawn_agents_on_csv_tool() } - async fn handle( - &self, - invocation: ToolInvocation, - ) -> Result, FunctionCallError> { - self.handle_call(invocation).await + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(self.handle_call(invocation)) } } diff --git a/codex-rs/core/src/tools/handlers/apply_patch.rs b/codex-rs/core/src/tools/handlers/apply_patch.rs index 6a322ed6ae..70452b6657 100644 --- a/codex-rs/core/src/tools/handlers/apply_patch.rs +++ b/codex-rs/core/src/tools/handlers/apply_patch.rs @@ -305,7 +305,6 @@ async fn effective_patch_permissions( ) } -#[async_trait::async_trait] impl ToolExecutor for ApplyPatchHandler { fn tool_name(&self) -> ToolName { ToolName::plain("apply_patch") @@ -315,11 +314,8 @@ impl ToolExecutor for ApplyPatchHandler { create_apply_patch_freeform_tool(self.multi_environment) } - async fn handle( - &self, - invocation: ToolInvocation, - ) -> Result, FunctionCallError> { - self.handle_call(invocation).await + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(self.handle_call(invocation)) } } diff --git a/codex-rs/core/src/tools/handlers/dynamic.rs b/codex-rs/core/src/tools/handlers/dynamic.rs index e0151cf189..dabcea8e5a 100644 --- a/codex-rs/core/src/tools/handlers/dynamic.rs +++ b/codex-rs/core/src/tools/handlers/dynamic.rs @@ -61,7 +61,6 @@ impl DynamicToolHandler { } } -#[async_trait::async_trait] impl ToolExecutor for DynamicToolHandler { fn tool_name(&self) -> ToolName { self.tool_name.clone() @@ -86,11 +85,8 @@ impl ToolExecutor for DynamicToolHandler { ) } - async fn handle( - &self, - invocation: ToolInvocation, - ) -> Result, FunctionCallError> { - self.handle_call(invocation).await + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(self.handle_call(invocation)) } } diff --git a/codex-rs/core/src/tools/handlers/extension_tools.rs b/codex-rs/core/src/tools/handlers/extension_tools.rs index a94c764a9f..d429b6c1fc 100644 --- a/codex-rs/core/src/tools/handlers/extension_tools.rs +++ b/codex-rs/core/src/tools/handlers/extension_tools.rs @@ -11,13 +11,11 @@ use codex_tools::ToolSpec; use codex_tools::TurnItemEmissionFuture; use codex_tools::TurnItemEmitter; -use crate::function_tool::FunctionCallError; use crate::session::session::Session; use crate::session::turn_context::TurnContext; use crate::stream_events_utils::TurnItemContributorPolicy; use crate::stream_events_utils::finalize_turn_item; use crate::tools::context::ToolInvocation; -use crate::tools::context::ToolOutput; use crate::tools::context::ToolPayload; use crate::tools::registry::CoreToolRuntime; use crate::tools::registry::ToolExecutor; @@ -30,7 +28,6 @@ impl ExtensionToolAdapter { } } -#[async_trait::async_trait] impl ToolExecutor for ExtensionToolAdapter { fn tool_name(&self) -> ToolName { self.0.tool_name() @@ -52,11 +49,8 @@ impl ToolExecutor for ExtensionToolAdapter { self.0.search_info() } - async fn handle( - &self, - invocation: ToolInvocation, - ) -> Result, FunctionCallError> { - self.0.handle(to_extension_call(&invocation).await).await + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(async move { self.0.handle(to_extension_call(&invocation).await).await }) } } @@ -162,7 +156,6 @@ mod tests { struct StubExtensionExecutor; - #[async_trait::async_trait] impl codex_extension_api::ToolExecutor for StubExtensionExecutor { fn tool_name(&self) -> codex_tools::ToolName { codex_tools::ToolName::plain("extension_echo") @@ -187,13 +180,13 @@ mod tests { }) } - async fn handle( - &self, - _call: codex_tools::ToolCall, - ) -> Result, codex_tools::FunctionCallError> { - Ok(Box::new(codex_tools::JsonToolOutput::new( - json!({ "ok": true }), - ))) + fn handle(&self, _call: codex_tools::ToolCall) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(async { + Ok( + Box::new(codex_tools::JsonToolOutput::new(json!({ "ok": true }))) + as Box, + ) + }) } } @@ -201,7 +194,6 @@ mod tests { captured_call: Arc>>, } - #[async_trait::async_trait] impl codex_extension_api::ToolExecutor for CapturingExtensionExecutor { fn tool_name(&self) -> codex_tools::ToolName { codex_tools::ToolName::plain("extension_echo") @@ -218,11 +210,8 @@ mod tests { }) } - async fn handle( - &self, - call: codex_tools::ToolCall, - ) -> Result, codex_tools::FunctionCallError> { - self.handle_call(call).await + fn handle(&self, call: codex_tools::ToolCall) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(self.handle_call(call)) } } @@ -242,9 +231,10 @@ mod tests { call.turn_item_emitter.emit_started(item.clone()).await; call.turn_item_emitter.emit_completed(item).await; *self.captured_call.lock().await = Some(call); - Ok(Box::new(codex_tools::JsonToolOutput::new( - json!({ "ok": true }), - ))) + Ok( + Box::new(codex_tools::JsonToolOutput::new(json!({ "ok": true }))) + as Box, + ) } } @@ -441,7 +431,6 @@ mod tests { ); } - #[async_trait::async_trait] impl codex_extension_api::ToolExecutor for ImageGenerationExtensionExecutor { fn tool_name(&self) -> codex_tools::ToolName { codex_tools::ToolName::namespaced("image_gen", "imagegen") @@ -458,11 +447,8 @@ mod tests { }) } - async fn handle( - &self, - call: codex_tools::ToolCall, - ) -> Result, codex_tools::FunctionCallError> { - self.handle_call(call).await + fn handle(&self, call: codex_tools::ToolCall) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(self.handle_call(call)) } } @@ -493,9 +479,10 @@ mod tests { }, )) .await; - Ok(Box::new(codex_tools::JsonToolOutput::new( - json!({ "ok": true }), - ))) + Ok( + Box::new(codex_tools::JsonToolOutput::new(json!({ "ok": true }))) + as Box, + ) } } diff --git a/codex-rs/core/src/tools/handlers/list_available_plugins_to_install.rs b/codex-rs/core/src/tools/handlers/list_available_plugins_to_install.rs index 2b782e4552..766b0e9b75 100644 --- a/codex-rs/core/src/tools/handlers/list_available_plugins_to_install.rs +++ b/codex-rs/core/src/tools/handlers/list_available_plugins_to_install.rs @@ -54,7 +54,6 @@ impl ListAvailablePluginsToInstallHandler { } } -#[async_trait::async_trait] impl ToolExecutor for ListAvailablePluginsToInstallHandler { fn tool_name(&self) -> ToolName { ToolName::plain(LIST_AVAILABLE_PLUGINS_TO_INSTALL_TOOL_NAME) @@ -68,11 +67,8 @@ impl ToolExecutor for ListAvailablePluginsToInstallHandler { false } - async fn handle( - &self, - invocation: ToolInvocation, - ) -> Result, FunctionCallError> { - self.handle_call(invocation).await + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(self.handle_call(invocation)) } } diff --git a/codex-rs/core/src/tools/handlers/mcp.rs b/codex-rs/core/src/tools/handlers/mcp.rs index d60cdfdd0c..efb32aab94 100644 --- a/codex-rs/core/src/tools/handlers/mcp.rs +++ b/codex-rs/core/src/tools/handlers/mcp.rs @@ -64,7 +64,6 @@ fn ensure_mcp_prefix(name: &str) -> String { } } -#[async_trait::async_trait] impl ToolExecutor for McpHandler { fn tool_name(&self) -> ToolName { self.tool_info.canonical_tool_name() @@ -113,11 +112,8 @@ impl ToolExecutor for McpHandler { ) } - async fn handle( - &self, - invocation: ToolInvocation, - ) -> Result, FunctionCallError> { - self.handle_call(invocation).await + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(self.handle_call(invocation)) } } diff --git a/codex-rs/core/src/tools/handlers/mcp_resource/list_mcp_resource_templates.rs b/codex-rs/core/src/tools/handlers/mcp_resource/list_mcp_resource_templates.rs index 8974d678fd..80b183daba 100644 --- a/codex-rs/core/src/tools/handlers/mcp_resource/list_mcp_resource_templates.rs +++ b/codex-rs/core/src/tools/handlers/mcp_resource/list_mcp_resource_templates.rs @@ -26,7 +26,6 @@ use super::serialize_function_output; pub struct ListMcpResourceTemplatesHandler; -#[async_trait::async_trait] impl ToolExecutor for ListMcpResourceTemplatesHandler { fn tool_name(&self) -> ToolName { ToolName::plain("list_mcp_resource_templates") @@ -40,11 +39,8 @@ impl ToolExecutor for ListMcpResourceTemplatesHandler { true } - async fn handle( - &self, - invocation: ToolInvocation, - ) -> Result, FunctionCallError> { - self.handle_call(invocation).await + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(self.handle_call(invocation)) } } diff --git a/codex-rs/core/src/tools/handlers/mcp_resource/list_mcp_resources.rs b/codex-rs/core/src/tools/handlers/mcp_resource/list_mcp_resources.rs index f3a7b43c28..a77d617b94 100644 --- a/codex-rs/core/src/tools/handlers/mcp_resource/list_mcp_resources.rs +++ b/codex-rs/core/src/tools/handlers/mcp_resource/list_mcp_resources.rs @@ -26,7 +26,6 @@ use super::serialize_function_output; pub struct ListMcpResourcesHandler; -#[async_trait::async_trait] impl ToolExecutor for ListMcpResourcesHandler { fn tool_name(&self) -> ToolName { ToolName::plain("list_mcp_resources") @@ -40,11 +39,8 @@ impl ToolExecutor for ListMcpResourcesHandler { true } - async fn handle( - &self, - invocation: ToolInvocation, - ) -> Result, FunctionCallError> { - self.handle_call(invocation).await + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(self.handle_call(invocation)) } } diff --git a/codex-rs/core/src/tools/handlers/mcp_resource/read_mcp_resource.rs b/codex-rs/core/src/tools/handlers/mcp_resource/read_mcp_resource.rs index 9861e5d22d..9ca15718a0 100644 --- a/codex-rs/core/src/tools/handlers/mcp_resource/read_mcp_resource.rs +++ b/codex-rs/core/src/tools/handlers/mcp_resource/read_mcp_resource.rs @@ -26,7 +26,6 @@ use super::serialize_function_output; pub struct ReadMcpResourceHandler; -#[async_trait::async_trait] impl ToolExecutor for ReadMcpResourceHandler { fn tool_name(&self) -> ToolName { ToolName::plain("read_mcp_resource") @@ -40,11 +39,8 @@ impl ToolExecutor for ReadMcpResourceHandler { true } - async fn handle( - &self, - invocation: ToolInvocation, - ) -> Result, FunctionCallError> { - self.handle_call(invocation).await + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(self.handle_call(invocation)) } } diff --git a/codex-rs/core/src/tools/handlers/multi_agents/close_agent.rs b/codex-rs/core/src/tools/handlers/multi_agents/close_agent.rs index 0d4e17447d..d1017adb65 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents/close_agent.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents/close_agent.rs @@ -6,7 +6,6 @@ use codex_tools::ToolSpec; pub(crate) struct Handler; -#[async_trait::async_trait] impl ToolExecutor for Handler { fn tool_name(&self) -> ToolName { ToolName::namespaced(MULTI_AGENT_V1_NAMESPACE, "close_agent") @@ -23,11 +22,8 @@ impl ToolExecutor for Handler { ) } - async fn handle( - &self, - invocation: ToolInvocation, - ) -> Result, FunctionCallError> { - handle_close_agent(invocation).await.map(boxed_tool_output) + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(async move { handle_close_agent(invocation).await.map(boxed_tool_output) }) } } diff --git a/codex-rs/core/src/tools/handlers/multi_agents/resume_agent.rs b/codex-rs/core/src/tools/handlers/multi_agents/resume_agent.rs index 8f95abe4e3..fa464ddbbb 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents/resume_agent.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents/resume_agent.rs @@ -7,7 +7,6 @@ use std::sync::Arc; pub(crate) struct Handler; -#[async_trait::async_trait] impl ToolExecutor for Handler { fn tool_name(&self) -> ToolName { ToolName::namespaced(MULTI_AGENT_V1_NAMESPACE, "resume_agent") @@ -24,11 +23,8 @@ impl ToolExecutor for Handler { ) } - async fn handle( - &self, - invocation: ToolInvocation, - ) -> Result, FunctionCallError> { - handle_resume_agent(invocation).await.map(boxed_tool_output) + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(async move { handle_resume_agent(invocation).await.map(boxed_tool_output) }) } } diff --git a/codex-rs/core/src/tools/handlers/multi_agents/send_input.rs b/codex-rs/core/src/tools/handlers/multi_agents/send_input.rs index b811faf91a..375bb50ac7 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents/send_input.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents/send_input.rs @@ -6,7 +6,6 @@ use codex_tools::ToolSpec; pub(crate) struct Handler; -#[async_trait::async_trait] impl ToolExecutor for Handler { fn tool_name(&self) -> ToolName { ToolName::namespaced(MULTI_AGENT_V1_NAMESPACE, "send_input") @@ -23,11 +22,8 @@ impl ToolExecutor for Handler { ) } - async fn handle( - &self, - invocation: ToolInvocation, - ) -> Result, FunctionCallError> { - self.handle_call(invocation).await + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(self.handle_call(invocation)) } } diff --git a/codex-rs/core/src/tools/handlers/multi_agents/spawn.rs b/codex-rs/core/src/tools/handlers/multi_agents/spawn.rs index 3f6c97fcbc..204385934e 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents/spawn.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents/spawn.rs @@ -22,7 +22,6 @@ impl Handler { } } -#[async_trait::async_trait] impl ToolExecutor for Handler { fn tool_name(&self) -> ToolName { ToolName::namespaced(MULTI_AGENT_V1_NAMESPACE, "spawn_agent") @@ -39,11 +38,8 @@ impl ToolExecutor for Handler { ) } - async fn handle( - &self, - invocation: ToolInvocation, - ) -> Result, FunctionCallError> { - handle_spawn_agent(invocation).await.map(boxed_tool_output) + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(async move { handle_spawn_agent(invocation).await.map(boxed_tool_output) }) } } diff --git a/codex-rs/core/src/tools/handlers/multi_agents/wait.rs b/codex-rs/core/src/tools/handlers/multi_agents/wait.rs index 1f6b7db8f1..be22b36908 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents/wait.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents/wait.rs @@ -27,7 +27,6 @@ impl Handler { } } -#[async_trait::async_trait] impl ToolExecutor for Handler { fn tool_name(&self) -> ToolName { ToolName::namespaced(MULTI_AGENT_V1_NAMESPACE, "wait_agent") @@ -44,11 +43,8 @@ impl ToolExecutor for Handler { ) } - async fn handle( - &self, - invocation: ToolInvocation, - ) -> Result, FunctionCallError> { - self.handle_call(invocation).await + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(self.handle_call(invocation)) } } diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/followup_task.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/followup_task.rs index 66a39cc79d..bd4dafe7c2 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/followup_task.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/followup_task.rs @@ -7,7 +7,6 @@ use codex_tools::ToolSpec; pub(crate) struct Handler; -#[async_trait::async_trait] impl ToolExecutor for Handler { fn tool_name(&self) -> ToolName { ToolName::plain("followup_task") @@ -17,11 +16,8 @@ impl ToolExecutor for Handler { create_followup_task_tool() } - async fn handle( - &self, - invocation: ToolInvocation, - ) -> Result, FunctionCallError> { - self.handle_call(invocation).await + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(self.handle_call(invocation)) } } diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/interrupt_agent.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/interrupt_agent.rs index 159c49127a..aeed055b4e 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/interrupt_agent.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/interrupt_agent.rs @@ -6,7 +6,6 @@ use codex_tools::ToolSpec; pub(crate) struct Handler; -#[async_trait::async_trait] impl ToolExecutor for Handler { fn tool_name(&self) -> ToolName { ToolName::plain("interrupt_agent") @@ -16,13 +15,12 @@ impl ToolExecutor for Handler { create_interrupt_agent_tool_v2() } - async fn handle( - &self, - invocation: ToolInvocation, - ) -> Result, FunctionCallError> { - handle_interrupt_agent(invocation) - .await - .map(boxed_tool_output) + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(async move { + handle_interrupt_agent(invocation) + .await + .map(boxed_tool_output) + }) } } diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/list_agents.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/list_agents.rs index 18421278fe..99e54e1ce9 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/list_agents.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/list_agents.rs @@ -5,7 +5,6 @@ use codex_tools::ToolSpec; pub(crate) struct Handler; -#[async_trait::async_trait] impl ToolExecutor for Handler { fn tool_name(&self) -> ToolName { ToolName::plain("list_agents") @@ -15,11 +14,8 @@ impl ToolExecutor for Handler { create_list_agents_tool() } - async fn handle( - &self, - invocation: ToolInvocation, - ) -> Result, FunctionCallError> { - self.handle_call(invocation).await + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(self.handle_call(invocation)) } } diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/send_message.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/send_message.rs index 2fd636a85c..717e3bc920 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/send_message.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/send_message.rs @@ -7,7 +7,6 @@ use codex_tools::ToolSpec; pub(crate) struct Handler; -#[async_trait::async_trait] impl ToolExecutor for Handler { fn tool_name(&self) -> ToolName { ToolName::plain("send_message") @@ -17,11 +16,8 @@ impl ToolExecutor for Handler { create_send_message_tool() } - async fn handle( - &self, - invocation: ToolInvocation, - ) -> Result, FunctionCallError> { - self.handle_call(invocation).await + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(self.handle_call(invocation)) } } diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/spawn.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/spawn.rs index 34fddc60b4..2186a4a6d6 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/spawn.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/spawn.rs @@ -22,7 +22,6 @@ impl Handler { } } -#[async_trait::async_trait] impl ToolExecutor for Handler { fn tool_name(&self) -> ToolName { ToolName::plain("spawn_agent") @@ -32,11 +31,8 @@ impl ToolExecutor for Handler { create_spawn_agent_tool_v2(self.options.clone()) } - async fn handle( - &self, - invocation: ToolInvocation, - ) -> Result, FunctionCallError> { - handle_spawn_agent(invocation).await.map(boxed_tool_output) + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(async move { handle_spawn_agent(invocation).await.map(boxed_tool_output) }) } } diff --git a/codex-rs/core/src/tools/handlers/multi_agents_v2/wait.rs b/codex-rs/core/src/tools/handlers/multi_agents_v2/wait.rs index 9d671b1989..36e5aa24f6 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_v2/wait.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_v2/wait.rs @@ -19,7 +19,6 @@ impl Handler { } } -#[async_trait::async_trait] impl ToolExecutor for Handler { fn tool_name(&self) -> ToolName { ToolName::plain("wait_agent") @@ -29,11 +28,8 @@ impl ToolExecutor for Handler { create_wait_agent_tool_v2(self.options) } - async fn handle( - &self, - invocation: ToolInvocation, - ) -> Result, FunctionCallError> { - self.handle_call(invocation).await + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(self.handle_call(invocation)) } } diff --git a/codex-rs/core/src/tools/handlers/plan.rs b/codex-rs/core/src/tools/handlers/plan.rs index ff18a32813..1b30c281ce 100644 --- a/codex-rs/core/src/tools/handlers/plan.rs +++ b/codex-rs/core/src/tools/handlers/plan.rs @@ -45,7 +45,6 @@ impl ToolOutput for PlanToolOutput { } } -#[async_trait::async_trait] impl ToolExecutor for PlanHandler { fn tool_name(&self) -> ToolName { ToolName::plain("update_plan") @@ -55,11 +54,8 @@ impl ToolExecutor for PlanHandler { create_update_plan_tool() } - async fn handle( - &self, - invocation: ToolInvocation, - ) -> Result, FunctionCallError> { - self.handle_call(invocation).await + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(self.handle_call(invocation)) } } diff --git a/codex-rs/core/src/tools/handlers/request_permissions.rs b/codex-rs/core/src/tools/handlers/request_permissions.rs index 7c24f15697..4faf9d328b 100644 --- a/codex-rs/core/src/tools/handlers/request_permissions.rs +++ b/codex-rs/core/src/tools/handlers/request_permissions.rs @@ -25,7 +25,6 @@ struct RequestPermissionsEnvironmentArgs { environment_id: Option, } -#[async_trait::async_trait] impl ToolExecutor for RequestPermissionsHandler { fn tool_name(&self) -> ToolName { ToolName::plain("request_permissions") @@ -35,11 +34,8 @@ impl ToolExecutor for RequestPermissionsHandler { create_request_permissions_tool(request_permissions_tool_description()) } - async fn handle( - &self, - invocation: ToolInvocation, - ) -> Result, FunctionCallError> { - self.handle_call(invocation).await + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(self.handle_call(invocation)) } } diff --git a/codex-rs/core/src/tools/handlers/request_plugin_install.rs b/codex-rs/core/src/tools/handlers/request_plugin_install.rs index 64625aaac0..801031348f 100644 --- a/codex-rs/core/src/tools/handlers/request_plugin_install.rs +++ b/codex-rs/core/src/tools/handlers/request_plugin_install.rs @@ -48,7 +48,6 @@ impl RequestPluginInstallHandler { } } -#[async_trait::async_trait] impl ToolExecutor for RequestPluginInstallHandler { fn tool_name(&self) -> ToolName { ToolName::plain(REQUEST_PLUGIN_INSTALL_TOOL_NAME) @@ -62,11 +61,8 @@ impl ToolExecutor for RequestPluginInstallHandler { true } - async fn handle( - &self, - invocation: ToolInvocation, - ) -> Result, FunctionCallError> { - self.handle_call(invocation).await + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(self.handle_call(invocation)) } } diff --git a/codex-rs/core/src/tools/handlers/request_user_input.rs b/codex-rs/core/src/tools/handlers/request_user_input.rs index 3f3adf0ba3..4f26865a68 100644 --- a/codex-rs/core/src/tools/handlers/request_user_input.rs +++ b/codex-rs/core/src/tools/handlers/request_user_input.rs @@ -20,7 +20,6 @@ pub struct RequestUserInputHandler { pub available_modes: Vec, } -#[async_trait::async_trait] impl ToolExecutor for RequestUserInputHandler { fn tool_name(&self) -> ToolName { ToolName::plain(REQUEST_USER_INPUT_TOOL_NAME) @@ -30,11 +29,8 @@ impl ToolExecutor for RequestUserInputHandler { create_request_user_input_tool(request_user_input_tool_description(&self.available_modes)) } - async fn handle( - &self, - invocation: ToolInvocation, - ) -> Result, FunctionCallError> { - self.handle_call(invocation).await + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(self.handle_call(invocation)) } } diff --git a/codex-rs/core/src/tools/handlers/shell/shell_command.rs b/codex-rs/core/src/tools/handlers/shell/shell_command.rs index e9fcf82046..b754187d4a 100644 --- a/codex-rs/core/src/tools/handlers/shell/shell_command.rs +++ b/codex-rs/core/src/tools/handlers/shell/shell_command.rs @@ -124,7 +124,6 @@ impl From for ShellCommandHandler { } } -#[async_trait::async_trait] impl ToolExecutor for ShellCommandHandler { fn tool_name(&self) -> ToolName { ToolName::plain("shell_command") @@ -141,11 +140,8 @@ impl ToolExecutor for ShellCommandHandler { true } - async fn handle( - &self, - invocation: ToolInvocation, - ) -> Result, FunctionCallError> { - self.handle_call(invocation).await + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(self.handle_call(invocation)) } } diff --git a/codex-rs/core/src/tools/handlers/test_sync.rs b/codex-rs/core/src/tools/handlers/test_sync.rs index 802859c916..08d983b616 100644 --- a/codex-rs/core/src/tools/handlers/test_sync.rs +++ b/codex-rs/core/src/tools/handlers/test_sync.rs @@ -57,7 +57,6 @@ fn barrier_map() -> &'static tokio::sync::Mutex> { BARRIERS.get_or_init(|| tokio::sync::Mutex::new(HashMap::new())) } -#[async_trait::async_trait] impl ToolExecutor for TestSyncHandler { fn tool_name(&self) -> ToolName { ToolName::plain("test_sync_tool") @@ -71,11 +70,8 @@ impl ToolExecutor for TestSyncHandler { true } - async fn handle( - &self, - invocation: ToolInvocation, - ) -> Result, FunctionCallError> { - self.handle_call(invocation).await + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(self.handle_call(invocation)) } } diff --git a/codex-rs/core/src/tools/handlers/tool_search.rs b/codex-rs/core/src/tools/handlers/tool_search.rs index a4781e75e2..9254f4bee0 100644 --- a/codex-rs/core/src/tools/handlers/tool_search.rs +++ b/codex-rs/core/src/tools/handlers/tool_search.rs @@ -53,7 +53,6 @@ impl ToolSearchHandler { } } -#[async_trait::async_trait] impl ToolExecutor for ToolSearchHandler { fn tool_name(&self) -> ToolName { ToolName::plain(TOOL_SEARCH_TOOL_NAME) @@ -67,11 +66,8 @@ impl ToolExecutor for ToolSearchHandler { true } - async fn handle( - &self, - invocation: ToolInvocation, - ) -> Result, FunctionCallError> { - self.handle_call(invocation).await + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(self.handle_call(invocation)) } } diff --git a/codex-rs/core/src/tools/handlers/unified_exec/exec_command.rs b/codex-rs/core/src/tools/handlers/unified_exec/exec_command.rs index 9fe3b343b5..7c914deb4f 100644 --- a/codex-rs/core/src/tools/handlers/unified_exec/exec_command.rs +++ b/codex-rs/core/src/tools/handlers/unified_exec/exec_command.rs @@ -71,7 +71,6 @@ impl ExecCommandHandler { } } -#[async_trait::async_trait] impl ToolExecutor for ExecCommandHandler { fn tool_name(&self) -> ToolName { ToolName::plain("exec_command") @@ -92,11 +91,8 @@ impl ToolExecutor for ExecCommandHandler { true } - async fn handle( - &self, - invocation: ToolInvocation, - ) -> Result, FunctionCallError> { - self.handle_call(invocation).await + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(self.handle_call(invocation)) } } diff --git a/codex-rs/core/src/tools/handlers/unified_exec/write_stdin.rs b/codex-rs/core/src/tools/handlers/unified_exec/write_stdin.rs index 9b319d5433..8ddce65840 100644 --- a/codex-rs/core/src/tools/handlers/unified_exec/write_stdin.rs +++ b/codex-rs/core/src/tools/handlers/unified_exec/write_stdin.rs @@ -31,7 +31,6 @@ struct WriteStdinArgs { pub struct WriteStdinHandler; -#[async_trait::async_trait] impl ToolExecutor for WriteStdinHandler { fn tool_name(&self) -> ToolName { ToolName::plain("write_stdin") @@ -41,11 +40,8 @@ impl ToolExecutor for WriteStdinHandler { create_write_stdin_tool() } - async fn handle( - &self, - invocation: ToolInvocation, - ) -> Result, FunctionCallError> { - self.handle_call(invocation).await + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(self.handle_call(invocation)) } } diff --git a/codex-rs/core/src/tools/handlers/view_image.rs b/codex-rs/core/src/tools/handlers/view_image.rs index eb761359a7..0432ad338f 100644 --- a/codex-rs/core/src/tools/handlers/view_image.rs +++ b/codex-rs/core/src/tools/handlers/view_image.rs @@ -64,7 +64,6 @@ enum ViewImageDetail { Original, } -#[async_trait::async_trait] impl ToolExecutor for ViewImageHandler { fn tool_name(&self) -> ToolName { ToolName::plain("view_image") @@ -78,11 +77,8 @@ impl ToolExecutor for ViewImageHandler { true } - async fn handle( - &self, - invocation: ToolInvocation, - ) -> Result, FunctionCallError> { - self.handle_call(invocation).await + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(self.handle_call(invocation)) } } diff --git a/codex-rs/core/src/tools/parallel.rs b/codex-rs/core/src/tools/parallel.rs index a6f630a342..c40886b56c 100644 --- a/codex-rs/core/src/tools/parallel.rs +++ b/codex-rs/core/src/tools/parallel.rs @@ -257,7 +257,6 @@ mod tests { tool_name: codex_tools::ToolName, } - #[async_trait::async_trait] impl ToolExecutor for ImmediateHandler { fn tool_name(&self) -> codex_tools::ToolName { self.tool_name.clone() @@ -274,14 +273,13 @@ mod tests { }) } - async fn handle( - &self, - _invocation: ToolInvocation, - ) -> Result, FunctionCallError> { - Ok(Box::new(FunctionToolOutput::from_text( - "ok".to_string(), - Some(true), - ))) + fn handle(&self, _invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(async { + Ok( + Box::new(FunctionToolOutput::from_text("ok".to_string(), Some(true))) + as Box, + ) + }) } } @@ -294,7 +292,6 @@ mod tests { allow_cleanup: Arc, } - #[async_trait::async_trait] impl ToolExecutor for CancellationCleanupHandler { fn tool_name(&self) -> codex_tools::ToolName { self.tool_name.clone() @@ -311,11 +308,8 @@ mod tests { }) } - async fn handle( - &self, - invocation: ToolInvocation, - ) -> Result, FunctionCallError> { - self.handle_call(invocation).await + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(self.handle_call(invocation)) } } @@ -345,7 +339,7 @@ mod tests { Ok(Box::new(FunctionToolOutput::from_text( "cleanup complete".to_string(), Some(false), - ))) + )) as Box) } } diff --git a/codex-rs/core/src/tools/registry.rs b/codex-rs/core/src/tools/registry.rs index 20e08e9f7d..76e3f5f341 100644 --- a/codex-rs/core/src/tools/registry.rs +++ b/codex-rs/core/src/tools/registry.rs @@ -250,7 +250,6 @@ struct ExposureOverride { exposure: ToolExposure, } -#[async_trait::async_trait] impl ToolExecutor for ExposureOverride { fn tool_name(&self) -> ToolName { self.handler.tool_name() @@ -272,11 +271,8 @@ impl ToolExecutor for ExposureOverride { self.handler.search_info() } - async fn handle( - &self, - invocation: ToolInvocation, - ) -> Result, FunctionCallError> { - self.handler.handle(invocation).await + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + self.handler.handle(invocation) } } diff --git a/codex-rs/core/src/tools/registry_tests.rs b/codex-rs/core/src/tools/registry_tests.rs index cc7f39e2a3..67613f1d08 100644 --- a/codex-rs/core/src/tools/registry_tests.rs +++ b/codex-rs/core/src/tools/registry_tests.rs @@ -5,7 +5,6 @@ struct TestHandler { tool_name: codex_tools::ToolName, } -#[async_trait::async_trait] impl ToolExecutor for TestHandler { fn tool_name(&self) -> codex_tools::ToolName { self.tool_name.clone() @@ -15,13 +14,15 @@ impl ToolExecutor for TestHandler { test_spec(&self.tool_name) } - async fn handle( - &self, - _invocation: ToolInvocation, - ) -> Result, FunctionCallError> { - Ok(Box::new( - crate::tools::context::FunctionToolOutput::from_text("ok".to_string(), Some(true)), - )) + fn handle(&self, _invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(async { + Ok( + Box::new(crate::tools::context::FunctionToolOutput::from_text( + "ok".to_string(), + Some(true), + )) as Box, + ) + }) } } @@ -38,7 +39,6 @@ struct LifecycleTestHandler { result: LifecycleTestResult, } -#[async_trait::async_trait] impl ToolExecutor for LifecycleTestHandler { fn tool_name(&self) -> codex_tools::ToolName { self.tool_name.clone() @@ -48,11 +48,8 @@ impl ToolExecutor for LifecycleTestHandler { test_spec(&self.tool_name) } - async fn handle( - &self, - _invocation: ToolInvocation, - ) -> Result, FunctionCallError> { - self.handle_call().await + fn handle(&self, _invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(self.handle_call()) } } @@ -66,7 +63,8 @@ impl LifecycleTestHandler { "ok".to_string(), Some(success), ), - )), + ) + as Box), LifecycleTestResult::Err => Err(FunctionCallError::RespondToModel( "handler failed".to_string(), )), diff --git a/codex-rs/core/src/tools/router_tests.rs b/codex-rs/core/src/tools/router_tests.rs index a9a52ca463..4024cb61f7 100644 --- a/codex-rs/core/src/tools/router_tests.rs +++ b/codex-rs/core/src/tools/router_tests.rs @@ -44,7 +44,6 @@ impl codex_extension_api::ToolContributor for ExtensionEchoContributor { struct ExtensionEchoExecutor; -#[async_trait::async_trait] impl ToolExecutor for ExtensionEchoExecutor { fn tool_name(&self) -> ToolName { ToolName::namespaced("extension/", "echo") @@ -73,11 +72,8 @@ impl ToolExecutor for ExtensionEchoExecutor { }) } - async fn handle( - &self, - call: ExtensionToolCall, - ) -> Result, codex_tools::FunctionCallError> { - self.handle_call(call).await + fn handle(&self, call: ExtensionToolCall) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(self.handle_call(call)) } } @@ -93,7 +89,7 @@ impl ExtensionEchoExecutor { "callId": call.call_id, "conversationHistory": call.conversation_history.items(), "ok": true, - })))) + }))) as Box) } } diff --git a/codex-rs/core/src/tools/spec_plan.rs b/codex-rs/core/src/tools/spec_plan.rs index 19f4c9f897..697eba1d4a 100644 --- a/codex-rs/core/src/tools/spec_plan.rs +++ b/codex-rs/core/src/tools/spec_plan.rs @@ -72,7 +72,6 @@ use codex_tools::ToolCall as ExtensionToolCall; use codex_tools::ToolEnvironmentMode; use codex_tools::ToolExecutor; use codex_tools::ToolName; -use codex_tools::ToolOutput; use codex_tools::ToolSearchInfo; use codex_tools::ToolSpec; use codex_tools::UnifiedExecShellMode; @@ -946,7 +945,6 @@ struct MultiAgentV2NamespaceOverride { namespace: String, } -#[async_trait::async_trait] impl ToolExecutor for MultiAgentV2NamespaceOverride { fn tool_name(&self) -> ToolName { ToolName::namespaced(self.namespace.clone(), self.handler.tool_name().name) @@ -975,11 +973,8 @@ impl ToolExecutor for MultiAgentV2NamespaceOverride { self.handler.search_info() } - async fn handle( - &self, - invocation: ToolInvocation, - ) -> Result, codex_tools::FunctionCallError> { - self.handler.handle(invocation).await + fn handle(&self, invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + self.handler.handle(invocation) } } diff --git a/codex-rs/core/src/tools/spec_plan_tests.rs b/codex-rs/core/src/tools/spec_plan_tests.rs index f96f41f0f2..a9eb5012d5 100644 --- a/codex-rs/core/src/tools/spec_plan_tests.rs +++ b/codex-rs/core/src/tools/spec_plan_tests.rs @@ -285,7 +285,6 @@ fn use_bedrock_provider(turn: &mut TurnContext) { struct WebRunExtensionTool; -#[async_trait::async_trait] impl ToolExecutor for WebRunExtensionTool { fn tool_name(&self) -> ToolName { ToolName::namespaced("web", "run") @@ -306,17 +305,15 @@ impl ToolExecutor for WebRunExtensionTool { }) } - async fn handle( - &self, - _call: ExtensionToolCall, - ) -> Result, codex_tools::FunctionCallError> { - Ok(Box::new(codex_tools::JsonToolOutput::new(json!({})))) + fn handle(&self, _call: ExtensionToolCall) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(async { + Ok(Box::new(codex_tools::JsonToolOutput::new(json!({}))) as Box) + }) } } struct DeferredExtensionTool; -#[async_trait::async_trait] impl ToolExecutor for DeferredExtensionTool { fn tool_name(&self) -> ToolName { ToolName::plain("extension_echo") @@ -344,11 +341,8 @@ impl ToolExecutor for DeferredExtensionTool { ToolExposure::Deferred } - async fn handle( - &self, - _call: ExtensionToolCall, - ) -> Result, codex_tools::FunctionCallError> { - panic!("spec planning should not execute extension tools") + fn handle(&self, _call: ExtensionToolCall) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(async { panic!("spec planning should not execute extension tools") }) } } diff --git a/codex-rs/core/src/tools/tool_dispatch_trace_tests.rs b/codex-rs/core/src/tools/tool_dispatch_trace_tests.rs index 69dc8ab580..b09996872e 100644 --- a/codex-rs/core/src/tools/tool_dispatch_trace_tests.rs +++ b/codex-rs/core/src/tools/tool_dispatch_trace_tests.rs @@ -30,7 +30,6 @@ struct TestHandler { tool_name: codex_tools::ToolName, } -#[async_trait::async_trait] impl ToolExecutor for TestHandler { fn tool_name(&self) -> codex_tools::ToolName { self.tool_name.clone() @@ -47,14 +46,13 @@ impl ToolExecutor for TestHandler { }) } - async fn handle( - &self, - _invocation: ToolInvocation, - ) -> Result, FunctionCallError> { - Ok(Box::new(FunctionToolOutput::from_text( - "ok".to_string(), - Some(true), - ))) + fn handle(&self, _invocation: ToolInvocation) -> codex_tools::ToolExecutorFuture<'_> { + Box::pin(async { + Ok( + Box::new(FunctionToolOutput::from_text("ok".to_string(), Some(true))) + as Box, + ) + }) } } diff --git a/codex-rs/ext/extension-api/src/lib.rs b/codex-rs/ext/extension-api/src/lib.rs index dc3495a71c..52f15c894c 100644 --- a/codex-rs/ext/extension-api/src/lib.rs +++ b/codex-rs/ext/extension-api/src/lib.rs @@ -20,6 +20,7 @@ pub use codex_tools::NoopTurnItemEmitter; pub use codex_tools::ResponsesApiTool; pub use codex_tools::ToolCall; pub use codex_tools::ToolExecutor; +pub use codex_tools::ToolExecutorFuture; pub use codex_tools::ToolName; pub use codex_tools::ToolOutput; pub use codex_tools::ToolPayload; diff --git a/codex-rs/ext/goal/Cargo.toml b/codex-rs/ext/goal/Cargo.toml index 1e21f5ac88..4d75ba2f26 100644 --- a/codex-rs/ext/goal/Cargo.toml +++ b/codex-rs/ext/goal/Cargo.toml @@ -14,7 +14,6 @@ doctest = false workspace = true [dependencies] -async-trait = { workspace = true } codex-analytics = { workspace = true } codex-core = { workspace = true } codex-extension-api = { workspace = true } diff --git a/codex-rs/ext/goal/src/tool.rs b/codex-rs/ext/goal/src/tool.rs index c45ed81b0a..d280f6e04b 100644 --- a/codex-rs/ext/goal/src/tool.rs +++ b/codex-rs/ext/goal/src/tool.rs @@ -1,6 +1,5 @@ use std::sync::Arc; -use async_trait::async_trait; use codex_extension_api::FunctionCallError; use codex_extension_api::JsonToolOutput; use codex_extension_api::ToolCall; @@ -132,7 +131,6 @@ impl GoalToolExecutor { } } -#[async_trait] impl ToolExecutor for GoalToolExecutor { fn tool_name(&self) -> ToolName { ToolName::plain(match self.kind { @@ -150,12 +148,14 @@ impl ToolExecutor for GoalToolExecutor { } } - async fn handle(&self, invocation: ToolCall) -> Result, FunctionCallError> { - match self.kind { - GoalToolKind::Get => self.handle_get(invocation).await, - GoalToolKind::Create => self.handle_create(invocation).await, - GoalToolKind::Update => self.handle_update(invocation).await, - } + fn handle(&self, invocation: ToolCall) -> codex_extension_api::ToolExecutorFuture<'_> { + Box::pin(async move { + match self.kind { + GoalToolKind::Get => self.handle_get(invocation).await, + GoalToolKind::Create => self.handle_create(invocation).await, + GoalToolKind::Update => self.handle_update(invocation).await, + } + }) } } diff --git a/codex-rs/ext/image-generation/Cargo.toml b/codex-rs/ext/image-generation/Cargo.toml index cb5ddda0a8..dfa85ee82e 100644 --- a/codex-rs/ext/image-generation/Cargo.toml +++ b/codex-rs/ext/image-generation/Cargo.toml @@ -13,7 +13,6 @@ doctest = false workspace = true [dependencies] -async-trait = { workspace = true } codex-api = { workspace = true } codex-core = { workspace = true } codex-extension-api = { workspace = true } diff --git a/codex-rs/ext/image-generation/src/tool.rs b/codex-rs/ext/image-generation/src/tool.rs index 70880b4661..07ed8f4bf6 100644 --- a/codex-rs/ext/image-generation/src/tool.rs +++ b/codex-rs/ext/image-generation/src/tool.rs @@ -78,7 +78,6 @@ struct ImagegenArgs { num_last_images_to_include: Option, } -#[async_trait::async_trait] impl ToolExecutor for ImageGenerationTool { /// Keeps the tool in the existing image-generation Responses namespace. fn tool_name(&self) -> ToolName { @@ -96,8 +95,8 @@ impl ToolExecutor for ImageGenerationTool { } /// Executes the selected image operation and returns the completed image result. - async fn handle(&self, call: ToolCall) -> Result, FunctionCallError> { - self.handle_call(call).await + fn handle(&self, call: ToolCall) -> codex_extension_api::ToolExecutorFuture<'_> { + Box::pin(self.handle_call(call)) } } diff --git a/codex-rs/ext/memories/Cargo.toml b/codex-rs/ext/memories/Cargo.toml index 68eeaf8ec1..e283140b15 100644 --- a/codex-rs/ext/memories/Cargo.toml +++ b/codex-rs/ext/memories/Cargo.toml @@ -13,7 +13,6 @@ doctest = false workspace = true [dependencies] -async-trait = { workspace = true } codex-core = { workspace = true } codex-extension-api = { workspace = true } codex-features = { workspace = true } diff --git a/codex-rs/ext/memories/src/tools/ad_hoc_note.rs b/codex-rs/ext/memories/src/tools/ad_hoc_note.rs index 3495d57d32..0ce9cade67 100644 --- a/codex-rs/ext/memories/src/tools/ad_hoc_note.rs +++ b/codex-rs/ext/memories/src/tools/ad_hoc_note.rs @@ -41,7 +41,6 @@ pub(super) struct AddAdHocNoteTool { pub(super) metrics_client: Option, } -#[async_trait::async_trait] impl ToolExecutor for AddAdHocNoteTool where B: MemoriesBackend, @@ -57,12 +56,8 @@ where ) } - async fn handle( - &self, - call: ToolCall, - ) -> Result, codex_extension_api::FunctionCallError> - { - self.handle_call(call).await + fn handle(&self, call: ToolCall) -> codex_extension_api::ToolExecutorFuture<'_> { + Box::pin(self.handle_call(call)) } } diff --git a/codex-rs/ext/memories/src/tools/list.rs b/codex-rs/ext/memories/src/tools/list.rs index d3b23b1ca2..b7b9ad0586 100644 --- a/codex-rs/ext/memories/src/tools/list.rs +++ b/codex-rs/ext/memories/src/tools/list.rs @@ -39,7 +39,6 @@ pub(super) struct ListTool { pub(super) metrics_client: Option, } -#[async_trait::async_trait] impl ToolExecutor for ListTool where B: MemoriesBackend, @@ -55,12 +54,8 @@ where ) } - async fn handle( - &self, - call: ToolCall, - ) -> Result, codex_extension_api::FunctionCallError> - { - self.handle_call(call).await + fn handle(&self, call: ToolCall) -> codex_extension_api::ToolExecutorFuture<'_> { + Box::pin(self.handle_call(call)) } } diff --git a/codex-rs/ext/memories/src/tools/read.rs b/codex-rs/ext/memories/src/tools/read.rs index e408a17cad..6625cdaf8b 100644 --- a/codex-rs/ext/memories/src/tools/read.rs +++ b/codex-rs/ext/memories/src/tools/read.rs @@ -38,7 +38,6 @@ pub(super) struct ReadTool { pub(super) metrics_client: Option, } -#[async_trait::async_trait] impl ToolExecutor for ReadTool where B: MemoriesBackend, @@ -54,12 +53,8 @@ where ) } - async fn handle( - &self, - call: ToolCall, - ) -> Result, codex_extension_api::FunctionCallError> - { - self.handle_call(call).await + fn handle(&self, call: ToolCall) -> codex_extension_api::ToolExecutorFuture<'_> { + Box::pin(self.handle_call(call)) } } diff --git a/codex-rs/ext/memories/src/tools/search.rs b/codex-rs/ext/memories/src/tools/search.rs index cc8ba201a6..928c98b475 100644 --- a/codex-rs/ext/memories/src/tools/search.rs +++ b/codex-rs/ext/memories/src/tools/search.rs @@ -47,7 +47,6 @@ pub(super) struct SearchTool { pub(super) metrics_client: Option, } -#[async_trait::async_trait] impl ToolExecutor for SearchTool where B: MemoriesBackend, @@ -63,12 +62,8 @@ where ) } - async fn handle( - &self, - call: ToolCall, - ) -> Result, codex_extension_api::FunctionCallError> - { - self.handle_call(call).await + fn handle(&self, call: ToolCall) -> codex_extension_api::ToolExecutorFuture<'_> { + Box::pin(self.handle_call(call)) } } diff --git a/codex-rs/ext/web-search/Cargo.toml b/codex-rs/ext/web-search/Cargo.toml index 954ac3dac9..d007d6146d 100644 --- a/codex-rs/ext/web-search/Cargo.toml +++ b/codex-rs/ext/web-search/Cargo.toml @@ -13,7 +13,6 @@ doctest = false workspace = true [dependencies] -async-trait = { workspace = true } codex-api = { workspace = true } codex-core = { workspace = true } codex-extension-api = { workspace = true } diff --git a/codex-rs/ext/web-search/src/tool.rs b/codex-rs/ext/web-search/src/tool.rs index 1de65edf86..9b8dbf56d6 100644 --- a/codex-rs/ext/web-search/src/tool.rs +++ b/codex-rs/ext/web-search/src/tool.rs @@ -39,7 +39,6 @@ pub(crate) struct WebSearchTool { pub(crate) settings: SearchSettings, } -#[async_trait::async_trait] impl ToolExecutor for WebSearchTool { fn tool_name(&self) -> ToolName { ToolName::namespaced(WEB_NAMESPACE, RUN_TOOL_NAME) @@ -74,8 +73,8 @@ impl ToolExecutor for WebSearchTool { true } - async fn handle(&self, call: ToolCall) -> Result, FunctionCallError> { - self.handle_call(call).await + fn handle(&self, call: ToolCall) -> codex_extension_api::ToolExecutorFuture<'_> { + Box::pin(self.handle_call(call)) } } diff --git a/codex-rs/tools/Cargo.toml b/codex-rs/tools/Cargo.toml index 7cc0e34845..4b2a3257f3 100644 --- a/codex-rs/tools/Cargo.toml +++ b/codex-rs/tools/Cargo.toml @@ -8,7 +8,6 @@ version.workspace = true workspace = true [dependencies] -async-trait = { workspace = true } codex-app-server-protocol = { workspace = true } codex-code-mode = { workspace = true } codex-features = { workspace = true } diff --git a/codex-rs/tools/src/lib.rs b/codex-rs/tools/src/lib.rs index 1cee3f7d2d..848bab3e1c 100644 --- a/codex-rs/tools/src/lib.rs +++ b/codex-rs/tools/src/lib.rs @@ -93,6 +93,7 @@ pub use tool_discovery::ToolSearchSourceInfo; pub use tool_discovery::collect_request_plugin_install_entries; pub use tool_discovery::filter_request_plugin_install_discoverable_tools_for_client; pub use tool_executor::ToolExecutor; +pub use tool_executor::ToolExecutorFuture; pub use tool_executor::ToolExposure; pub use tool_output::JsonToolOutput; pub use tool_output::ToolOutput; diff --git a/codex-rs/tools/src/tool_executor.rs b/codex-rs/tools/src/tool_executor.rs index 541d2443e0..708ed1c3b1 100644 --- a/codex-rs/tools/src/tool_executor.rs +++ b/codex-rs/tools/src/tool_executor.rs @@ -3,6 +3,12 @@ use crate::ToolName; use crate::ToolOutput; use crate::ToolSearchInfo; use crate::ToolSpec; +use std::future::Future; +use std::pin::Pin; + +/// The boxed future returned by [`ToolExecutor::handle`]. +pub type ToolExecutorFuture<'a> = + Pin, FunctionCallError>> + Send + 'a>>; /// Controls where a tool is exposed to the model. #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -40,7 +46,6 @@ impl ToolExposure { /// Implementations keep the model-visible spec tied to the executable runtime. /// Host crates can layer routing, hooks, telemetry, or other orchestration on /// top without reopening the spec/runtime split. -#[async_trait::async_trait] pub trait ToolExecutor: Send + Sync { /// The concrete tool name handled by this runtime instance. fn tool_name(&self) -> ToolName; @@ -60,8 +65,5 @@ pub trait ToolExecutor: Send + Sync { false } - async fn handle( - &self, - invocation: Invocation, - ) -> Result, FunctionCallError>; + fn handle(&self, invocation: Invocation) -> ToolExecutorFuture<'_>; } From a1a8807e9d67fad4b95f2730a9669eca5a9d27d0 Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Wed, 10 Jun 2026 11:18:09 -0700 Subject: [PATCH 40/47] Add app-server background terminal process APIs (#26041) ## Summary Codex Apps needs app-server as the source of truth for chat-started background terminals instead of guessing from local process trees. This PR adds experimental v2 APIs to list and terminate background terminals for a loaded thread using app-server process ids, so clients can manage background terminals without local PID discovery. ## Changes - `thread/backgroundTerminals/list` returns paginated background terminal records with `itemId`, app-server `processId`, `command`, `cwd`, nullable `osPid`, nullable `cpuPercent`, and nullable `rssKb`. - `thread/backgroundTerminals/terminate` terminates one running background terminal by app-server `processId` and returns whether a process was terminated. - Background terminal list and terminate operations use unified-exec process manager state as their source of truth. --- codex-rs/app-server-protocol/src/export.rs | 7 +- .../src/protocol/common.rs | 60 ++++ .../src/protocol/v2/thread.rs | 51 ++++ codex-rs/app-server/README.md | 28 ++ codex-rs/app-server/src/message_processor.rs | 10 + codex-rs/app-server/src/request_processors.rs | 5 + .../request_processors/thread_processor.rs | 94 +++++++ .../thread_processor_tests.rs | 59 ++++ codex-rs/core/src/codex_thread.rs | 19 ++ codex-rs/core/src/lib.rs | 1 + codex-rs/core/src/tasks/mod.rs | 12 + codex-rs/core/src/unified_exec/mod.rs | 2 + codex-rs/core/src/unified_exec/mod_tests.rs | 260 ++++++++++++++++++ codex-rs/core/src/unified_exec/process.rs | 27 +- .../core/src/unified_exec/process_manager.rs | 131 +++++++-- .../core/src/unified_exec/process_tests.rs | 43 ++- 16 files changed, 773 insertions(+), 36 deletions(-) diff --git a/codex-rs/app-server-protocol/src/export.rs b/codex-rs/app-server-protocol/src/export.rs index cd6c7e47b6..5e6c2ad015 100644 --- a/codex-rs/app-server-protocol/src/export.rs +++ b/codex-rs/app-server-protocol/src/export.rs @@ -39,8 +39,11 @@ use ts_rs::TS; pub(crate) const GENERATED_TS_HEADER: &str = "// GENERATED CODE! DO NOT MODIFY BY HAND!\n\n"; const IGNORED_DEFINITIONS: &[&str] = &["Option<()>"]; const JSON_V1_ALLOWLIST: &[&str] = &["InitializeParams", "InitializeResponse"]; -const EXPERIMENTAL_CLIENT_METHOD_DEPENDENCY_TYPES: &[&str] = - &["RemoteControlClient", "RemoteControlClientsListOrder"]; +const EXPERIMENTAL_CLIENT_METHOD_DEPENDENCY_TYPES: &[&str] = &[ + "RemoteControlClient", + "RemoteControlClientsListOrder", + "ThreadBackgroundTerminal", +]; const SPECIAL_DEFINITIONS: &[&str] = &[ "ClientNotification", "ClientRequest", diff --git a/codex-rs/app-server-protocol/src/protocol/common.rs b/codex-rs/app-server-protocol/src/protocol/common.rs index d3245091a9..8bd9a2c175 100644 --- a/codex-rs/app-server-protocol/src/protocol/common.rs +++ b/codex-rs/app-server-protocol/src/protocol/common.rs @@ -574,6 +574,18 @@ client_request_definitions! { serialization: thread_id(params.thread_id), response: v2::ThreadBackgroundTerminalsCleanResponse, }, + #[experimental("thread/backgroundTerminals/list")] + ThreadBackgroundTerminalsList => "thread/backgroundTerminals/list" { + params: v2::ThreadBackgroundTerminalsListParams, + serialization: thread_id(params.thread_id), + response: v2::ThreadBackgroundTerminalsListResponse, + }, + #[experimental("thread/backgroundTerminals/terminate")] + ThreadBackgroundTerminalsTerminate => "thread/backgroundTerminals/terminate" { + params: v2::ThreadBackgroundTerminalsTerminateParams, + serialization: thread_id(params.thread_id), + response: v2::ThreadBackgroundTerminalsTerminateResponse, + }, ThreadRollback => "thread/rollback" { params: v2::ThreadRollbackParams, serialization: thread_id(params.thread_id), @@ -2936,6 +2948,54 @@ mod tests { Ok(()) } + #[test] + fn serialize_thread_background_terminals_list() -> Result<()> { + let request = ClientRequest::ThreadBackgroundTerminalsList { + request_id: RequestId::Integer(8), + params: v2::ThreadBackgroundTerminalsListParams { + thread_id: "thr_123".to_string(), + cursor: None, + limit: None, + }, + }; + assert_eq!( + json!({ + "method": "thread/backgroundTerminals/list", + "id": 8, + "params": { + "threadId": "thr_123", + "cursor": null, + "limit": null + } + }), + serde_json::to_value(&request)?, + ); + Ok(()) + } + + #[test] + fn serialize_thread_background_terminals_terminate() -> Result<()> { + let request = ClientRequest::ThreadBackgroundTerminalsTerminate { + request_id: RequestId::Integer(8), + params: v2::ThreadBackgroundTerminalsTerminateParams { + thread_id: "thr_123".to_string(), + process_id: "42".to_string(), + }, + }; + assert_eq!( + json!({ + "method": "thread/backgroundTerminals/terminate", + "id": 8, + "params": { + "threadId": "thr_123", + "processId": "42" + } + }), + serde_json::to_value(&request)?, + ); + Ok(()) + } + #[test] fn serialize_thread_realtime_start() -> Result<()> { let request = ClientRequest::ThreadRealtimeStart { diff --git a/codex-rs/app-server-protocol/src/protocol/v2/thread.rs b/codex-rs/app-server-protocol/src/protocol/v2/thread.rs index 0c7c362b52..9e1770fde0 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/thread.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/thread.rs @@ -935,6 +935,57 @@ pub struct ThreadBackgroundTerminalsCleanParams { #[ts(export_to = "v2/")] pub struct ThreadBackgroundTerminalsCleanResponse {} +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadBackgroundTerminalsListParams { + pub thread_id: String, + /// Opaque pagination cursor returned by a previous call. + #[ts(optional = nullable)] + pub cursor: Option, + /// Optional page size. + #[ts(optional = nullable)] + pub limit: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadBackgroundTerminal { + pub item_id: String, + pub process_id: String, + pub command: String, + pub cwd: AbsolutePathBuf, + pub os_pid: Option, + pub cpu_percent: Option, + pub rss_kb: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadBackgroundTerminalsListResponse { + pub data: Vec, + /// Opaque cursor to pass to the next call to continue after the last item. + /// If None, there are no more items to return. + pub next_cursor: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadBackgroundTerminalsTerminateParams { + pub thread_id: String, + pub process_id: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadBackgroundTerminalsTerminateResponse { + pub terminated: bool, +} + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index 999cc8fa37..766a5b0478 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -157,6 +157,8 @@ Example with notification opt-out: - `thread/compact/start` — trigger conversation history compaction for a thread; returns `{}` immediately while progress streams through standard turn/item notifications. - `thread/shellCommand` — run a user-initiated `!` shell command against a thread; this runs unsandboxed with full access rather than inheriting the thread sandbox policy. Returns `{}` immediately while progress streams through standard turn/item notifications and any active turn receives the formatted output in its message stream. - `thread/backgroundTerminals/clean` — terminate all running background terminals for a thread (experimental; requires `capabilities.experimentalApi`); returns `{}` when the cleanup request is accepted. +- `thread/backgroundTerminals/list` — list running background terminals for a loaded thread (experimental; requires `capabilities.experimentalApi`); returns `data` with the running terminal ids. +- `thread/backgroundTerminals/terminate` — terminate one running background terminal by app-server `processId` (experimental; requires `capabilities.experimentalApi`); returns whether a process was terminated. - `thread/rollback` — drop the last N turns from the agent’s in-memory context and persist a rollback marker in the rollout so future resumes see the pruned history; returns the updated `thread` (with `turns` populated) on success. - `turn/start` — add user input to a thread and begin Codex generation; responds with the initial `turn` object and streams `turn/started`, `item/*`, and `turn/completed` notifications. `clientUserMessageId` is optional; when supplied, the corresponding `userMessage` item echoes it as `clientId`. Experimental `runtimeWorkspaceRoots` replaces the thread-scoped runtime workspace roots used to materialize `:workspace_roots`; paths must be absolute. Prefer experimental `permissions` profile selection by id for permission overrides; the legacy `sandboxPolicy` field is still accepted but cannot be combined with `permissions`. For `collaborationMode`, `settings.developer_instructions: null` means "use built-in instructions for the selected mode". - `thread/inject_items` — append raw Responses API items to a loaded thread’s model-visible history without starting a user turn; returns `{}` on success. @@ -872,6 +874,32 @@ Use `thread/backgroundTerminals/clean` to terminate all running background termi { "id": 35, "result": {} } ``` +### Example: List and terminate background terminals + +Use `thread/backgroundTerminals/list` to inspect running background terminals associated with a loaded thread. The `backgroundTerminals` segment intentionally follows the existing `thread/backgroundTerminals/clean` method. The returned `processId` is the app-server process id; host OS metadata is nullable. The request accepts the standard `cursor` and `limit` pagination fields. When `nextCursor` is non-null, pass it as `cursor` to fetch the next page. + +```json +{ "method": "thread/backgroundTerminals/list", "id": 36, "params": { "threadId": "thr_123" } } +{ "id": 36, "result": { "data": [ + { + "itemId": "item_456", + "processId": "42", + "command": "python3 -m http.server", + "cwd": "/workspace", + "osPid": null, + "cpuPercent": null, + "rssKb": null + } +], "nextCursor": null } } +``` + +Use `thread/backgroundTerminals/terminate` to terminate one running background terminal by that `processId`. + +```json +{ "method": "thread/backgroundTerminals/terminate", "id": 37, "params": { "threadId": "thr_123", "processId": "42" } } +{ "id": 37, "result": { "terminated": true } } +``` + ### Example: Steer an active turn Use `turn/steer` to append additional user input to the currently active regular turn. This does diff --git a/codex-rs/app-server/src/message_processor.rs b/codex-rs/app-server/src/message_processor.rs index 6bb5121dc9..ce239a2718 100644 --- a/codex-rs/app-server/src/message_processor.rs +++ b/codex-rs/app-server/src/message_processor.rs @@ -1126,6 +1126,16 @@ impl MessageProcessor { .thread_background_terminals_clean(&request_id, params) .await } + ClientRequest::ThreadBackgroundTerminalsList { params, .. } => { + self.thread_processor + .thread_background_terminals_list(params) + .await + } + ClientRequest::ThreadBackgroundTerminalsTerminate { params, .. } => { + self.thread_processor + .thread_background_terminals_terminate(params) + .await + } ClientRequest::ThreadRollback { params, .. } => { self.thread_processor .thread_rollback(&request_id, params) diff --git a/codex-rs/app-server/src/request_processors.rs b/codex-rs/app-server/src/request_processors.rs index 00f17937ba..2bce5b8ddd 100644 --- a/codex-rs/app-server/src/request_processors.rs +++ b/codex-rs/app-server/src/request_processors.rs @@ -171,8 +171,13 @@ use codex_app_server_protocol::ThreadApproveGuardianDeniedActionResponse; use codex_app_server_protocol::ThreadArchiveParams; use codex_app_server_protocol::ThreadArchiveResponse; use codex_app_server_protocol::ThreadArchivedNotification; +use codex_app_server_protocol::ThreadBackgroundTerminal; use codex_app_server_protocol::ThreadBackgroundTerminalsCleanParams; use codex_app_server_protocol::ThreadBackgroundTerminalsCleanResponse; +use codex_app_server_protocol::ThreadBackgroundTerminalsListParams; +use codex_app_server_protocol::ThreadBackgroundTerminalsListResponse; +use codex_app_server_protocol::ThreadBackgroundTerminalsTerminateParams; +use codex_app_server_protocol::ThreadBackgroundTerminalsTerminateResponse; use codex_app_server_protocol::ThreadClosedNotification; use codex_app_server_protocol::ThreadCompactStartParams; use codex_app_server_protocol::ThreadCompactStartResponse; diff --git a/codex-rs/app-server/src/request_processors/thread_processor.rs b/codex-rs/app-server/src/request_processors/thread_processor.rs index fd6432531d..b10358d8d2 100644 --- a/codex-rs/app-server/src/request_processors/thread_processor.rs +++ b/codex-rs/app-server/src/request_processors/thread_processor.rs @@ -569,6 +569,24 @@ impl ThreadRequestProcessor { .map(|response| Some(response.into())) } + pub(crate) async fn thread_background_terminals_list( + &self, + params: ThreadBackgroundTerminalsListParams, + ) -> Result, JSONRPCErrorError> { + self.thread_background_terminals_list_inner(params) + .await + .map(|response| Some(response.into())) + } + + pub(crate) async fn thread_background_terminals_terminate( + &self, + params: ThreadBackgroundTerminalsTerminateParams, + ) -> Result, JSONRPCErrorError> { + self.thread_background_terminals_terminate_inner(params) + .await + .map(|response| Some(response.into())) + } + pub(crate) async fn thread_rollback( &self, request_id: &ConnectionRequestId, @@ -1725,6 +1743,54 @@ impl ThreadRequestProcessor { Ok(ThreadBackgroundTerminalsCleanResponse {}) } + async fn thread_background_terminals_list_inner( + &self, + params: ThreadBackgroundTerminalsListParams, + ) -> Result { + let ThreadBackgroundTerminalsListParams { + thread_id, + cursor, + limit, + } = params; + + let (_, thread) = self.load_thread(&thread_id).await?; + let terminals = thread + .list_background_terminals() + .await + .into_iter() + .map(|terminal| ThreadBackgroundTerminal { + item_id: terminal.item_id, + process_id: terminal.process_id, + command: terminal.command, + cwd: terminal.cwd, + os_pid: None, + cpu_percent: None, + rss_kb: None, + }) + .collect::>(); + + let (data, next_cursor) = paginate_background_terminals(&terminals, cursor, limit)?; + + Ok(ThreadBackgroundTerminalsListResponse { data, next_cursor }) + } + + async fn thread_background_terminals_terminate_inner( + &self, + params: ThreadBackgroundTerminalsTerminateParams, + ) -> Result { + let ThreadBackgroundTerminalsTerminateParams { + thread_id, + process_id, + } = params; + let process_id = process_id.parse::().map_err(|err| { + invalid_request(format!("invalid background terminal process id: {err}")) + })?; + + let (_, thread) = self.load_thread(&thread_id).await?; + let terminated = thread.terminate_background_terminal(process_id).await; + Ok(ThreadBackgroundTerminalsTerminateResponse { terminated }) + } + async fn thread_shell_command_inner( &self, request_id: &ConnectionRequestId, @@ -4250,6 +4316,34 @@ fn build_thread_from_snapshot( } } +fn paginate_background_terminals( + terminals: &[ThreadBackgroundTerminal], + cursor: Option, + limit: Option, +) -> Result<(Vec, Option), JSONRPCErrorError> { + let start = match cursor { + Some(cursor) => { + let cursor = cursor + .parse::() + .map_err(|err| invalid_request(format!("invalid cursor: {err}")))?; + terminals + .iter() + .position(|terminal| { + terminal + .process_id + .parse::() + .is_ok_and(|process_id| process_id > cursor) + }) + .unwrap_or(terminals.len()) + } + None => 0, + }; + let effective_limit = limit.unwrap_or(terminals.len() as u32).max(1) as usize; + let end = start.saturating_add(effective_limit).min(terminals.len()); + let next_cursor = (end < terminals.len()).then(|| terminals[end - 1].process_id.clone()); + Ok((terminals[start..end].to_vec(), next_cursor)) +} + fn build_thread_from_loaded_snapshot( thread_id: ThreadId, config_snapshot: &ThreadConfigSnapshot, diff --git a/codex-rs/app-server/src/request_processors/thread_processor_tests.rs b/codex-rs/app-server/src/request_processors/thread_processor_tests.rs index 54c0f0343f..7952517c92 100644 --- a/codex-rs/app-server/src/request_processors/thread_processor_tests.rs +++ b/codex-rs/app-server/src/request_processors/thread_processor_tests.rs @@ -36,6 +36,65 @@ mod thread_list_cwd_filter_tests { } } +mod background_terminal_pagination_tests { + use super::super::paginate_background_terminals; + use codex_app_server_protocol::ThreadBackgroundTerminal; + use codex_utils_absolute_path::AbsolutePathBuf; + use pretty_assertions::assert_eq; + + fn terminal(process_id: &str) -> ThreadBackgroundTerminal { + let cwd = if cfg!(windows) { r"C:\tmp" } else { "/tmp" }; + + ThreadBackgroundTerminal { + item_id: format!("item-{process_id}"), + process_id: process_id.to_string(), + command: format!("command-{process_id}"), + cwd: AbsolutePathBuf::from_absolute_path(cwd).expect("absolute cwd"), + os_pid: None, + cpu_percent: None, + rss_kb: None, + } + } + + #[test] + fn paginates_with_process_id_cursor() { + let terminals = vec![ + terminal("1"), + terminal("2"), + terminal("3"), + terminal("4"), + terminal("5"), + ]; + + let (data, next_cursor) = + paginate_background_terminals(&terminals, /*cursor*/ None, Some(2)) + .expect("valid page"); + + assert_eq!(data, vec![terminal("1"), terminal("2")]); + assert_eq!(next_cursor, Some("2".to_string())); + let first_cursor = next_cursor; + + let terminals_without_anchor = vec![terminal("1"), terminal("3"), terminal("4")]; + let (data, next_cursor) = + paginate_background_terminals(&terminals_without_anchor, first_cursor.clone(), Some(2)) + .expect("valid page"); + + assert_eq!(data, vec![terminal("3"), terminal("4")]); + assert_eq!(next_cursor, None); + + let (data, next_cursor) = + paginate_background_terminals(&terminals, first_cursor, Some(2)).expect("valid page"); + + assert_eq!(data, vec![terminal("3"), terminal("4")]); + assert_eq!(next_cursor, Some("4".to_string())); + + assert!( + paginate_background_terminals(&terminals, Some("missing".to_string()), Some(1)) + .is_err() + ); + } +} + mod thread_processor_behavior_tests { async fn forked_from_id_from_rollout(path: &Path) -> Option { codex_core::read_session_meta_line(path) diff --git a/codex-rs/core/src/codex_thread.rs b/codex-rs/core/src/codex_thread.rs index ab1f6df03e..c4cb1e8e12 100644 --- a/codex-rs/core/src/codex_thread.rs +++ b/codex-rs/core/src/codex_thread.rs @@ -159,6 +159,14 @@ pub struct CodexThread { out_of_band_elicitation_count: Mutex, } +#[derive(Debug, Eq, PartialEq)] +pub struct BackgroundTerminalInfo { + pub item_id: String, + pub process_id: String, + pub command: String, + pub cwd: AbsolutePathBuf, +} + /// Conduit for the bidirectional stream of messages that compose a thread /// (formerly called a conversation) in Codex. impl CodexThread { @@ -396,6 +404,17 @@ impl CodexThread { self.codex.agent_status().await } + pub async fn list_background_terminals(&self) -> Vec { + self.codex.session.list_background_terminals().await + } + + pub async fn terminate_background_terminal(&self, process_id: i32) -> bool { + self.codex + .session + .terminate_background_terminal(process_id) + .await + } + pub(crate) fn subscribe_status(&self) -> watch::Receiver { self.codex.agent_status.clone() } diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index 400c803680..a9aad7a1e8 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -19,6 +19,7 @@ mod codex_thread; mod compact_remote; mod compact_remote_v2; mod config_lock; +pub use codex_thread::BackgroundTerminalInfo; pub use codex_thread::CodexThread; pub use codex_thread::CodexThreadSettingsOverrides; pub use codex_thread::ThreadConfigSnapshot; diff --git a/codex-rs/core/src/tasks/mod.rs b/codex-rs/core/src/tasks/mod.rs index 911f30b319..52af0d04db 100644 --- a/codex-rs/core/src/tasks/mod.rs +++ b/codex-rs/core/src/tasks/mod.rs @@ -21,6 +21,7 @@ use tracing::info_span; use tracing::trace; use tracing::warn; +use crate::codex_thread::BackgroundTerminalInfo; use crate::config::Config; use crate::context::ContextualUserFragment; use crate::hook_runtime::inspect_pending_input; @@ -782,6 +783,17 @@ impl Session { .await; } + pub(crate) async fn list_background_terminals(&self) -> Vec { + self.services.unified_exec_manager.list_processes().await + } + + pub(crate) async fn terminate_background_terminal(&self, process_id: i32) -> bool { + self.services + .unified_exec_manager + .terminate_process(process_id) + .await + } + async fn handle_task_abort(self: &Arc, task: RunningTask, reason: TurnAbortReason) { let sub_id = task.turn_context.sub_id.clone(); if task.cancellation_token.is_cancelled() { diff --git a/codex-rs/core/src/unified_exec/mod.rs b/codex-rs/core/src/unified_exec/mod.rs index 94a7ab7c73..65b18b064d 100644 --- a/codex-rs/core/src/unified_exec/mod.rs +++ b/codex-rs/core/src/unified_exec/mod.rs @@ -155,6 +155,8 @@ struct ProcessEntry { process: Arc, call_id: String, process_id: i32, + cwd: AbsolutePathBuf, + initial_exec_command_active: Arc, hook_command: String, tty: bool, network_approval: Option, diff --git a/codex-rs/core/src/unified_exec/mod_tests.rs b/codex-rs/core/src/unified_exec/mod_tests.rs index 9f24314da2..d994354130 100644 --- a/codex-rs/core/src/unified_exec/mod_tests.rs +++ b/codex-rs/core/src/unified_exec/mod_tests.rs @@ -1,5 +1,6 @@ use super::head_tail_buffer::HeadTailBuffer; use super::*; +use crate::codex_thread::BackgroundTerminalInfo; use crate::exec::ExecCapturePolicy; use crate::exec::ExecExpiration; use crate::sandboxing::ExecRequest; @@ -9,6 +10,16 @@ use crate::session::turn_context::TurnContext; use crate::tools::context::ExecCommandToolOutput; use crate::unified_exec::WriteStdinRequest; use crate::unified_exec::process::OutputHandles; +use async_trait::async_trait; +use codex_exec_server::ExecProcess; +use codex_exec_server::ExecProcessEventReceiver; +use codex_exec_server::ExecServerError; +use codex_exec_server::ProcessId; +use codex_exec_server::ProcessSignal; +use codex_exec_server::ReadResponse; +use codex_exec_server::StartedExecProcess; +use codex_exec_server::WriteResponse; +use codex_exec_server::WriteStatus; use codex_sandboxing::SandboxType; use codex_utils_output_truncation::TruncationPolicy; use codex_utils_output_truncation::approx_token_count; @@ -19,6 +30,8 @@ use pretty_assertions::assert_eq; use std::collections::HashMap; use std::path::PathBuf; use std::sync::Arc; +use tokio::sync::Notify; +use tokio::sync::watch; use tokio::time::Duration; use tokio::time::Instant; @@ -116,6 +129,8 @@ async fn exec_command_with_tty( process: Arc::clone(&process), call_id: context.call_id.clone(), process_id, + cwd: cwd.clone(), + initial_exec_command_active: Arc::new(std::sync::atomic::AtomicBool::new(true)), hook_command: cmd.to_string(), tty, network_approval: None, @@ -158,6 +173,18 @@ async fn exec_command_with_tty( manager.release_process_id(process_id).await; None }; + if response_process_id.is_some() + && let Some(entry) = manager + .process_store + .lock() + .await + .processes + .get_mut(&process_id) + { + entry + .initial_exec_command_active + .store(false, std::sync::atomic::Ordering::Release); + } Ok(ExecCommandToolOutput { event_call_id: context.call_id, @@ -184,6 +211,82 @@ impl SpawnLifecycle for TestSpawnLifecycle { } } +struct BlockingTerminateExecProcess { + process_id: ProcessId, + terminate_started: watch::Sender, + allow_terminate: Arc, + wake_tx: watch::Sender, +} + +#[async_trait] +impl ExecProcess for BlockingTerminateExecProcess { + fn process_id(&self) -> &ProcessId { + &self.process_id + } + + fn subscribe_wake(&self) -> watch::Receiver { + self.wake_tx.subscribe() + } + + fn subscribe_events(&self) -> ExecProcessEventReceiver { + ExecProcessEventReceiver::empty() + } + + async fn read( + &self, + _after_seq: Option, + _max_bytes: Option, + _wait_ms: Option, + ) -> Result { + Ok(ReadResponse { + chunks: Vec::new(), + next_seq: 1, + exited: false, + exit_code: None, + closed: false, + failure: None, + }) + } + + async fn write(&self, _chunk: Vec) -> Result { + Ok(WriteResponse { + status: WriteStatus::Accepted, + }) + } + + async fn signal(&self, _signal: ProcessSignal) -> Result<(), ExecServerError> { + Ok(()) + } + + async fn terminate(&self) -> Result<(), ExecServerError> { + let _ = self.terminate_started.send(true); + self.allow_terminate.notified().await; + Ok(()) + } +} + +async fn blocking_terminate_unified_process( + process_id: i32, + terminate_started: watch::Sender, + allow_terminate: Arc, +) -> anyhow::Result> { + let (wake_tx, _wake_rx) = watch::channel(0); + Ok(Arc::new( + UnifiedExecProcess::from_exec_server_started( + StartedExecProcess { + process: Arc::new(BlockingTerminateExecProcess { + process_id: process_id.to_string().into(), + terminate_started, + allow_terminate, + wake_tx, + }), + }, + SandboxType::None, + ) + .await?, + )) +} + async fn write_stdin( session: &Arc, process_id: i32, @@ -241,12 +344,23 @@ async fn unified_exec_persists_across_requests() -> anyhow::Result<()> { skip_if_sandbox!(Ok(())); let (session, turn) = test_session_and_turn().await; + #[allow(deprecated)] + let cwd = turn.cwd.clone(); let open_shell = exec_command( &session, &turn, "bash -i", /*yield_time_ms*/ 2_500, /*workdir*/ None, ) .await?; let process_id = open_shell.process_id.expect("expected process_id"); + assert_eq!( + session.list_background_terminals().await, + vec![BackgroundTerminalInfo { + item_id: "call".to_string(), + process_id: process_id.to_string(), + command: "bash -i".to_string(), + cwd, + }] + ); write_stdin( &session, @@ -270,6 +384,10 @@ async fn unified_exec_persists_across_requests() -> anyhow::Result<()> { "expected environment variable output" ); + assert!(session.terminate_background_terminal(process_id).await); + assert!(!session.terminate_background_terminal(process_id).await); + assert!(session.list_background_terminals().await.is_empty()); + Ok(()) } @@ -523,6 +641,148 @@ async fn reusing_completed_process_returns_unknown_process() -> anyhow::Result<( Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn terminating_initial_exec_command_rechecks_initial_response_state() -> anyhow::Result<()> { + let (session, turn) = test_session_and_turn().await; + let manager = &session.services.unified_exec_manager; + let process_id = manager.allocate_process_id().await; + let (terminate_started_tx, mut terminate_started_rx) = watch::channel(false); + let allow_terminate = Arc::new(Notify::new()); + let process = blocking_terminate_unified_process( + process_id, + terminate_started_tx, + Arc::clone(&allow_terminate), + ) + .await?; + #[allow(deprecated)] + let cwd = turn.cwd.clone(); + manager.process_store.lock().await.processes.insert( + process_id, + ProcessEntry { + process, + call_id: "call".to_string(), + process_id, + cwd, + initial_exec_command_active: Arc::new(std::sync::atomic::AtomicBool::new(true)), + hook_command: "sleep 60".to_string(), + tty: true, + network_approval: None, + session: Arc::downgrade(&session), + last_used: Instant::now(), + }, + ); + + let terminate_task = tokio::spawn({ + let session = Arc::clone(&session); + async move { session.terminate_background_terminal(process_id).await } + }); + tokio::time::timeout( + Duration::from_secs(2), + terminate_started_rx.wait_for(|started| *started), + ) + .await + .expect("terminate should start") + .expect("terminate signal sender should stay open"); + + { + let mut store = manager.process_store.lock().await; + let entry = store + .processes + .get_mut(&process_id) + .expect("process should remain stored until initial response returns"); + entry + .initial_exec_command_active + .store(false, std::sync::atomic::Ordering::Release); + } + + allow_terminate.notify_waiters(); + let terminated = tokio::time::timeout(Duration::from_secs(2), terminate_task) + .await + .expect("terminate should finish") + .expect("terminate task should not panic"); + assert!(terminated); + assert!( + !manager + .process_store + .lock() + .await + .processes + .contains_key(&process_id) + ); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn terminating_during_stdin_poll_returns_exited_response() -> anyhow::Result<()> { + let (session, turn) = test_session_and_turn().await; + let manager = &session.services.unified_exec_manager; + let process_id = manager.allocate_process_id().await; + let (terminate_started_tx, _terminate_started_rx) = watch::channel(false); + let allow_terminate = Arc::new(Notify::new()); + let process = blocking_terminate_unified_process( + process_id, + terminate_started_tx, + Arc::clone(&allow_terminate), + ) + .await?; + #[allow(deprecated)] + let cwd = turn.cwd.clone(); + let last_used = Instant::now() - Duration::from_secs(1); + manager.process_store.lock().await.processes.insert( + process_id, + ProcessEntry { + process: Arc::clone(&process), + call_id: "call".to_string(), + process_id, + cwd, + initial_exec_command_active: Arc::new(std::sync::atomic::AtomicBool::new(false)), + hook_command: "sleep 60".to_string(), + tty: true, + network_approval: None, + session: Arc::downgrade(&session), + last_used, + }, + ); + + let poll_task = tokio::spawn({ + let session = Arc::clone(&session); + async move { + write_stdin(&session, process_id, "", /*yield_time_ms*/ 60_000).await + } + }); + tokio::time::timeout(Duration::from_secs(2), async { + loop { + let poll_started = manager + .process_store + .lock() + .await + .processes + .get(&process_id) + .is_some_and(|entry| entry.last_used != last_used); + if poll_started { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("poll should clone process handles"); + + manager.release_process_id(process_id).await; + allow_terminate.notify_one(); + process.terminate_confirmed().await?; + + let output = tokio::time::timeout(Duration::from_secs(2), poll_task) + .await + .expect("poll should finish") + .expect("poll task should not panic")?; + assert_eq!(output.process_id, None); + assert!(manager.process_store.lock().await.processes.is_empty()); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn completed_pipe_commands_preserve_exit_code() -> anyhow::Result<()> { let (_, turn) = make_session_and_context().await; diff --git a/codex-rs/core/src/unified_exec/process.rs b/codex-rs/core/src/unified_exec/process.rs index 229e5b52f8..725be9eed2 100644 --- a/codex-rs/core/src/unified_exec/process.rs +++ b/codex-rs/core/src/unified_exec/process.rs @@ -195,9 +195,16 @@ impl UnifiedExecProcess { } } - pub(super) fn terminate(&self) { + fn finish_termination(&self) { self.output_closed.store(true, Ordering::Release); self.output_closed_notify.notify_waiters(); + self.cancellation_token.cancel(); + if let Some(output_task) = &self.output_task { + output_task.abort(); + } + } + + pub(super) fn terminate(&self) { match &self.process_handle { ProcessHandle::Local(process_handle) => process_handle.terminate(), ProcessHandle::ExecServer(process_handle) => { @@ -207,10 +214,22 @@ impl UnifiedExecProcess { }); } } - self.cancellation_token.cancel(); - if let Some(output_task) = &self.output_task { - output_task.abort(); + self.finish_termination(); + } + + pub(super) async fn terminate_confirmed(&self) -> Result<(), UnifiedExecError> { + match &self.process_handle { + ProcessHandle::Local(process_handle) => process_handle.terminate(), + ProcessHandle::ExecServer(process_handle) => { + process_handle + .terminate() + .await + .map_err(|err| UnifiedExecError::process_failed(err.to_string()))?; + } } + self.signal_exit(self.exit_code()); + self.finish_termination(); + Ok(()) } pub(super) async fn interrupt(&self) -> Result<(), UnifiedExecError> { diff --git a/codex-rs/core/src/unified_exec/process_manager.rs b/codex-rs/core/src/unified_exec/process_manager.rs index c9e6f3fdca..d744a72c95 100644 --- a/codex-rs/core/src/unified_exec/process_manager.rs +++ b/codex-rs/core/src/unified_exec/process_manager.rs @@ -11,6 +11,7 @@ use tokio::time::Duration; use tokio::time::Instant; use tokio_util::sync::CancellationToken; +use crate::codex_thread::BackgroundTerminalInfo; use crate::exec_env::CODEX_THREAD_ID_ENV_VAR; use crate::exec_env::create_env; use crate::exec_policy::ExecApprovalRequest; @@ -175,11 +176,22 @@ struct PreparedProcessHandles { pause_state: Option>, session: Option>, network_approval: Option, + call_id: String, hook_command: String, process_id: i32, tty: bool, } +struct InitialExecCommandGuard { + active: Arc, +} + +impl Drop for InitialExecCommandGuard { + fn drop(&mut self) { + self.active.store(false, Ordering::Release); + } +} + fn exec_server_process_id(process_id: i32) -> String { process_id.to_string() } @@ -414,7 +426,8 @@ impl UnifiedExecProcessManager { // Persist live sessions before the initial yield wait so interrupting the // turn cannot drop the last Arc and terminate the background process. let process_started_alive = !process.has_exited() && process.exit_code().is_none(); - if process_started_alive { + let _initial_exec_command_guard = if process_started_alive { + let initial_exec_command_active = Arc::new(AtomicBool::new(true)); self.store_process( Arc::clone(&process), context, @@ -426,9 +439,15 @@ impl UnifiedExecProcessManager { request.tty, deferred_network_approval.clone(), Arc::clone(&transcript), + Arc::clone(&initial_exec_command_active), ) .await; - } + Some(InitialExecCommandGuard { + active: initial_exec_command_active, + }) + } else { + None + }; let yield_time_ms = clamp_yield_time(request.yield_time_ms); // For the initial exec_command call, we both stream output to events @@ -609,6 +628,7 @@ impl UnifiedExecProcessManager { pause_state, session, network_approval, + call_id, hook_command, process_id, tty, @@ -721,9 +741,13 @@ impl UnifiedExecProcessManager { (None, exit_code, call_id) } ProcessStatus::Unknown => { - return Err(UnifiedExecError::UnknownProcessId { - process_id: request.process_id, - }); + if process.has_exited() { + (None, process.exit_code(), call_id) + } else { + return Err(UnifiedExecError::UnknownProcessId { + process_id: request.process_id, + }); + } } }; @@ -744,29 +768,27 @@ impl UnifiedExecProcessManager { } async fn refresh_process_state(&self, process_id: i32) -> ProcessStatus { - { - let mut store = self.process_store.lock().await; - let Some(entry) = store.processes.get(&process_id) else { + let mut store = self.process_store.lock().await; + let Some(entry) = store.processes.get_mut(&process_id) else { + return ProcessStatus::Unknown; + }; + + let exit_code = entry.process.exit_code(); + let process_id = entry.process_id; + + if entry.process.has_exited() { + let Some(entry) = store.remove(process_id) else { return ProcessStatus::Unknown; }; - - let exit_code = entry.process.exit_code(); - let process_id = entry.process_id; - - if entry.process.has_exited() { - let Some(entry) = store.remove(process_id) else { - return ProcessStatus::Unknown; - }; - ProcessStatus::Exited { - exit_code, - entry: Box::new(entry), - } - } else { - ProcessStatus::Alive { - exit_code, - call_id: entry.call_id.clone(), - process_id, - } + ProcessStatus::Exited { + exit_code, + entry: Box::new(entry), + } + } else { + ProcessStatus::Alive { + exit_code, + call_id: entry.call_id.clone(), + process_id, } } } @@ -804,6 +826,7 @@ impl UnifiedExecProcessManager { pause_state, session, network_approval: entry.network_approval.clone(), + call_id: entry.call_id.clone(), hook_command: entry.hook_command.clone(), process_id: entry.process_id, tty: entry.tty, @@ -823,11 +846,14 @@ impl UnifiedExecProcessManager { tty: bool, network_approval: Option, transcript: Arc>, + initial_exec_command_active: Arc, ) { let entry = ProcessEntry { process: Arc::clone(&process), call_id: context.call_id.clone(), process_id, + cwd: cwd.clone(), + initial_exec_command_active, hook_command, tty, network_approval, @@ -1274,6 +1300,59 @@ impl UnifiedExecProcessManager { entry.process.terminate(); } } + + pub(crate) async fn list_processes(&self) -> Vec { + let store = self.process_store.lock().await; + let mut entries = store + .processes + .values() + .filter(|entry| !entry.process.has_exited()) + .collect::>(); + entries.sort_by_key(|entry| entry.process_id); + entries + .into_iter() + .map(|entry| BackgroundTerminalInfo { + item_id: entry.call_id.clone(), + process_id: entry.process_id.to_string(), + command: entry.hook_command.clone(), + cwd: entry.cwd.clone(), + }) + .collect() + } + + pub(crate) async fn terminate_process(&self, process_id: i32) -> bool { + let (process, already_exited) = { + let store = self.process_store.lock().await; + let Some(entry) = store.processes.get(&process_id) else { + return false; + }; + (Arc::clone(&entry.process), entry.process.has_exited()) + }; + + if !already_exited && process.terminate_confirmed().await.is_err() { + return false; + } + + let entry = { + let mut store = self.process_store.lock().await; + let Some(entry) = store.processes.get(&process_id) else { + return true; + }; + if !Arc::ptr_eq(&entry.process, &process) { + return true; + } + if entry.initial_exec_command_active.load(Ordering::Acquire) { + return true; + } + let Some(entry) = store.remove(process_id) else { + return false; + }; + entry + }; + + unregister_network_approval_for_entry(&entry).await; + true + } } enum ProcessStatus { diff --git a/codex-rs/core/src/unified_exec/process_tests.rs b/codex-rs/core/src/unified_exec/process_tests.rs index 42db18ff78..814de95af4 100644 --- a/codex-rs/core/src/unified_exec/process_tests.rs +++ b/codex-rs/core/src/unified_exec/process_tests.rs @@ -22,6 +22,7 @@ struct MockExecProcess { process_id: ProcessId, write_response: WriteResponse, read_responses: Mutex>, + terminate_error: Option, wake_tx: watch::Sender, } @@ -69,11 +70,17 @@ impl ExecProcess for MockExecProcess { } async fn terminate(&self) -> Result<(), ExecServerError> { + if let Some(message) = &self.terminate_error { + return Err(ExecServerError::Protocol(message.clone())); + } Ok(()) } } -async fn remote_process(write_status: WriteStatus) -> UnifiedExecProcess { +async fn remote_process( + write_status: WriteStatus, + terminate_error: Option, +) -> UnifiedExecProcess { let (wake_tx, _wake_rx) = watch::channel(0); let started = StartedExecProcess { process: Arc::new(MockExecProcess { @@ -82,6 +89,7 @@ async fn remote_process(write_status: WriteStatus) -> UnifiedExecProcess { status: write_status, }, read_responses: Mutex::new(VecDeque::new()), + terminate_error, wake_tx, }), }; @@ -93,7 +101,7 @@ async fn remote_process(write_status: WriteStatus) -> UnifiedExecProcess { #[tokio::test] async fn remote_write_unknown_process_marks_process_exited() { - let process = remote_process(WriteStatus::UnknownProcess).await; + let process = remote_process(WriteStatus::UnknownProcess, /*terminate_error*/ None).await; let err = process .write(b"hello") @@ -106,7 +114,7 @@ async fn remote_write_unknown_process_marks_process_exited() { #[tokio::test] async fn remote_write_closed_stdin_marks_process_exited() { - let process = remote_process(WriteStatus::StdinClosed).await; + let process = remote_process(WriteStatus::StdinClosed, /*terminate_error*/ None).await; let err = process .write(b"hello") @@ -119,7 +127,7 @@ async fn remote_write_closed_stdin_marks_process_exited() { #[tokio::test] async fn fail_and_terminate_preserves_failure_message() { - let process = remote_process(WriteStatus::Accepted).await; + let process = remote_process(WriteStatus::Accepted, /*terminate_error*/ None).await; process.fail_and_terminate("network denied".to_string()); process.fail_and_terminate("second failure".to_string()); @@ -131,6 +139,32 @@ async fn fail_and_terminate_preserves_failure_message() { ); } +#[tokio::test] +async fn remote_terminate_confirmed_updates_state_on_success_only() { + let process = remote_process( + WriteStatus::Accepted, + Some("terminate unavailable".to_string()), + ) + .await; + + let err = process + .terminate_confirmed() + .await + .expect_err("expected terminate failure"); + + assert!(matches!(err, UnifiedExecError::ProcessFailed { .. })); + assert!(!process.has_exited()); + + let process = remote_process(WriteStatus::Accepted, /*terminate_error*/ None).await; + + process + .terminate_confirmed() + .await + .expect("terminate should succeed"); + + assert!(process.has_exited()); +} + #[tokio::test] async fn remote_process_waits_for_early_exit_event() { let (wake_tx, _wake_rx) = watch::channel(0); @@ -148,6 +182,7 @@ async fn remote_process_waits_for_early_exit_event() { closed: true, failure: None, }])), + terminate_error: None, wake_tx: wake_tx.clone(), }), }; From a19d43a40aee5f3308ce57eda604fa085fd9f356 Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Wed, 10 Jun 2026 11:22:12 -0700 Subject: [PATCH 41/47] Add app-server `thread/delete` API (#25018) ## Why Clients can archive and unarchive threads today, but there is no app-server API for permanently removing a thread. Deletion also needs to cover the full session tree: deleting a main thread should remove spawned subagent threads and the related local metadata instead of leaving orphaned rollout files, goals, or subagent state behind. ## What - Adds the v2 `thread/delete` request and `thread/deleted` notification, with the response shape kept consistent with `thread/archive`. - Implements local hard delete for active and archived rollout files. - Deletes the requested thread's state DB row as the commit point, then best-effort cleans associated state including spawned descendants, goals, spawn edges, logs, dynamic tools, and agent job assignments. - Updates app-server API docs and generated protocol schema/TypeScript fixtures. --- .../schema/json/ClientRequest.json | 35 +++ .../schema/json/ServerNotification.json | 31 ++ .../codex_app_server_protocol.schemas.json | 75 +++++ .../codex_app_server_protocol.v2.schemas.json | 75 +++++ .../schema/json/v2/ThreadDeleteParams.json | 13 + .../schema/json/v2/ThreadDeleteResponse.json | 5 + .../json/v2/ThreadDeletedNotification.json | 13 + .../schema/typescript/ClientRequest.ts | 3 +- .../schema/typescript/ServerNotification.ts | 3 +- .../typescript/v2/ThreadDeleteParams.ts | 5 + .../typescript/v2/ThreadDeleteResponse.ts | 5 + .../v2/ThreadDeletedNotification.ts | 5 + .../schema/typescript/v2/index.ts | 3 + .../src/protocol/common.rs | 6 + .../src/protocol/v2/thread.rs | 19 ++ codex-rs/app-server/README.md | 11 + codex-rs/app-server/src/message_processor.rs | 8 +- codex-rs/app-server/src/request_processors.rs | 5 + .../src/request_processors/thread_delete.rs | 188 +++++++++++ .../request_processors/thread_processor.rs | 60 ++-- .../tests/common/test_app_server.rs | 10 + codex-rs/app-server/tests/suite/v2/mod.rs | 1 + .../tests/suite/v2/remote_thread_store.rs | 108 ++++--- .../tests/suite/v2/thread_delete.rs | 200 ++++++++++++ .../app-server/tests/suite/v2/turn_start.rs | 48 +++ codex-rs/core/src/agent/control_tests.rs | 4 +- codex-rs/core/src/thread_manager.rs | 11 +- codex-rs/rollout/src/lib.rs | 1 + codex-rs/rollout/src/session_index.rs | 56 +++- codex-rs/state/src/runtime/threads.rs | 294 +++++++++++++++++- codex-rs/thread-store/src/in_memory.rs | 21 ++ codex-rs/thread-store/src/lib.rs | 1 + .../thread-store/src/local/delete_thread.rs | 210 +++++++++++++ codex-rs/thread-store/src/local/mod.rs | 6 + codex-rs/thread-store/src/store.rs | 4 + codex-rs/thread-store/src/types.rs | 7 + .../tui/src/app/app_server_event_targets.rs | 1 + codex-rs/tui/src/chatwidget/protocol.rs | 1 + 38 files changed, 1464 insertions(+), 88 deletions(-) create mode 100644 codex-rs/app-server-protocol/schema/json/v2/ThreadDeleteParams.json create mode 100644 codex-rs/app-server-protocol/schema/json/v2/ThreadDeleteResponse.json create mode 100644 codex-rs/app-server-protocol/schema/json/v2/ThreadDeletedNotification.json create mode 100644 codex-rs/app-server-protocol/schema/typescript/v2/ThreadDeleteParams.ts create mode 100644 codex-rs/app-server-protocol/schema/typescript/v2/ThreadDeleteResponse.ts create mode 100644 codex-rs/app-server-protocol/schema/typescript/v2/ThreadDeletedNotification.ts create mode 100644 codex-rs/app-server/src/request_processors/thread_delete.rs create mode 100644 codex-rs/app-server/tests/suite/v2/thread_delete.rs create mode 100644 codex-rs/thread-store/src/local/delete_thread.rs diff --git a/codex-rs/app-server-protocol/schema/json/ClientRequest.json b/codex-rs/app-server-protocol/schema/json/ClientRequest.json index 2701dfb34d..6bdf195361 100644 --- a/codex-rs/app-server-protocol/schema/json/ClientRequest.json +++ b/codex-rs/app-server-protocol/schema/json/ClientRequest.json @@ -3203,6 +3203,17 @@ ], "type": "object" }, + "ThreadDeleteParams": { + "properties": { + "threadId": { + "type": "string" + } + }, + "required": [ + "threadId" + ], + "type": "object" + }, "ThreadForkParams": { "description": "There are two ways to fork a thread: 1. By thread_id: load the thread from disk by thread_id and fork it into a new thread. 2. By path: load the thread from disk by path and fork it into a new thread.\n\nIf using a non-empty path, the thread_id param will be ignored. Empty string path values are treated as absent.\n\nPrefer using thread_id whenever possible.", "properties": { @@ -4514,6 +4525,30 @@ "title": "Thread/archiveRequest", "type": "object" }, + { + "properties": { + "id": { + "$ref": "#/definitions/RequestId" + }, + "method": { + "enum": [ + "thread/delete" + ], + "title": "Thread/deleteRequestMethod", + "type": "string" + }, + "params": { + "$ref": "#/definitions/ThreadDeleteParams" + } + }, + "required": [ + "id", + "method", + "params" + ], + "title": "Thread/deleteRequest", + "type": "object" + }, { "properties": { "id": { diff --git a/codex-rs/app-server-protocol/schema/json/ServerNotification.json b/codex-rs/app-server-protocol/schema/json/ServerNotification.json index e0651d1727..89831b3feb 100644 --- a/codex-rs/app-server-protocol/schema/json/ServerNotification.json +++ b/codex-rs/app-server-protocol/schema/json/ServerNotification.json @@ -3523,6 +3523,17 @@ ], "type": "object" }, + "ThreadDeletedNotification": { + "properties": { + "threadId": { + "type": "string" + } + }, + "required": [ + "threadId" + ], + "type": "object" + }, "ThreadGoal": { "properties": { "createdAt": { @@ -5447,6 +5458,26 @@ "title": "Thread/archivedNotification", "type": "object" }, + { + "properties": { + "method": { + "enum": [ + "thread/deleted" + ], + "title": "Thread/deletedNotificationMethod", + "type": "string" + }, + "params": { + "$ref": "#/definitions/ThreadDeletedNotification" + } + }, + "required": [ + "method", + "params" + ], + "title": "Thread/deletedNotification", + "type": "object" + }, { "properties": { "method": { diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index d65c043a7d..fcf014d26f 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -324,6 +324,30 @@ "title": "Thread/archiveRequest", "type": "object" }, + { + "properties": { + "id": { + "$ref": "#/definitions/v2/RequestId" + }, + "method": { + "enum": [ + "thread/delete" + ], + "title": "Thread/deleteRequestMethod", + "type": "string" + }, + "params": { + "$ref": "#/definitions/v2/ThreadDeleteParams" + } + }, + "required": [ + "id", + "method", + "params" + ], + "title": "Thread/deleteRequest", + "type": "object" + }, { "properties": { "id": { @@ -4061,6 +4085,26 @@ "title": "Thread/archivedNotification", "type": "object" }, + { + "properties": { + "method": { + "enum": [ + "thread/deleted" + ], + "title": "Thread/deletedNotificationMethod", + "type": "string" + }, + "params": { + "$ref": "#/definitions/v2/ThreadDeletedNotification" + } + }, + "required": [ + "method", + "params" + ], + "title": "Thread/deletedNotification", + "type": "object" + }, { "properties": { "method": { @@ -16053,6 +16097,37 @@ "title": "ThreadCompactStartResponse", "type": "object" }, + "ThreadDeleteParams": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "threadId": { + "type": "string" + } + }, + "required": [ + "threadId" + ], + "title": "ThreadDeleteParams", + "type": "object" + }, + "ThreadDeleteResponse": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "ThreadDeleteResponse", + "type": "object" + }, + "ThreadDeletedNotification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "threadId": { + "type": "string" + } + }, + "required": [ + "threadId" + ], + "title": "ThreadDeletedNotification", + "type": "object" + }, "ThreadForkParams": { "$schema": "http://json-schema.org/draft-07/schema#", "description": "There are two ways to fork a thread: 1. By thread_id: load the thread from disk by thread_id and fork it into a new thread. 2. By path: load the thread from disk by path and fork it into a new thread.\n\nIf using a non-empty path, the thread_id param will be ignored. Empty string path values are treated as absent.\n\nPrefer using thread_id whenever possible.", diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index 8271dc485d..ee0415e13d 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -1263,6 +1263,30 @@ "title": "Thread/archiveRequest", "type": "object" }, + { + "properties": { + "id": { + "$ref": "#/definitions/RequestId" + }, + "method": { + "enum": [ + "thread/delete" + ], + "title": "Thread/deleteRequestMethod", + "type": "string" + }, + "params": { + "$ref": "#/definitions/ThreadDeleteParams" + } + }, + "required": [ + "id", + "method", + "params" + ], + "title": "Thread/deleteRequest", + "type": "object" + }, { "properties": { "id": { @@ -11726,6 +11750,26 @@ "title": "Thread/archivedNotification", "type": "object" }, + { + "properties": { + "method": { + "enum": [ + "thread/deleted" + ], + "title": "Thread/deletedNotificationMethod", + "type": "string" + }, + "params": { + "$ref": "#/definitions/ThreadDeletedNotification" + } + }, + "required": [ + "method", + "params" + ], + "title": "Thread/deletedNotification", + "type": "object" + }, { "properties": { "method": { @@ -13870,6 +13914,37 @@ "title": "ThreadCompactStartResponse", "type": "object" }, + "ThreadDeleteParams": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "threadId": { + "type": "string" + } + }, + "required": [ + "threadId" + ], + "title": "ThreadDeleteParams", + "type": "object" + }, + "ThreadDeleteResponse": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "ThreadDeleteResponse", + "type": "object" + }, + "ThreadDeletedNotification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "threadId": { + "type": "string" + } + }, + "required": [ + "threadId" + ], + "title": "ThreadDeletedNotification", + "type": "object" + }, "ThreadForkParams": { "$schema": "http://json-schema.org/draft-07/schema#", "description": "There are two ways to fork a thread: 1. By thread_id: load the thread from disk by thread_id and fork it into a new thread. 2. By path: load the thread from disk by path and fork it into a new thread.\n\nIf using a non-empty path, the thread_id param will be ignored. Empty string path values are treated as absent.\n\nPrefer using thread_id whenever possible.", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadDeleteParams.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadDeleteParams.json new file mode 100644 index 0000000000..1711e11a2f --- /dev/null +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadDeleteParams.json @@ -0,0 +1,13 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "threadId": { + "type": "string" + } + }, + "required": [ + "threadId" + ], + "title": "ThreadDeleteParams", + "type": "object" +} \ No newline at end of file diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadDeleteResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadDeleteResponse.json new file mode 100644 index 0000000000..ff9f485338 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadDeleteResponse.json @@ -0,0 +1,5 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "ThreadDeleteResponse", + "type": "object" +} \ No newline at end of file diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadDeletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadDeletedNotification.json new file mode 100644 index 0000000000..53011ea014 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadDeletedNotification.json @@ -0,0 +1,13 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "threadId": { + "type": "string" + } + }, + "required": [ + "threadId" + ], + "title": "ThreadDeletedNotification", + "type": "object" +} \ No newline at end of file diff --git a/codex-rs/app-server-protocol/schema/typescript/ClientRequest.ts b/codex-rs/app-server-protocol/schema/typescript/ClientRequest.ts index c91792c726..b6a13fe307 100644 --- a/codex-rs/app-server-protocol/schema/typescript/ClientRequest.ts +++ b/codex-rs/app-server-protocol/schema/typescript/ClientRequest.ts @@ -62,6 +62,7 @@ import type { SkillsListParams } from "./v2/SkillsListParams"; import type { ThreadApproveGuardianDeniedActionParams } from "./v2/ThreadApproveGuardianDeniedActionParams"; import type { ThreadArchiveParams } from "./v2/ThreadArchiveParams"; import type { ThreadCompactStartParams } from "./v2/ThreadCompactStartParams"; +import type { ThreadDeleteParams } from "./v2/ThreadDeleteParams"; import type { ThreadForkParams } from "./v2/ThreadForkParams"; import type { ThreadGoalClearParams } from "./v2/ThreadGoalClearParams"; import type { ThreadGoalGetParams } from "./v2/ThreadGoalGetParams"; @@ -86,4 +87,4 @@ import type { WindowsSandboxSetupStartParams } from "./v2/WindowsSandboxSetupSta /** * Request from the client to the server. */ -export type ClientRequest ={ "method": "initialize", id: RequestId, params: InitializeParams, } | { "method": "thread/start", id: RequestId, params: ThreadStartParams, } | { "method": "thread/resume", id: RequestId, params: ThreadResumeParams, } | { "method": "thread/fork", id: RequestId, params: ThreadForkParams, } | { "method": "thread/archive", id: RequestId, params: ThreadArchiveParams, } | { "method": "thread/unsubscribe", id: RequestId, params: ThreadUnsubscribeParams, } | { "method": "thread/name/set", id: RequestId, params: ThreadSetNameParams, } | { "method": "thread/goal/set", id: RequestId, params: ThreadGoalSetParams, } | { "method": "thread/goal/get", id: RequestId, params: ThreadGoalGetParams, } | { "method": "thread/goal/clear", id: RequestId, params: ThreadGoalClearParams, } | { "method": "thread/metadata/update", id: RequestId, params: ThreadMetadataUpdateParams, } | { "method": "thread/unarchive", id: RequestId, params: ThreadUnarchiveParams, } | { "method": "thread/compact/start", id: RequestId, params: ThreadCompactStartParams, } | { "method": "thread/shellCommand", id: RequestId, params: ThreadShellCommandParams, } | { "method": "thread/approveGuardianDeniedAction", id: RequestId, params: ThreadApproveGuardianDeniedActionParams, } | { "method": "thread/rollback", id: RequestId, params: ThreadRollbackParams, } | { "method": "thread/list", id: RequestId, params: ThreadListParams, } | { "method": "thread/loaded/list", id: RequestId, params: ThreadLoadedListParams, } | { "method": "thread/read", id: RequestId, params: ThreadReadParams, } | { "method": "thread/inject_items", id: RequestId, params: ThreadInjectItemsParams, } | { "method": "skills/list", id: RequestId, params: SkillsListParams, } | { "method": "skills/extraRoots/set", id: RequestId, params: SkillsExtraRootsSetParams, } | { "method": "hooks/list", id: RequestId, params: HooksListParams, } | { "method": "marketplace/add", id: RequestId, params: MarketplaceAddParams, } | { "method": "marketplace/remove", id: RequestId, params: MarketplaceRemoveParams, } | { "method": "marketplace/upgrade", id: RequestId, params: MarketplaceUpgradeParams, } | { "method": "plugin/list", id: RequestId, params: PluginListParams, } | { "method": "plugin/installed", id: RequestId, params: PluginInstalledParams, } | { "method": "plugin/read", id: RequestId, params: PluginReadParams, } | { "method": "plugin/skill/read", id: RequestId, params: PluginSkillReadParams, } | { "method": "plugin/share/save", id: RequestId, params: PluginShareSaveParams, } | { "method": "plugin/share/updateTargets", id: RequestId, params: PluginShareUpdateTargetsParams, } | { "method": "plugin/share/list", id: RequestId, params: PluginShareListParams, } | { "method": "plugin/share/checkout", id: RequestId, params: PluginShareCheckoutParams, } | { "method": "plugin/share/delete", id: RequestId, params: PluginShareDeleteParams, } | { "method": "app/list", id: RequestId, params: AppsListParams, } | { "method": "fs/readFile", id: RequestId, params: FsReadFileParams, } | { "method": "fs/writeFile", id: RequestId, params: FsWriteFileParams, } | { "method": "fs/createDirectory", id: RequestId, params: FsCreateDirectoryParams, } | { "method": "fs/getMetadata", id: RequestId, params: FsGetMetadataParams, } | { "method": "fs/readDirectory", id: RequestId, params: FsReadDirectoryParams, } | { "method": "fs/remove", id: RequestId, params: FsRemoveParams, } | { "method": "fs/copy", id: RequestId, params: FsCopyParams, } | { "method": "fs/watch", id: RequestId, params: FsWatchParams, } | { "method": "fs/unwatch", id: RequestId, params: FsUnwatchParams, } | { "method": "skills/config/write", id: RequestId, params: SkillsConfigWriteParams, } | { "method": "plugin/install", id: RequestId, params: PluginInstallParams, } | { "method": "plugin/uninstall", id: RequestId, params: PluginUninstallParams, } | { "method": "turn/start", id: RequestId, params: TurnStartParams, } | { "method": "turn/steer", id: RequestId, params: TurnSteerParams, } | { "method": "turn/interrupt", id: RequestId, params: TurnInterruptParams, } | { "method": "review/start", id: RequestId, params: ReviewStartParams, } | { "method": "model/list", id: RequestId, params: ModelListParams, } | { "method": "modelProvider/capabilities/read", id: RequestId, params: ModelProviderCapabilitiesReadParams, } | { "method": "experimentalFeature/list", id: RequestId, params: ExperimentalFeatureListParams, } | { "method": "permissionProfile/list", id: RequestId, params: PermissionProfileListParams, } | { "method": "experimentalFeature/enablement/set", id: RequestId, params: ExperimentalFeatureEnablementSetParams, } | { "method": "mcpServer/oauth/login", id: RequestId, params: McpServerOauthLoginParams, } | { "method": "config/mcpServer/reload", id: RequestId, params: undefined, } | { "method": "mcpServerStatus/list", id: RequestId, params: ListMcpServerStatusParams, } | { "method": "mcpServer/resource/read", id: RequestId, params: McpResourceReadParams, } | { "method": "mcpServer/tool/call", id: RequestId, params: McpServerToolCallParams, } | { "method": "windowsSandbox/setupStart", id: RequestId, params: WindowsSandboxSetupStartParams, } | { "method": "windowsSandbox/readiness", id: RequestId, params: undefined, } | { "method": "account/login/start", id: RequestId, params: LoginAccountParams, } | { "method": "account/login/cancel", id: RequestId, params: CancelLoginAccountParams, } | { "method": "account/logout", id: RequestId, params: undefined, } | { "method": "account/rateLimits/read", id: RequestId, params: undefined, } | { "method": "account/usage/read", id: RequestId, params: undefined, } | { "method": "account/sendAddCreditsNudgeEmail", id: RequestId, params: SendAddCreditsNudgeEmailParams, } | { "method": "feedback/upload", id: RequestId, params: FeedbackUploadParams, } | { "method": "command/exec", id: RequestId, params: CommandExecParams, } | { "method": "command/exec/write", id: RequestId, params: CommandExecWriteParams, } | { "method": "command/exec/terminate", id: RequestId, params: CommandExecTerminateParams, } | { "method": "command/exec/resize", id: RequestId, params: CommandExecResizeParams, } | { "method": "config/read", id: RequestId, params: ConfigReadParams, } | { "method": "externalAgentConfig/detect", id: RequestId, params: ExternalAgentConfigDetectParams, } | { "method": "externalAgentConfig/import", id: RequestId, params: ExternalAgentConfigImportParams, } | { "method": "config/value/write", id: RequestId, params: ConfigValueWriteParams, } | { "method": "config/batchWrite", id: RequestId, params: ConfigBatchWriteParams, } | { "method": "configRequirements/read", id: RequestId, params: undefined, } | { "method": "account/read", id: RequestId, params: GetAccountParams, } | { "method": "getConversationSummary", id: RequestId, params: GetConversationSummaryParams, } | { "method": "gitDiffToRemote", id: RequestId, params: GitDiffToRemoteParams, } | { "method": "getAuthStatus", id: RequestId, params: GetAuthStatusParams, } | { "method": "fuzzyFileSearch", id: RequestId, params: FuzzyFileSearchParams, }; +export type ClientRequest ={ "method": "initialize", id: RequestId, params: InitializeParams, } | { "method": "thread/start", id: RequestId, params: ThreadStartParams, } | { "method": "thread/resume", id: RequestId, params: ThreadResumeParams, } | { "method": "thread/fork", id: RequestId, params: ThreadForkParams, } | { "method": "thread/archive", id: RequestId, params: ThreadArchiveParams, } | { "method": "thread/delete", id: RequestId, params: ThreadDeleteParams, } | { "method": "thread/unsubscribe", id: RequestId, params: ThreadUnsubscribeParams, } | { "method": "thread/name/set", id: RequestId, params: ThreadSetNameParams, } | { "method": "thread/goal/set", id: RequestId, params: ThreadGoalSetParams, } | { "method": "thread/goal/get", id: RequestId, params: ThreadGoalGetParams, } | { "method": "thread/goal/clear", id: RequestId, params: ThreadGoalClearParams, } | { "method": "thread/metadata/update", id: RequestId, params: ThreadMetadataUpdateParams, } | { "method": "thread/unarchive", id: RequestId, params: ThreadUnarchiveParams, } | { "method": "thread/compact/start", id: RequestId, params: ThreadCompactStartParams, } | { "method": "thread/shellCommand", id: RequestId, params: ThreadShellCommandParams, } | { "method": "thread/approveGuardianDeniedAction", id: RequestId, params: ThreadApproveGuardianDeniedActionParams, } | { "method": "thread/rollback", id: RequestId, params: ThreadRollbackParams, } | { "method": "thread/list", id: RequestId, params: ThreadListParams, } | { "method": "thread/loaded/list", id: RequestId, params: ThreadLoadedListParams, } | { "method": "thread/read", id: RequestId, params: ThreadReadParams, } | { "method": "thread/inject_items", id: RequestId, params: ThreadInjectItemsParams, } | { "method": "skills/list", id: RequestId, params: SkillsListParams, } | { "method": "skills/extraRoots/set", id: RequestId, params: SkillsExtraRootsSetParams, } | { "method": "hooks/list", id: RequestId, params: HooksListParams, } | { "method": "marketplace/add", id: RequestId, params: MarketplaceAddParams, } | { "method": "marketplace/remove", id: RequestId, params: MarketplaceRemoveParams, } | { "method": "marketplace/upgrade", id: RequestId, params: MarketplaceUpgradeParams, } | { "method": "plugin/list", id: RequestId, params: PluginListParams, } | { "method": "plugin/installed", id: RequestId, params: PluginInstalledParams, } | { "method": "plugin/read", id: RequestId, params: PluginReadParams, } | { "method": "plugin/skill/read", id: RequestId, params: PluginSkillReadParams, } | { "method": "plugin/share/save", id: RequestId, params: PluginShareSaveParams, } | { "method": "plugin/share/updateTargets", id: RequestId, params: PluginShareUpdateTargetsParams, } | { "method": "plugin/share/list", id: RequestId, params: PluginShareListParams, } | { "method": "plugin/share/checkout", id: RequestId, params: PluginShareCheckoutParams, } | { "method": "plugin/share/delete", id: RequestId, params: PluginShareDeleteParams, } | { "method": "app/list", id: RequestId, params: AppsListParams, } | { "method": "fs/readFile", id: RequestId, params: FsReadFileParams, } | { "method": "fs/writeFile", id: RequestId, params: FsWriteFileParams, } | { "method": "fs/createDirectory", id: RequestId, params: FsCreateDirectoryParams, } | { "method": "fs/getMetadata", id: RequestId, params: FsGetMetadataParams, } | { "method": "fs/readDirectory", id: RequestId, params: FsReadDirectoryParams, } | { "method": "fs/remove", id: RequestId, params: FsRemoveParams, } | { "method": "fs/copy", id: RequestId, params: FsCopyParams, } | { "method": "fs/watch", id: RequestId, params: FsWatchParams, } | { "method": "fs/unwatch", id: RequestId, params: FsUnwatchParams, } | { "method": "skills/config/write", id: RequestId, params: SkillsConfigWriteParams, } | { "method": "plugin/install", id: RequestId, params: PluginInstallParams, } | { "method": "plugin/uninstall", id: RequestId, params: PluginUninstallParams, } | { "method": "turn/start", id: RequestId, params: TurnStartParams, } | { "method": "turn/steer", id: RequestId, params: TurnSteerParams, } | { "method": "turn/interrupt", id: RequestId, params: TurnInterruptParams, } | { "method": "review/start", id: RequestId, params: ReviewStartParams, } | { "method": "model/list", id: RequestId, params: ModelListParams, } | { "method": "modelProvider/capabilities/read", id: RequestId, params: ModelProviderCapabilitiesReadParams, } | { "method": "experimentalFeature/list", id: RequestId, params: ExperimentalFeatureListParams, } | { "method": "permissionProfile/list", id: RequestId, params: PermissionProfileListParams, } | { "method": "experimentalFeature/enablement/set", id: RequestId, params: ExperimentalFeatureEnablementSetParams, } | { "method": "mcpServer/oauth/login", id: RequestId, params: McpServerOauthLoginParams, } | { "method": "config/mcpServer/reload", id: RequestId, params: undefined, } | { "method": "mcpServerStatus/list", id: RequestId, params: ListMcpServerStatusParams, } | { "method": "mcpServer/resource/read", id: RequestId, params: McpResourceReadParams, } | { "method": "mcpServer/tool/call", id: RequestId, params: McpServerToolCallParams, } | { "method": "windowsSandbox/setupStart", id: RequestId, params: WindowsSandboxSetupStartParams, } | { "method": "windowsSandbox/readiness", id: RequestId, params: undefined, } | { "method": "account/login/start", id: RequestId, params: LoginAccountParams, } | { "method": "account/login/cancel", id: RequestId, params: CancelLoginAccountParams, } | { "method": "account/logout", id: RequestId, params: undefined, } | { "method": "account/rateLimits/read", id: RequestId, params: undefined, } | { "method": "account/usage/read", id: RequestId, params: undefined, } | { "method": "account/sendAddCreditsNudgeEmail", id: RequestId, params: SendAddCreditsNudgeEmailParams, } | { "method": "feedback/upload", id: RequestId, params: FeedbackUploadParams, } | { "method": "command/exec", id: RequestId, params: CommandExecParams, } | { "method": "command/exec/write", id: RequestId, params: CommandExecWriteParams, } | { "method": "command/exec/terminate", id: RequestId, params: CommandExecTerminateParams, } | { "method": "command/exec/resize", id: RequestId, params: CommandExecResizeParams, } | { "method": "config/read", id: RequestId, params: ConfigReadParams, } | { "method": "externalAgentConfig/detect", id: RequestId, params: ExternalAgentConfigDetectParams, } | { "method": "externalAgentConfig/import", id: RequestId, params: ExternalAgentConfigImportParams, } | { "method": "config/value/write", id: RequestId, params: ConfigValueWriteParams, } | { "method": "config/batchWrite", id: RequestId, params: ConfigBatchWriteParams, } | { "method": "configRequirements/read", id: RequestId, params: undefined, } | { "method": "account/read", id: RequestId, params: GetAccountParams, } | { "method": "getConversationSummary", id: RequestId, params: GetConversationSummaryParams, } | { "method": "gitDiffToRemote", id: RequestId, params: GitDiffToRemoteParams, } | { "method": "getAuthStatus", id: RequestId, params: GetAuthStatusParams, } | { "method": "fuzzyFileSearch", id: RequestId, params: FuzzyFileSearchParams, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/ServerNotification.ts b/codex-rs/app-server-protocol/schema/typescript/ServerNotification.ts index 2520c71c3c..7cae63a553 100644 --- a/codex-rs/app-server-protocol/schema/typescript/ServerNotification.ts +++ b/codex-rs/app-server-protocol/schema/typescript/ServerNotification.ts @@ -43,6 +43,7 @@ import type { SkillsChangedNotification } from "./v2/SkillsChangedNotification"; import type { TerminalInteractionNotification } from "./v2/TerminalInteractionNotification"; import type { ThreadArchivedNotification } from "./v2/ThreadArchivedNotification"; import type { ThreadClosedNotification } from "./v2/ThreadClosedNotification"; +import type { ThreadDeletedNotification } from "./v2/ThreadDeletedNotification"; import type { ThreadGoalClearedNotification } from "./v2/ThreadGoalClearedNotification"; import type { ThreadGoalUpdatedNotification } from "./v2/ThreadGoalUpdatedNotification"; import type { ThreadNameUpdatedNotification } from "./v2/ThreadNameUpdatedNotification"; @@ -71,4 +72,4 @@ import type { WindowsWorldWritableWarningNotification } from "./v2/WindowsWorldW /** * Notification sent from the server to the client. */ -export type ServerNotification = { "method": "error", "params": ErrorNotification } | { "method": "thread/started", "params": ThreadStartedNotification } | { "method": "thread/status/changed", "params": ThreadStatusChangedNotification } | { "method": "thread/archived", "params": ThreadArchivedNotification } | { "method": "thread/unarchived", "params": ThreadUnarchivedNotification } | { "method": "thread/closed", "params": ThreadClosedNotification } | { "method": "skills/changed", "params": SkillsChangedNotification } | { "method": "thread/name/updated", "params": ThreadNameUpdatedNotification } | { "method": "thread/goal/updated", "params": ThreadGoalUpdatedNotification } | { "method": "thread/goal/cleared", "params": ThreadGoalClearedNotification } | { "method": "thread/settings/updated", "params": ThreadSettingsUpdatedNotification } | { "method": "thread/tokenUsage/updated", "params": ThreadTokenUsageUpdatedNotification } | { "method": "turn/started", "params": TurnStartedNotification } | { "method": "hook/started", "params": HookStartedNotification } | { "method": "turn/completed", "params": TurnCompletedNotification } | { "method": "hook/completed", "params": HookCompletedNotification } | { "method": "turn/diff/updated", "params": TurnDiffUpdatedNotification } | { "method": "turn/plan/updated", "params": TurnPlanUpdatedNotification } | { "method": "item/started", "params": ItemStartedNotification } | { "method": "item/autoApprovalReview/started", "params": ItemGuardianApprovalReviewStartedNotification } | { "method": "item/autoApprovalReview/completed", "params": ItemGuardianApprovalReviewCompletedNotification } | { "method": "item/completed", "params": ItemCompletedNotification } | { "method": "rawResponseItem/completed", "params": RawResponseItemCompletedNotification } | { "method": "item/agentMessage/delta", "params": AgentMessageDeltaNotification } | { "method": "item/plan/delta", "params": PlanDeltaNotification } | { "method": "command/exec/outputDelta", "params": CommandExecOutputDeltaNotification } | { "method": "process/outputDelta", "params": ProcessOutputDeltaNotification } | { "method": "process/exited", "params": ProcessExitedNotification } | { "method": "item/commandExecution/outputDelta", "params": CommandExecutionOutputDeltaNotification } | { "method": "item/commandExecution/terminalInteraction", "params": TerminalInteractionNotification } | { "method": "item/fileChange/outputDelta", "params": FileChangeOutputDeltaNotification } | { "method": "item/fileChange/patchUpdated", "params": FileChangePatchUpdatedNotification } | { "method": "serverRequest/resolved", "params": ServerRequestResolvedNotification } | { "method": "item/mcpToolCall/progress", "params": McpToolCallProgressNotification } | { "method": "mcpServer/oauthLogin/completed", "params": McpServerOauthLoginCompletedNotification } | { "method": "mcpServer/startupStatus/updated", "params": McpServerStatusUpdatedNotification } | { "method": "account/updated", "params": AccountUpdatedNotification } | { "method": "account/rateLimits/updated", "params": AccountRateLimitsUpdatedNotification } | { "method": "app/list/updated", "params": AppListUpdatedNotification } | { "method": "remoteControl/status/changed", "params": RemoteControlStatusChangedNotification } | { "method": "externalAgentConfig/import/completed", "params": ExternalAgentConfigImportCompletedNotification } | { "method": "fs/changed", "params": FsChangedNotification } | { "method": "item/reasoning/summaryTextDelta", "params": ReasoningSummaryTextDeltaNotification } | { "method": "item/reasoning/summaryPartAdded", "params": ReasoningSummaryPartAddedNotification } | { "method": "item/reasoning/textDelta", "params": ReasoningTextDeltaNotification } | { "method": "thread/compacted", "params": ContextCompactedNotification } | { "method": "model/rerouted", "params": ModelReroutedNotification } | { "method": "model/verification", "params": ModelVerificationNotification } | { "method": "turn/moderationMetadata", "params": TurnModerationMetadataNotification } | { "method": "warning", "params": WarningNotification } | { "method": "guardianWarning", "params": GuardianWarningNotification } | { "method": "deprecationNotice", "params": DeprecationNoticeNotification } | { "method": "configWarning", "params": ConfigWarningNotification } | { "method": "fuzzyFileSearch/sessionUpdated", "params": FuzzyFileSearchSessionUpdatedNotification } | { "method": "fuzzyFileSearch/sessionCompleted", "params": FuzzyFileSearchSessionCompletedNotification } | { "method": "thread/realtime/started", "params": ThreadRealtimeStartedNotification } | { "method": "thread/realtime/itemAdded", "params": ThreadRealtimeItemAddedNotification } | { "method": "thread/realtime/transcript/delta", "params": ThreadRealtimeTranscriptDeltaNotification } | { "method": "thread/realtime/transcript/done", "params": ThreadRealtimeTranscriptDoneNotification } | { "method": "thread/realtime/outputAudio/delta", "params": ThreadRealtimeOutputAudioDeltaNotification } | { "method": "thread/realtime/sdp", "params": ThreadRealtimeSdpNotification } | { "method": "thread/realtime/error", "params": ThreadRealtimeErrorNotification } | { "method": "thread/realtime/closed", "params": ThreadRealtimeClosedNotification } | { "method": "windows/worldWritableWarning", "params": WindowsWorldWritableWarningNotification } | { "method": "windowsSandbox/setupCompleted", "params": WindowsSandboxSetupCompletedNotification } | { "method": "account/login/completed", "params": AccountLoginCompletedNotification }; +export type ServerNotification = { "method": "error", "params": ErrorNotification } | { "method": "thread/started", "params": ThreadStartedNotification } | { "method": "thread/status/changed", "params": ThreadStatusChangedNotification } | { "method": "thread/archived", "params": ThreadArchivedNotification } | { "method": "thread/deleted", "params": ThreadDeletedNotification } | { "method": "thread/unarchived", "params": ThreadUnarchivedNotification } | { "method": "thread/closed", "params": ThreadClosedNotification } | { "method": "skills/changed", "params": SkillsChangedNotification } | { "method": "thread/name/updated", "params": ThreadNameUpdatedNotification } | { "method": "thread/goal/updated", "params": ThreadGoalUpdatedNotification } | { "method": "thread/goal/cleared", "params": ThreadGoalClearedNotification } | { "method": "thread/settings/updated", "params": ThreadSettingsUpdatedNotification } | { "method": "thread/tokenUsage/updated", "params": ThreadTokenUsageUpdatedNotification } | { "method": "turn/started", "params": TurnStartedNotification } | { "method": "hook/started", "params": HookStartedNotification } | { "method": "turn/completed", "params": TurnCompletedNotification } | { "method": "hook/completed", "params": HookCompletedNotification } | { "method": "turn/diff/updated", "params": TurnDiffUpdatedNotification } | { "method": "turn/plan/updated", "params": TurnPlanUpdatedNotification } | { "method": "item/started", "params": ItemStartedNotification } | { "method": "item/autoApprovalReview/started", "params": ItemGuardianApprovalReviewStartedNotification } | { "method": "item/autoApprovalReview/completed", "params": ItemGuardianApprovalReviewCompletedNotification } | { "method": "item/completed", "params": ItemCompletedNotification } | { "method": "rawResponseItem/completed", "params": RawResponseItemCompletedNotification } | { "method": "item/agentMessage/delta", "params": AgentMessageDeltaNotification } | { "method": "item/plan/delta", "params": PlanDeltaNotification } | { "method": "command/exec/outputDelta", "params": CommandExecOutputDeltaNotification } | { "method": "process/outputDelta", "params": ProcessOutputDeltaNotification } | { "method": "process/exited", "params": ProcessExitedNotification } | { "method": "item/commandExecution/outputDelta", "params": CommandExecutionOutputDeltaNotification } | { "method": "item/commandExecution/terminalInteraction", "params": TerminalInteractionNotification } | { "method": "item/fileChange/outputDelta", "params": FileChangeOutputDeltaNotification } | { "method": "item/fileChange/patchUpdated", "params": FileChangePatchUpdatedNotification } | { "method": "serverRequest/resolved", "params": ServerRequestResolvedNotification } | { "method": "item/mcpToolCall/progress", "params": McpToolCallProgressNotification } | { "method": "mcpServer/oauthLogin/completed", "params": McpServerOauthLoginCompletedNotification } | { "method": "mcpServer/startupStatus/updated", "params": McpServerStatusUpdatedNotification } | { "method": "account/updated", "params": AccountUpdatedNotification } | { "method": "account/rateLimits/updated", "params": AccountRateLimitsUpdatedNotification } | { "method": "app/list/updated", "params": AppListUpdatedNotification } | { "method": "remoteControl/status/changed", "params": RemoteControlStatusChangedNotification } | { "method": "externalAgentConfig/import/completed", "params": ExternalAgentConfigImportCompletedNotification } | { "method": "fs/changed", "params": FsChangedNotification } | { "method": "item/reasoning/summaryTextDelta", "params": ReasoningSummaryTextDeltaNotification } | { "method": "item/reasoning/summaryPartAdded", "params": ReasoningSummaryPartAddedNotification } | { "method": "item/reasoning/textDelta", "params": ReasoningTextDeltaNotification } | { "method": "thread/compacted", "params": ContextCompactedNotification } | { "method": "model/rerouted", "params": ModelReroutedNotification } | { "method": "model/verification", "params": ModelVerificationNotification } | { "method": "turn/moderationMetadata", "params": TurnModerationMetadataNotification } | { "method": "warning", "params": WarningNotification } | { "method": "guardianWarning", "params": GuardianWarningNotification } | { "method": "deprecationNotice", "params": DeprecationNoticeNotification } | { "method": "configWarning", "params": ConfigWarningNotification } | { "method": "fuzzyFileSearch/sessionUpdated", "params": FuzzyFileSearchSessionUpdatedNotification } | { "method": "fuzzyFileSearch/sessionCompleted", "params": FuzzyFileSearchSessionCompletedNotification } | { "method": "thread/realtime/started", "params": ThreadRealtimeStartedNotification } | { "method": "thread/realtime/itemAdded", "params": ThreadRealtimeItemAddedNotification } | { "method": "thread/realtime/transcript/delta", "params": ThreadRealtimeTranscriptDeltaNotification } | { "method": "thread/realtime/transcript/done", "params": ThreadRealtimeTranscriptDoneNotification } | { "method": "thread/realtime/outputAudio/delta", "params": ThreadRealtimeOutputAudioDeltaNotification } | { "method": "thread/realtime/sdp", "params": ThreadRealtimeSdpNotification } | { "method": "thread/realtime/error", "params": ThreadRealtimeErrorNotification } | { "method": "thread/realtime/closed", "params": ThreadRealtimeClosedNotification } | { "method": "windows/worldWritableWarning", "params": WindowsWorldWritableWarningNotification } | { "method": "windowsSandbox/setupCompleted", "params": WindowsSandboxSetupCompletedNotification } | { "method": "account/login/completed", "params": AccountLoginCompletedNotification }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadDeleteParams.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadDeleteParams.ts new file mode 100644 index 0000000000..909ccda715 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadDeleteParams.ts @@ -0,0 +1,5 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type ThreadDeleteParams = { threadId: string, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadDeleteResponse.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadDeleteResponse.ts new file mode 100644 index 0000000000..1af1c3077f --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadDeleteResponse.ts @@ -0,0 +1,5 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type ThreadDeleteResponse = Record; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadDeletedNotification.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadDeletedNotification.ts new file mode 100644 index 0000000000..5122a2229c --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadDeletedNotification.ts @@ -0,0 +1,5 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type ThreadDeletedNotification = { threadId: string, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/index.ts b/codex-rs/app-server-protocol/schema/typescript/v2/index.ts index 71f4c67cd0..b4cd88aadf 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/index.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/index.ts @@ -374,6 +374,9 @@ export type { ThreadArchivedNotification } from "./ThreadArchivedNotification"; export type { ThreadClosedNotification } from "./ThreadClosedNotification"; export type { ThreadCompactStartParams } from "./ThreadCompactStartParams"; export type { ThreadCompactStartResponse } from "./ThreadCompactStartResponse"; +export type { ThreadDeleteParams } from "./ThreadDeleteParams"; +export type { ThreadDeleteResponse } from "./ThreadDeleteResponse"; +export type { ThreadDeletedNotification } from "./ThreadDeletedNotification"; export type { ThreadForkParams } from "./ThreadForkParams"; export type { ThreadForkResponse } from "./ThreadForkResponse"; export type { ThreadGoal } from "./ThreadGoal"; diff --git a/codex-rs/app-server-protocol/src/protocol/common.rs b/codex-rs/app-server-protocol/src/protocol/common.rs index 8bd9a2c175..5526f5c548 100644 --- a/codex-rs/app-server-protocol/src/protocol/common.rs +++ b/codex-rs/app-server-protocol/src/protocol/common.rs @@ -480,6 +480,11 @@ client_request_definitions! { serialization: thread_id(params.thread_id), response: v2::ThreadArchiveResponse, }, + ThreadDelete => "thread/delete" { + params: v2::ThreadDeleteParams, + serialization: thread_id(params.thread_id), + response: v2::ThreadDeleteResponse, + }, ThreadUnsubscribe => "thread/unsubscribe" { params: v2::ThreadUnsubscribeParams, serialization: thread_id(params.thread_id), @@ -1534,6 +1539,7 @@ server_notification_definitions! { ThreadStarted => "thread/started" (v2::ThreadStartedNotification), ThreadStatusChanged => "thread/status/changed" (v2::ThreadStatusChangedNotification), ThreadArchived => "thread/archived" (v2::ThreadArchivedNotification), + ThreadDeleted => "thread/deleted" (v2::ThreadDeletedNotification), ThreadUnarchived => "thread/unarchived" (v2::ThreadUnarchivedNotification), ThreadClosed => "thread/closed" (v2::ThreadClosedNotification), SkillsChanged => "skills/changed" (v2::SkillsChangedNotification), diff --git a/codex-rs/app-server-protocol/src/protocol/v2/thread.rs b/codex-rs/app-server-protocol/src/protocol/v2/thread.rs index 9e1770fde0..5b412c2edd 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/thread.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/thread.rs @@ -594,6 +594,18 @@ pub struct ThreadArchiveParams { #[ts(export_to = "v2/")] pub struct ThreadArchiveResponse {} +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadDeleteParams { + pub thread_id: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadDeleteResponse {} + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] @@ -1381,6 +1393,13 @@ pub struct ThreadArchivedNotification { pub thread_id: String, } +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadDeletedNotification { + pub thread_id: String, +} + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index 766a5b0478..0532ffa706 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -151,6 +151,7 @@ Example with notification opt-out: - `thread/settings/updated` — experimental notification emitted to subscribed clients when a loaded thread’s effective next-turn settings change; includes `threadId` and the full `threadSettings`. - `thread/status/changed` — notification emitted when a loaded thread’s status changes (`threadId` + new `status`). - `thread/archive` — move a thread’s rollout file into the archived directory and attempt to move any spawned descendant thread rollout files; returns `{}` on success and emits `thread/archived` for each archived thread. +- `thread/delete` — hard-delete an active or archived thread and any spawned descendant threads; returns `{}` on success and emits `thread/deleted` for each deleted thread. - `thread/unsubscribe` — unsubscribe this connection from thread turn/item events. If this was the last subscriber, the server keeps the thread loaded and unloads it only after it has had no subscribers and no thread activity for 30 minutes, then emits `thread/closed`. - `thread/name/set` — set or update a thread’s user-facing name for either a loaded thread or a persisted rollout; returns `{}` on success and emits `thread/name/updated` to initialized, opted-in clients. Thread names are not required to be unique; name lookups resolve to the most recently updated thread. - `thread/unarchive` — move an archived rollout file back into the sessions directory; returns the restored `thread` on success and emits `thread/unarchived`. @@ -612,6 +613,16 @@ Use `thread/archive` to move the persisted rollout (stored as a JSONL file on di An archived thread will not appear in `thread/list` unless `archived` is set to `true`. +### Example: Delete a thread + +Use `thread/delete` to hard-delete a thread and its spawned descendant threads. Existing rollout files and associated metadata must be removed before the request succeeds; missing rollout files are treated as already deleted. + +```json +{ "method": "thread/delete", "id": 23, "params": { "threadId": "thr_b" } } +{ "id": 23, "result": {} } +{ "method": "thread/deleted", "params": { "threadId": "thr_b" } } +``` + ### Example: Unarchive a thread Use `thread/unarchive` to move an archived rollout back into the sessions directory. diff --git a/codex-rs/app-server/src/message_processor.rs b/codex-rs/app-server/src/message_processor.rs index ce239a2718..30e57208e7 100644 --- a/codex-rs/app-server/src/message_processor.rs +++ b/codex-rs/app-server/src/message_processor.rs @@ -400,7 +400,7 @@ impl MessageProcessor { Arc::clone(&thread_manager), Arc::clone(&config), feedback, - log_db, + log_db.clone(), state_db.clone(), ); let git_processor = GitRequestProcessor::new(); @@ -454,6 +454,7 @@ impl MessageProcessor { Arc::clone(&thread_list_state_permit), thread_goal_processor.clone(), state_db, + log_db, Arc::clone(&skills_watcher), ); let turn_processor = TurnRequestProcessor::new( @@ -1071,6 +1072,11 @@ impl MessageProcessor { .thread_archive(request_id.clone(), params) .await } + ClientRequest::ThreadDelete { params, .. } => { + self.thread_processor + .thread_delete(request_id.clone(), params) + .await + } ClientRequest::ThreadIncrementElicitation { params, .. } => { self.thread_processor .thread_increment_elicitation(params) diff --git a/codex-rs/app-server/src/request_processors.rs b/codex-rs/app-server/src/request_processors.rs index 2bce5b8ddd..83e704cdfd 100644 --- a/codex-rs/app-server/src/request_processors.rs +++ b/codex-rs/app-server/src/request_processors.rs @@ -183,6 +183,9 @@ use codex_app_server_protocol::ThreadCompactStartParams; use codex_app_server_protocol::ThreadCompactStartResponse; use codex_app_server_protocol::ThreadDecrementElicitationParams; use codex_app_server_protocol::ThreadDecrementElicitationResponse; +use codex_app_server_protocol::ThreadDeleteParams; +use codex_app_server_protocol::ThreadDeleteResponse; +use codex_app_server_protocol::ThreadDeletedNotification; use codex_app_server_protocol::ThreadForkParams; use codex_app_server_protocol::ThreadForkResponse; use codex_app_server_protocol::ThreadGoal; @@ -419,6 +422,7 @@ use codex_rollout::state_db::reconcile_rollout; use codex_state::ThreadMetadata; use codex_state::log_db::LogDbLayer; use codex_thread_store::ArchiveThreadParams as StoreArchiveThreadParams; +use codex_thread_store::DeleteThreadParams as StoreDeleteThreadParams; use codex_thread_store::GitInfoPatch as StoreGitInfoPatch; use codex_thread_store::ListThreadsParams as StoreListThreadsParams; use codex_thread_store::LocalThreadStore; @@ -539,6 +543,7 @@ fn resolve_runtime_workspace_roots(workspace_roots: Vec) -> Vec mod config_errors; mod request_errors; +mod thread_delete; mod thread_goal_processor; mod thread_lifecycle; mod thread_resume_redaction; diff --git a/codex-rs/app-server/src/request_processors/thread_delete.rs b/codex-rs/app-server/src/request_processors/thread_delete.rs new file mode 100644 index 0000000000..fc7e58573c --- /dev/null +++ b/codex-rs/app-server/src/request_processors/thread_delete.rs @@ -0,0 +1,188 @@ +//! `thread/delete` request handling. + +use super::thread_processor::core_thread_write_error; +use super::thread_processor::unsupported_thread_store_operation; +use super::*; + +impl ThreadRequestProcessor { + pub(crate) async fn thread_delete( + &self, + request_id: ConnectionRequestId, + params: ThreadDeleteParams, + ) -> Result, JSONRPCErrorError> { + let mut deleted_thread_ids = Vec::new(); + let result = { + let _thread_list_state_permit = self.acquire_thread_list_state_permit().await?; + self.thread_delete_response(params, &mut deleted_thread_ids) + .await + }; + match result { + Ok(response) => { + self.outgoing + .send_response(request_id.clone(), response) + .await; + self.send_thread_deleted_notifications(deleted_thread_ids) + .await; + Ok(None) + } + Err(error) => Err(error), + } + } + + async fn thread_delete_response( + &self, + params: ThreadDeleteParams, + deleted_thread_ids: &mut Vec, + ) -> Result { + let thread_id = ThreadId::from_string(¶ms.thread_id) + .map_err(|err| invalid_request(format!("invalid thread id: {err}")))?; + + let mut thread_ids = self.state_db_spawn_subtree_thread_ids(thread_id).await?; + let mut seen = thread_ids.iter().copied().collect::>(); + + match self + .thread_manager + .list_agent_subtree_thread_ids(thread_id) + .await + { + Ok(live_thread_ids) => { + for live_thread_id in live_thread_ids { + if seen.insert(live_thread_id) { + thread_ids.push(live_thread_id); + } + } + } + Err(err) => return Err(core_thread_write_error("delete thread", err)), + } + + self.validate_root_thread_delete(thread_id, thread_ids.len() > 1) + .await?; + for thread_id_to_delete in thread_ids.iter().copied() { + self.prepare_thread_for_delete(thread_id_to_delete).await; + } + + let mut delete_order: Vec<_> = thread_ids.iter().skip(1).rev().copied().collect(); + delete_order.push(thread_id); + + for thread_id_to_delete in delete_order.iter().copied() { + match self + .thread_store + .delete_thread(StoreDeleteThreadParams { + thread_id: thread_id_to_delete, + }) + .await + { + Ok(()) => {} + Err(ThreadStoreError::ThreadNotFound { .. }) => { + warn!( + "thread {thread_id_to_delete} was already missing while deleting {thread_id}" + ); + } + Err(err) => { + return Err(thread_store_delete_error(err)); + } + } + } + + if let Some(state_db) = self.state_db.as_ref() { + state_db + .delete_threads_strict(thread_ids.as_slice()) + .await + .map_err(|err| { + internal_error(format!( + "failed to delete app-server state for {thread_id}: {err}" + )) + })?; + } + + deleted_thread_ids.extend( + delete_order + .into_iter() + .map(|thread_id| thread_id.to_string()), + ); + Ok(ThreadDeleteResponse {}) + } + + async fn send_thread_deleted_notifications(&self, deleted_thread_ids: Vec) { + for thread_id in deleted_thread_ids { + self.outgoing + .send_server_notification(ServerNotification::ThreadDeleted( + ThreadDeletedNotification { thread_id }, + )) + .await; + } + } + + async fn validate_root_thread_delete( + &self, + thread_id: ThreadId, + has_descendants: bool, + ) -> Result<(), JSONRPCErrorError> { + if let Ok(thread) = self.thread_manager.get_thread(thread_id).await { + if !thread.config_snapshot().await.ephemeral { + return Ok(()); + } + return Err(invalid_request(format!( + "thread is not persisted and cannot be deleted: {thread_id}" + ))); + } + match self + .thread_store + .read_thread(StoreReadThreadParams { + thread_id, + include_archived: true, + include_history: false, + }) + .await + { + Ok(_) => Ok(()), + Err(ThreadStoreError::ThreadNotFound { .. }) => { + if has_descendants { + return Ok(()); + } + let Some(state_db) = self.state_db.as_ref() else { + return Err(thread_store_delete_error( + ThreadStoreError::ThreadNotFound { thread_id }, + )); + }; + if state_db + .get_thread(thread_id) + .await + .map_err(|err| { + internal_error(format!( + "failed to read app-server state for {thread_id}: {err}" + )) + })? + .is_some() + { + Ok(()) + } else { + Err(thread_store_delete_error( + ThreadStoreError::ThreadNotFound { thread_id }, + )) + } + } + Err(err) => Err(thread_store_delete_error(err)), + } + } + + async fn prepare_thread_for_delete(&self, thread_id: ThreadId) { + self.prepare_thread_for_removal(thread_id, "delete").await; + if let Some(log_db) = self.log_db.as_ref() { + log_db.flush().await; + } + } +} + +fn thread_store_delete_error(err: ThreadStoreError) -> JSONRPCErrorError { + match err { + ThreadStoreError::ThreadNotFound { thread_id } => { + invalid_request(format!("thread not found: {thread_id}")) + } + ThreadStoreError::InvalidRequest { message } => invalid_request(message), + ThreadStoreError::Unsupported { operation } => { + unsupported_thread_store_operation(operation) + } + err => internal_error(format!("failed to delete thread: {err}")), + } +} diff --git a/codex-rs/app-server/src/request_processors/thread_processor.rs b/codex-rs/app-server/src/request_processors/thread_processor.rs index b10358d8d2..58719d55ba 100644 --- a/codex-rs/app-server/src/request_processors/thread_processor.rs +++ b/codex-rs/app-server/src/request_processors/thread_processor.rs @@ -325,6 +325,7 @@ pub(crate) struct ThreadRequestProcessor { pub(super) thread_list_state_permit: Arc, pub(super) thread_goal_processor: ThreadGoalRequestProcessor, pub(super) state_db: Option, + pub(super) log_db: Option, pub(super) background_tasks: TaskTracker, pub(super) skills_watcher: Arc, } @@ -356,6 +357,7 @@ impl ThreadRequestProcessor { thread_list_state_permit: Arc, thread_goal_processor: ThreadGoalRequestProcessor, state_db: Option, + log_db: Option, skills_watcher: Arc, ) -> Self { Self { @@ -372,6 +374,7 @@ impl ThreadRequestProcessor { thread_list_state_permit, thread_goal_processor, state_db, + log_db, background_tasks: TaskTracker::new(), skills_watcher, } @@ -696,7 +699,7 @@ impl ThreadRequestProcessor { Ok((thread_id, thread)) } - async fn acquire_thread_list_state_permit( + pub(super) async fn acquire_thread_list_state_permit( &self, ) -> Result, JSONRPCErrorError> { self.thread_list_state_permit @@ -768,6 +771,10 @@ impl ThreadRequestProcessor { } async fn prepare_thread_for_archive(&self, thread_id: ThreadId) { + self.prepare_thread_for_removal(thread_id, "archive").await; + } + + pub(super) async fn prepare_thread_for_removal(&self, thread_id: ThreadId, operation: &str) { let removed_conversation = self.thread_manager.remove_thread(&thread_id).await; if let Some(conversation) = removed_conversation { info!("thread {thread_id} was active; shutting down"); @@ -775,11 +782,11 @@ impl ThreadRequestProcessor { ThreadShutdownResult::Complete => {} ThreadShutdownResult::SubmitFailed => { error!( - "failed to submit Shutdown to thread {thread_id}; proceeding with archive" + "failed to submit Shutdown to thread {thread_id}; proceeding with {operation}" ); } ThreadShutdownResult::TimedOut => { - warn!("thread {thread_id} shutdown timed out; proceeding with archive"); + warn!("thread {thread_id} shutdown timed out; proceeding with {operation}"); } } } @@ -1312,23 +1319,7 @@ impl ThreadRequestProcessor { let thread_id = ThreadId::from_string(¶ms.thread_id) .map_err(|err| invalid_request(format!("invalid session id: {err}")))?; - let mut thread_ids = vec![thread_id]; - if let Some(state_db_ctx) = self.state_db.as_ref() { - let descendants = state_db_ctx - .list_thread_spawn_descendants(thread_id) - .await - .map_err(|err| { - internal_error(format!( - "failed to list spawned descendants for session {thread_id}: {err}" - )) - })?; - let mut seen = HashSet::from([thread_id]); - for descendant_id in descendants { - if seen.insert(descendant_id) { - thread_ids.push(descendant_id); - } - } - } + let thread_ids = self.state_db_spawn_subtree_thread_ids(thread_id).await?; let mut archive_thread_ids = Vec::new(); match self @@ -1413,6 +1404,31 @@ impl ThreadRequestProcessor { Ok((ThreadArchiveResponse {}, archived_thread_ids)) } + pub(super) async fn state_db_spawn_subtree_thread_ids( + &self, + thread_id: ThreadId, + ) -> Result, JSONRPCErrorError> { + let mut thread_ids = vec![thread_id]; + let Some(state_db_ctx) = self.state_db.as_ref() else { + return Ok(thread_ids); + }; + let mut seen = HashSet::from([thread_id]); + let descendants = state_db_ctx + .list_thread_spawn_descendants(thread_id) + .await + .map_err(|err| { + internal_error(format!( + "failed to list spawned descendants for thread id {thread_id}: {err}" + )) + })?; + for descendant_id in descendants { + if seen.insert(descendant_id) { + thread_ids.push(descendant_id); + } + } + Ok(thread_ids) + } + async fn thread_increment_elicitation_inner( &self, params: ThreadIncrementElicitationParams, @@ -3909,7 +3925,7 @@ fn thread_read_view_error(err: ThreadReadViewError) -> JSONRPCErrorError { } } -fn unsupported_thread_store_operation(operation: &'static str) -> JSONRPCErrorError { +pub(super) fn unsupported_thread_store_operation(operation: &'static str) -> JSONRPCErrorError { method_not_found(format!("{operation} is not supported yet")) } @@ -4030,7 +4046,7 @@ fn conversation_summary_rollout_path_read_error( } } -fn core_thread_write_error(operation: &str, err: CodexErr) -> JSONRPCErrorError { +pub(super) fn core_thread_write_error(operation: &str, err: CodexErr) -> JSONRPCErrorError { match err { CodexErr::ThreadNotFound(thread_id) => { invalid_request(format!("thread not found: {thread_id}")) diff --git a/codex-rs/app-server/tests/common/test_app_server.rs b/codex-rs/app-server/tests/common/test_app_server.rs index 646de6ad21..91c42e2fe4 100644 --- a/codex-rs/app-server/tests/common/test_app_server.rs +++ b/codex-rs/app-server/tests/common/test_app_server.rs @@ -79,6 +79,7 @@ use codex_app_server_protocol::SkillsExtraRootsSetParams; use codex_app_server_protocol::SkillsListParams; use codex_app_server_protocol::ThreadArchiveParams; use codex_app_server_protocol::ThreadCompactStartParams; +use codex_app_server_protocol::ThreadDeleteParams; use codex_app_server_protocol::ThreadForkParams; use codex_app_server_protocol::ThreadInjectItemsParams; use codex_app_server_protocol::ThreadListParams; @@ -456,6 +457,15 @@ impl TestAppServer { self.send_request("thread/archive", params).await } + /// Send a `thread/delete` JSON-RPC request. + pub async fn send_thread_delete_request( + &mut self, + params: ThreadDeleteParams, + ) -> anyhow::Result { + let params = Some(serde_json::to_value(params)?); + self.send_request("thread/delete", params).await + } + /// Send a `thread/name/set` JSON-RPC request. pub async fn send_thread_set_name_request( &mut self, diff --git a/codex-rs/app-server/tests/suite/v2/mod.rs b/codex-rs/app-server/tests/suite/v2/mod.rs index 77d4b37589..e038a131e4 100644 --- a/codex-rs/app-server/tests/suite/v2/mod.rs +++ b/codex-rs/app-server/tests/suite/v2/mod.rs @@ -50,6 +50,7 @@ mod review; mod safety_check_downgrade; mod skills_list; mod thread_archive; +mod thread_delete; mod thread_fork; mod thread_inject_items; mod thread_list; diff --git a/codex-rs/app-server/tests/suite/v2/remote_thread_store.rs b/codex-rs/app-server/tests/suite/v2/remote_thread_store.rs index ca2121dd4c..18d1dcd43f 100644 --- a/codex-rs/app-server/tests/suite/v2/remote_thread_store.rs +++ b/codex-rs/app-server/tests/suite/v2/remote_thread_store.rs @@ -20,6 +20,7 @@ use std::sync::Arc; use anyhow::Result; use app_test_support::create_mock_responses_server_repeating_assistant; use codex_app_server::in_process; +use codex_app_server::in_process::InProcessClientHandle; use codex_app_server::in_process::InProcessServerEvent; use codex_app_server::in_process::InProcessStartArgs; use codex_app_server_protocol::ClientInfo; @@ -27,6 +28,8 @@ use codex_app_server_protocol::ClientRequest; use codex_app_server_protocol::InitializeParams; use codex_app_server_protocol::RequestId; use codex_app_server_protocol::ServerNotification; +use codex_app_server_protocol::ThreadDeleteParams; +use codex_app_server_protocol::ThreadDeleteResponse; use codex_app_server_protocol::ThreadListParams; use codex_app_server_protocol::ThreadListResponse; use codex_app_server_protocol::ThreadResumeParams; @@ -42,8 +45,14 @@ use codex_core::config::Config; use codex_core::config::ConfigBuilder; use codex_exec_server::EnvironmentManager; use codex_feedback::CodexFeedback; +use codex_protocol::ThreadId; +use codex_protocol::models::BaseInstructions; use codex_protocol::protocol::SessionSource; +use codex_protocol::protocol::ThreadMemoryMode; +use codex_thread_store::CreateThreadParams as StoreCreateThreadParams; use codex_thread_store::InMemoryThreadStore; +use codex_thread_store::ThreadPersistenceMetadata; +use codex_thread_store::ThreadStore; use pretty_assertions::assert_eq; use tempfile::TempDir; use tokio::time::timeout; @@ -52,7 +61,7 @@ use uuid::Uuid; const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); #[tokio::test] -async fn thread_start_with_non_local_thread_store_does_not_create_local_persistence() -> Result<()> +async fn thread_delete_with_non_local_thread_store_does_not_create_local_persistence() -> Result<()> { let server = create_mock_responses_server_repeating_assistant("Done").await; let codex_home = TempDir::new()?; @@ -61,43 +70,10 @@ async fn thread_start_with_non_local_thread_store_does_not_create_local_persiste // here so this regression stays focused on thread persistence artifacts. create_config_toml_with_thread_store(codex_home.path(), &server.uri(), &store_id)?; - let loader_overrides = LoaderOverrides::without_managed_config_for_tests(); - let config = ConfigBuilder::default() - .codex_home(codex_home.path().to_path_buf()) - .fallback_cwd(Some(codex_home.path().to_path_buf())) - .loader_overrides(loader_overrides.clone()) - .build() - .await?; - let thread_store = InMemoryThreadStore::for_id(store_id.clone()); let _in_memory_store = InMemoryThreadStoreId { store_id }; - let mut client = in_process::start(InProcessStartArgs { - arg0_paths: Arg0DispatchPaths::default(), - config: Arc::new(config), - cli_overrides: Vec::new(), - loader_overrides, - strict_config: false, - cloud_config_bundle: CloudConfigBundleLoader::default(), - thread_config_loader: Arc::new(NoopThreadConfigLoader), - feedback: CodexFeedback::new(), - log_db: None, - state_db: None, - environment_manager: Arc::new(EnvironmentManager::default_for_tests()), - config_warnings: Vec::new(), - session_source: SessionSource::Cli, - enable_codex_api_key_env: false, - initialize: InitializeParams { - client_info: ClientInfo { - name: "codex-app-server-tests".to_string(), - title: None, - version: "0.1.0".to_string(), - }, - capabilities: None, - }, - channel_capacity: in_process::DEFAULT_IN_PROCESS_CHANNEL_CAPACITY, - }) - .await?; + let mut client = start_in_process_server(codex_home.path()).await?; let response = client .request(ClientRequest::ThreadStart { @@ -166,11 +142,39 @@ async fn thread_start_with_non_local_thread_store_does_not_create_local_persiste assert_eq!(data[0].id, thread.id); assert_eq!(data[0].path, None); + delete_thread(&client, /*request_id*/ 4, thread.id.clone()).await?; + let unloaded_thread_id = ThreadId::from_string(&Uuid::new_v4().to_string())?; + thread_store + .create_thread(StoreCreateThreadParams { + thread_id: unloaded_thread_id, + extra_config: None, + forked_from_id: None, + parent_thread_id: None, + source: SessionSource::Cli, + thread_source: None, + base_instructions: BaseInstructions::default(), + dynamic_tools: Vec::new(), + multi_agent_version: None, + metadata: ThreadPersistenceMetadata { + cwd: Some(codex_home.path().to_path_buf()), + model_provider: "mock_provider".to_string(), + memory_mode: ThreadMemoryMode::Enabled, + }, + }) + .await?; + delete_thread( + &client, + /*request_id*/ 5, + unloaded_thread_id.to_string(), + ) + .await?; + client.shutdown().await?; let calls = thread_store.calls().await; - assert_eq!(calls.create_thread, 1); + assert_eq!(calls.create_thread, 2); assert_eq!(calls.list_threads, 1); + assert_eq!(calls.delete_thread, 2); assert!( calls.append_items > 0, "turn/start should append rollout items through the injected store" @@ -269,10 +273,24 @@ async fn cold_thread_resume_reuses_non_local_history_probe() -> Result<()> { Ok(()) } +async fn start_in_process_server(codex_home: &Path) -> Result { + let loader_overrides = LoaderOverrides::without_managed_config_for_tests(); + let config = Arc::new( + ConfigBuilder::default() + .codex_home(codex_home.to_path_buf()) + .fallback_cwd(Some(codex_home.to_path_buf())) + .loader_overrides(loader_overrides.clone()) + .build() + .await?, + ); + + Ok(start_in_process_client(config, loader_overrides).await?) +} + async fn start_in_process_client( config: Arc, loader_overrides: LoaderOverrides, -) -> std::io::Result { +) -> std::io::Result { in_process::start(InProcessStartArgs { arg0_paths: Arg0DispatchPaths::default(), config, @@ -301,6 +319,22 @@ async fn start_in_process_client( .await } +async fn delete_thread( + client: &InProcessClientHandle, + request_id: i64, + thread_id: String, +) -> Result<()> { + let response = client + .request(ClientRequest::ThreadDelete { + request_id: RequestId::Integer(request_id), + params: ThreadDeleteParams { thread_id }, + }) + .await? + .map_err(|error| anyhow::anyhow!("thread/delete failed: {}", error.message))?; + let _: ThreadDeleteResponse = serde_json::from_value(response)?; + Ok(()) +} + fn assert_no_local_persistence_artifacts(codex_home: &Path) -> Result<()> { // These are the observable tripwires for accidental local persistence. If a // future code path constructs a local rollout/session store or opens the diff --git a/codex-rs/app-server/tests/suite/v2/thread_delete.rs b/codex-rs/app-server/tests/suite/v2/thread_delete.rs new file mode 100644 index 0000000000..bef155f9ad --- /dev/null +++ b/codex-rs/app-server/tests/suite/v2/thread_delete.rs @@ -0,0 +1,200 @@ +use anyhow::Result; +use app_test_support::TestAppServer; +use app_test_support::create_fake_rollout; +use app_test_support::to_response; +use codex_app_server_protocol::JSONRPCError; +use codex_app_server_protocol::JSONRPCResponse; +use codex_app_server_protocol::RequestId; +use codex_app_server_protocol::ThreadDeleteParams; +use codex_app_server_protocol::ThreadDeleteResponse; +use codex_app_server_protocol::ThreadDeletedNotification; +use codex_app_server_protocol::ThreadLoadedListParams; +use codex_app_server_protocol::ThreadLoadedListResponse; +use codex_app_server_protocol::ThreadStartParams; +use codex_app_server_protocol::ThreadStartResponse; +use codex_core::find_thread_path_by_id_str; +use codex_protocol::ThreadId; +use codex_state::DirectionalThreadSpawnEdgeStatus; +use codex_state::StateRuntime; +use pretty_assertions::assert_eq; +use std::path::Path; +use tempfile::TempDir; +use tokio::time::timeout; + +const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); + +#[tokio::test] +async fn thread_delete_deletes_spawned_descendants() -> Result<()> { + let codex_home = TempDir::new()?; + + let parent_id = create_delete_test_rollout(codex_home.path(), /*minute*/ 0, "parent")?; + let child_id = create_delete_test_rollout(codex_home.path(), /*minute*/ 1, "child")?; + let grandchild_id = + create_delete_test_rollout(codex_home.path(), /*minute*/ 2, "grandchild")?; + + let state_db = + StateRuntime::init(codex_home.path().to_path_buf(), "mock_provider".into()).await?; + let parent_thread_id = ThreadId::from_string(&parent_id)?; + let child_thread_id = ThreadId::from_string(&child_id)?; + let grandchild_thread_id = ThreadId::from_string(&grandchild_id)?; + + for (parent, child, status) in [ + ( + parent_thread_id, + child_thread_id, + DirectionalThreadSpawnEdgeStatus::Closed, + ), + ( + child_thread_id, + grandchild_thread_id, + DirectionalThreadSpawnEdgeStatus::Open, + ), + ] { + state_db + .upsert_thread_spawn_edge(parent, child, status) + .await?; + } + + let mut mcp = TestAppServer::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let delete_id = mcp + .send_thread_delete_request(ThreadDeleteParams { + thread_id: parent_id.clone(), + }) + .await?; + let delete_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(delete_id)), + ) + .await??; + let _: ThreadDeleteResponse = to_response::(delete_resp)?; + + let mut deleted_ids = Vec::new(); + for _ in 0..3 { + let notification = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("thread/deleted"), + ) + .await??; + let deleted_notification: ThreadDeletedNotification = serde_json::from_value( + notification + .params + .expect("thread/deleted notification params"), + )?; + deleted_ids.push(deleted_notification.thread_id); + } + assert_eq!(deleted_ids, vec![grandchild_id, child_id, parent_id]); + + for thread_id in [parent_thread_id, child_thread_id, grandchild_thread_id] { + let rollout_path = find_thread_path_by_id_str( + codex_home.path(), + &thread_id.to_string(), + /*state_db_ctx*/ None, + ) + .await?; + assert!( + rollout_path.is_none(), + "expected active rollout for {thread_id} to be deleted" + ); + } + assert_eq!( + state_db + .list_thread_spawn_descendants(parent_thread_id) + .await?, + Vec::::new() + ); + Ok(()) +} + +fn create_delete_test_rollout(codex_home: &Path, minute: u8, preview: &str) -> Result { + create_fake_rollout( + codex_home, + &format!("2025-01-01T00-{minute:02}-00"), + &format!("2025-01-01T00:{minute:02}:00Z"), + preview, + Some("mock_provider"), + /*git_info*/ None, + ) +} + +#[tokio::test] +async fn thread_delete_handles_live_threads_before_rollout_exists() -> Result<()> { + let codex_home = TempDir::new()?; + + let mut mcp = TestAppServer::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let start_id = mcp + .send_thread_start_request(ThreadStartParams::default()) + .await?; + let start_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(start_id)), + ) + .await??; + let persisted_thread = to_response::(start_resp)?.thread; + let rollout_path = find_thread_path_by_id_str( + codex_home.path(), + &persisted_thread.id, + /*state_db_ctx*/ None, + ) + .await?; + assert_eq!(rollout_path, None); + + let delete_id = mcp + .send_thread_delete_request(ThreadDeleteParams { + thread_id: persisted_thread.id, + }) + .await?; + let delete_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(delete_id)), + ) + .await??; + let _: ThreadDeleteResponse = to_response::(delete_resp)?; + + let start_id = mcp + .send_thread_start_request(ThreadStartParams { + ephemeral: Some(true), + ..Default::default() + }) + .await?; + let start_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(start_id)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response::(start_resp)?; + + let delete_id = mcp + .send_thread_delete_request(ThreadDeleteParams { + thread_id: thread.id.clone(), + }) + .await?; + let delete_err: JSONRPCError = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_error_message(RequestId::Integer(delete_id)), + ) + .await??; + let expected_message = format!( + "thread is not persisted and cannot be deleted: {}", + thread.id + ); + assert_eq!(delete_err.error.message, expected_message); + + let list_id = mcp + .send_thread_loaded_list_request(ThreadLoadedListParams::default()) + .await?; + let list_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(list_id)), + ) + .await??; + let ThreadLoadedListResponse { mut data, .. } = + to_response::(list_resp)?; + data.sort(); + assert_eq!(data, vec![thread.id]); + + Ok(()) +} diff --git a/codex-rs/app-server/tests/suite/v2/turn_start.rs b/codex-rs/app-server/tests/suite/v2/turn_start.rs index 823851b715..7fc0d8c71d 100644 --- a/codex-rs/app-server/tests/suite/v2/turn_start.rs +++ b/codex-rs/app-server/tests/suite/v2/turn_start.rs @@ -43,7 +43,12 @@ use codex_app_server_protocol::ServerRequest; use codex_app_server_protocol::ServerRequestResolvedNotification; use codex_app_server_protocol::SubAgentActivityKind; use codex_app_server_protocol::TextElement; +use codex_app_server_protocol::ThreadDeleteParams; +use codex_app_server_protocol::ThreadDeleteResponse; +use codex_app_server_protocol::ThreadDeletedNotification; use codex_app_server_protocol::ThreadItem; +use codex_app_server_protocol::ThreadLoadedListParams; +use codex_app_server_protocol::ThreadLoadedListResponse; use codex_app_server_protocol::ThreadSource; use codex_app_server_protocol::ThreadStartParams; use codex_app_server_protocol::ThreadStartResponse; @@ -3370,6 +3375,49 @@ async fn turn_start_emits_spawn_agent_item_with_model_metadata_v2() -> Result<() assert_eq!(turn_completed.thread_id, thread.id); assert_eq!(turn_completed.turn.id, turn.turn.id); + // Reuse this live spawn setup to cover thread/delete's ThreadManager descendant path. + let delete_req = mcp + .send_thread_delete_request(ThreadDeleteParams { + thread_id: thread.id.clone(), + }) + .await?; + let delete_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(delete_req)), + ) + .await??; + let _: ThreadDeleteResponse = to_response::(delete_resp)?; + + let mut deleted_thread_ids = Vec::new(); + for _ in 0..2 { + let deleted_notif = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("thread/deleted"), + ) + .await??; + let deleted: ThreadDeletedNotification = serde_json::from_value( + deleted_notif + .params + .expect("thread/deleted notification params"), + )?; + deleted_thread_ids.push(deleted.thread_id); + } + assert_eq!( + deleted_thread_ids, + vec![receiver_thread_id, thread.id.clone()] + ); + + let list_req = mcp + .send_thread_loaded_list_request(ThreadLoadedListParams::default()) + .await?; + let list_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(list_req)), + ) + .await??; + let ThreadLoadedListResponse { data, .. } = to_response::(list_resp)?; + assert_eq!(data, Vec::::new()); + Ok(()) } diff --git a/codex-rs/core/src/agent/control_tests.rs b/codex-rs/core/src/agent/control_tests.rs index 772da10c30..ac42984b58 100644 --- a/codex-rs/core/src/agent/control_tests.rs +++ b/codex-rs/core/src/agent/control_tests.rs @@ -2491,7 +2491,7 @@ async fn list_agent_subtree_thread_ids_includes_anonymous_and_closed_descendants } #[tokio::test] -async fn list_agent_subtree_thread_ids_includes_live_descendants_without_state_db() { +async fn list_agent_subtree_thread_ids_finds_live_descendants_of_unloaded_root() { let (_home, config) = test_config().await; let manager = ThreadManager::with_models_provider_home_and_state_for_tests( CodexAuth::from_api_key("dummy"), @@ -2536,6 +2536,8 @@ async fn list_agent_subtree_thread_ids_includes_live_descendants_without_state_d .await .expect("grandchild spawn should succeed"); + manager.remove_thread(&parent_thread_id).await; + let mut subtree_thread_ids = manager .list_agent_subtree_thread_ids(parent_thread_id) .await diff --git a/codex-rs/core/src/thread_manager.rs b/codex-rs/core/src/thread_manager.rs index d5f54d555e..5758c0cadf 100644 --- a/codex-rs/core/src/thread_manager.rs +++ b/codex-rs/core/src/thread_manager.rs @@ -515,14 +515,12 @@ impl ThreadManager { &self, thread_id: ThreadId, ) -> CodexResult> { - let thread = self.state.get_thread(thread_id).await?; - let mut subtree_thread_ids = Vec::new(); let mut seen_thread_ids = HashSet::new(); subtree_thread_ids.push(thread_id); seen_thread_ids.insert(thread_id); - if let Some(state_db_ctx) = thread.state_db() { + if let Some(state_db_ctx) = self.state.state_db() { for status in [ DirectionalThreadSpawnEdgeStatus::Open, DirectionalThreadSpawnEdgeStatus::Closed, @@ -541,11 +539,8 @@ impl ThreadManager { } } - for descendant_id in thread - .codex - .session - .services - .agent_control + for descendant_id in self + .agent_control() .list_live_agent_subtree_thread_ids(thread_id) .await? { diff --git a/codex-rs/rollout/src/lib.rs b/codex-rs/rollout/src/lib.rs index 9e6db0fb8d..23be07996c 100644 --- a/codex-rs/rollout/src/lib.rs +++ b/codex-rs/rollout/src/lib.rs @@ -73,6 +73,7 @@ pub use session_index::append_thread_name; pub use session_index::find_thread_meta_by_name_str; pub use session_index::find_thread_name_by_id; pub use session_index::find_thread_names_by_ids; +pub use session_index::remove_thread_name_entries; pub use state_db::StateDbHandle; pub use state_db::sqlite_telemetry_recorder; diff --git a/codex-rs/rollout/src/session_index.rs b/codex-rs/rollout/src/session_index.rs index e227515357..96ff9d825e 100644 --- a/codex-rs/rollout/src/session_index.rs +++ b/codex-rs/rollout/src/session_index.rs @@ -1,21 +1,25 @@ use std::collections::HashMap; use std::collections::HashSet; use std::fs::File; +use std::io::ErrorKind; use std::io::Read; use std::io::Seek; use std::io::SeekFrom; +use std::io::Write; use std::path::Path; use std::path::PathBuf; +use std::sync::LazyLock; +use std::sync::Mutex; use codex_protocol::ThreadId; use codex_protocol::protocol::SessionMetaLine; use serde::Deserialize; use serde::Serialize; use tokio::io::AsyncBufReadExt; -use tokio::io::AsyncWriteExt; const SESSION_INDEX_FILE: &str = "session_index.jsonl"; const READ_CHUNK_SIZE: usize = 8192; +static SESSION_INDEX_LOCK: LazyLock> = LazyLock::new(|| Mutex::new(())); #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct SessionIndexEntry { @@ -25,7 +29,7 @@ pub struct SessionIndexEntry { } /// Append a thread name update to the session index. -/// The index is append-only; the most recent entry wins when resolving names or ids. +/// Name updates are append-only; the most recent entry wins when resolving names or ids. pub async fn append_thread_name( codex_home: &Path, thread_id: ThreadId, @@ -46,24 +50,60 @@ pub async fn append_thread_name( } /// Append a raw session index entry to `session_index.jsonl`. -/// The file is append-only; consumers scan from the end to find the newest match. +/// Consumers scan from the end to find the newest match. pub async fn append_session_index_entry( codex_home: &Path, entry: &SessionIndexEntry, ) -> std::io::Result<()> { + let _guard = SESSION_INDEX_LOCK + .lock() + .map_err(|err| std::io::Error::other(err.to_string()))?; let path = session_index_path(codex_home); - let mut file = tokio::fs::OpenOptions::new() + let mut file = std::fs::OpenOptions::new() .create(true) .append(true) - .open(&path) - .await?; + .open(&path)?; let mut line = serde_json::to_string(entry).map_err(std::io::Error::other)?; line.push('\n'); - file.write_all(line.as_bytes()).await?; - file.flush().await?; + file.write_all(line.as_bytes())?; + file.flush()?; Ok(()) } +/// Remove all recorded names for a thread from the session index. +pub async fn remove_thread_name_entries( + codex_home: &Path, + thread_id: ThreadId, +) -> std::io::Result<()> { + let _guard = SESSION_INDEX_LOCK + .lock() + .map_err(|err| std::io::Error::other(err.to_string()))?; + let path = session_index_path(codex_home); + let contents = match std::fs::read_to_string(&path) { + Ok(contents) => contents, + Err(err) if err.kind() == ErrorKind::NotFound => return Ok(()), + Err(err) => return Err(err), + }; + let mut removed = false; + let mut remaining = String::with_capacity(contents.len()); + for line in contents.lines() { + let should_remove = serde_json::from_str::(line.trim()) + .is_ok_and(|entry| entry.id == thread_id); + if should_remove { + removed = true; + } else { + remaining.push_str(line); + remaining.push('\n'); + } + } + if !removed { + return Ok(()); + } + let temp_path = path.with_extension("jsonl.tmp"); + std::fs::write(&temp_path, remaining)?; + std::fs::rename(temp_path, path) +} + /// Find the latest thread name for a thread id, if any. pub async fn find_thread_name_by_id( codex_home: &Path, diff --git a/codex-rs/state/src/runtime/threads.rs b/codex-rs/state/src/runtime/threads.rs index 1ccfab5e12..98eff16c50 100644 --- a/codex-rs/state/src/runtime/threads.rs +++ b/codex-rs/state/src/runtime/threads.rs @@ -884,17 +884,117 @@ ON CONFLICT(id) DO UPDATE SET self.upsert_thread(&metadata).await } - /// Delete a thread metadata row by id. + /// Delete a thread and all associated state by id. pub async fn delete_thread(&self, thread_id: ThreadId) -> anyhow::Result { - let result = sqlx::query("DELETE FROM threads WHERE id = ?") - .bind(thread_id.to_string()) - .execute(self.pool.as_ref()) - .await?; - let rows_affected = result.rows_affected(); - self.memories.delete_thread_memory(thread_id).await?; - if rows_affected > 0 { - let _ = self.thread_goals.delete_thread_goal(thread_id).await?; + self.delete_threads_strict(&[thread_id]).await + } + + /// Delete a set of threads and all associated state. + /// + /// Spawn edges and thread rows are deleted last so a failed delete can be retried with enough + /// state left to rediscover the same spawned subtree. + pub async fn delete_threads_strict(&self, thread_ids: &[ThreadId]) -> anyhow::Result { + if thread_ids.is_empty() { + return Ok(0); } + + let thread_id_strings = thread_ids + .iter() + .map(ThreadId::to_string) + .collect::>(); + for (thread_id, thread_id_string) in thread_ids.iter().zip(&thread_id_strings) { + sqlx::query("DELETE FROM logs WHERE thread_id = ?") + .bind(thread_id_string) + .execute(self.logs_pool.as_ref()) + .await?; + self.memories.delete_thread_memory(*thread_id).await?; + self.thread_goals.delete_thread_goal(*thread_id).await?; + } + + let now = Utc::now().timestamp(); + let mut tx = self.pool.begin().await?; + for thread_id_string in &thread_id_strings { + for parent_thread_id_string in &thread_id_strings { + // If both the job runner and worker are being deleted, requeueing + // the worker item would leave a running job with no loop to consume it. + sqlx::query( + r#" +UPDATE agent_jobs +SET status = ?, updated_at = ?, completed_at = ?, last_error = ? +WHERE status IN (?, ?) + AND id IN ( + SELECT item.job_id + FROM agent_job_items AS item + JOIN thread_spawn_edges AS edge ON edge.child_thread_id = item.assigned_thread_id + WHERE item.status = ? AND item.assigned_thread_id = ? AND edge.parent_thread_id = ? + ) + "#, + ) + .bind(AgentJobStatus::Cancelled.as_str()) + .bind(now) + .bind(now) + .bind("agent job runner thread was deleted") + .bind(AgentJobStatus::Pending.as_str()) + .bind(AgentJobStatus::Running.as_str()) + .bind(AgentJobItemStatus::Running.as_str()) + .bind(thread_id_string) + .bind(parent_thread_id_string) + .execute(&mut *tx) + .await?; + } + sqlx::query("DELETE FROM thread_dynamic_tools WHERE thread_id = ?") + .bind(thread_id_string) + .execute(&mut *tx) + .await?; + sqlx::query( + r#" +UPDATE agent_job_items +SET + status = ?, + assigned_thread_id = NULL, + updated_at = ?, + last_error = ? +WHERE assigned_thread_id = ? AND status = ? + "#, + ) + .bind(AgentJobItemStatus::Pending.as_str()) + .bind(now) + .bind("assigned thread was deleted") + .bind(thread_id_string) + .bind(AgentJobItemStatus::Running.as_str()) + .execute(&mut *tx) + .await?; + sqlx::query( + r#" +UPDATE agent_job_items +SET assigned_thread_id = NULL, updated_at = ? +WHERE assigned_thread_id = ? + "#, + ) + .bind(now) + .bind(thread_id_string) + .execute(&mut *tx) + .await?; + } + for thread_id_string in &thread_id_strings { + sqlx::query( + "DELETE FROM thread_spawn_edges WHERE parent_thread_id = ? OR child_thread_id = ?", + ) + .bind(thread_id_string) + .bind(thread_id_string) + .execute(&mut *tx) + .await?; + } + let mut rows_affected = 0; + for thread_id_string in &thread_id_strings { + rows_affected += sqlx::query("DELETE FROM threads WHERE id = ?") + .bind(thread_id_string) + .execute(&mut *tx) + .await? + .rows_affected(); + } + tx.commit().await?; + Ok(rows_affected) } } @@ -1136,12 +1236,14 @@ mod tests { use crate::DirectionalThreadSpawnEdgeStatus; use crate::runtime::test_support::test_thread_metadata; use crate::runtime::test_support::unique_temp_dir; + use anyhow::Result; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::GitInfo; use codex_protocol::protocol::SessionMeta; use codex_protocol::protocol::SessionMetaLine; use codex_protocol::protocol::SessionSource; use pretty_assertions::assert_eq; + use serde_json::json; use std::path::PathBuf; #[tokio::test] @@ -1182,6 +1284,180 @@ mod tests { assert_eq!(memory_mode, "disabled"); } + #[tokio::test] + async fn delete_thread_cleans_associated_state() -> Result<()> { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string()).await?; + let thread_id = ThreadId::from_string("00000000-0000-0000-0000-000000000401")?; + let child_thread_id = ThreadId::from_string("00000000-0000-0000-0000-000000000402")?; + runtime + .upsert_thread(&test_thread_metadata( + &codex_home, + thread_id, + codex_home.clone(), + )) + .await?; + seed_thread_cleanup_state(&runtime, thread_id, child_thread_id).await?; + sqlx::query("INSERT INTO thread_dynamic_tools (thread_id, position, name, description, input_schema) VALUES (?, ?, ?, ?, ?)") + .bind(thread_id.to_string()) + .bind(0_i64) + .bind("test_tool") + .bind("test dynamic tool") + .bind("{}") + .execute(runtime.pool.as_ref()) + .await?; + runtime + .create_agent_job( + &AgentJobCreateParams { + id: "job-1".to_string(), + name: "test-job".to_string(), + instruction: "Return a result".to_string(), + auto_export: true, + max_runtime_seconds: None, + output_schema_json: None, + input_headers: vec!["path".to_string()], + input_csv_path: "/tmp/in.csv".to_string(), + output_csv_path: "/tmp/out.csv".to_string(), + }, + &[AgentJobItemCreateParams { + item_id: "item-1".to_string(), + row_index: 0, + source_id: None, + row_json: json!({"path": "file-1"}), + }], + ) + .await?; + runtime.mark_agent_job_running("job-1").await?; + runtime + .mark_agent_job_item_running_with_thread( + "job-1", + "item-1", + &child_thread_id.to_string(), + ) + .await?; + + let rows = runtime + .delete_threads_strict(&[thread_id, child_thread_id]) + .await?; + + assert_eq!(rows, 1); + assert!(runtime.get_thread(thread_id).await?.is_none()); + let dynamic_tool_count: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM thread_dynamic_tools WHERE thread_id = ?") + .bind(thread_id.to_string()) + .fetch_one(runtime.pool.as_ref()) + .await?; + assert_eq!(dynamic_tool_count, 0); + assert_thread_cleanup_state(&runtime, thread_id).await?; + let job_item = runtime + .get_agent_job_item("job-1", "item-1") + .await? + .expect("job item should exist"); + assert_eq!(job_item.status, AgentJobItemStatus::Pending); + assert_eq!(job_item.assigned_thread_id, None); + assert_eq!( + job_item.last_error, + Some("assigned thread was deleted".to_string()) + ); + let job = runtime + .get_agent_job("job-1") + .await? + .expect("job should exist"); + assert_eq!(job.status, AgentJobStatus::Cancelled); + + let missing_thread_id = ThreadId::from_string("00000000-0000-0000-0000-000000000403")?; + let missing_child_thread_id = + ThreadId::from_string("00000000-0000-0000-0000-000000000404")?; + seed_thread_cleanup_state(&runtime, missing_thread_id, missing_child_thread_id).await?; + + assert_eq!(runtime.delete_thread(missing_thread_id).await?, 0); + assert_thread_cleanup_state(&runtime, missing_thread_id).await?; + Ok(()) + } + + #[tokio::test] + async fn delete_thread_keeps_retry_graph_on_cleanup_failure() -> Result<()> { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string()).await?; + let thread_id = ThreadId::from_string("00000000-0000-0000-0000-000000000405")?; + let child_thread_id = ThreadId::from_string("00000000-0000-0000-0000-000000000406")?; + runtime + .upsert_thread(&test_thread_metadata( + &codex_home, + thread_id, + codex_home.clone(), + )) + .await?; + seed_thread_cleanup_state(&runtime, thread_id, child_thread_id).await?; + + runtime.logs_pool.close().await; + runtime + .delete_thread(thread_id) + .await + .expect_err("closed log db should fail deletion"); + + assert!(runtime.get_thread(thread_id).await?.is_some()); + assert_eq!( + runtime.list_thread_spawn_descendants(thread_id).await?, + vec![child_thread_id] + ); + Ok(()) + } + + async fn seed_thread_cleanup_state( + runtime: &StateRuntime, + thread_id: ThreadId, + child_thread_id: ThreadId, + ) -> Result<()> { + runtime + .upsert_thread_spawn_edge( + thread_id, + child_thread_id, + DirectionalThreadSpawnEdgeStatus::Closed, + ) + .await?; + runtime + .thread_goals() + .replace_thread_goal( + thread_id, + "test goal", + crate::ThreadGoalStatus::Active, + /*token_budget*/ None, + ) + .await?; + sqlx::query("INSERT INTO logs (ts, ts_nanos, level, target, feedback_log_body, thread_id) VALUES (1, 0, 'INFO', 'test', 'feedback log', ?)") + .bind(thread_id.to_string()) + .execute(runtime.logs_pool.as_ref()) + .await?; + Ok(()) + } + + async fn assert_thread_cleanup_state( + runtime: &StateRuntime, + thread_id: ThreadId, + ) -> Result<()> { + let spawn_edge_count: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM thread_spawn_edges WHERE parent_thread_id = ? OR child_thread_id = ?", + ) + .bind(thread_id.to_string()) + .bind(thread_id.to_string()) + .fetch_one(runtime.pool.as_ref()) + .await?; + assert_eq!(spawn_edge_count, 0); + assert_eq!( + runtime.thread_goals().get_thread_goal(thread_id).await?, + None + ); + let logs = runtime + .query_logs(&LogQuery { + thread_ids: vec![thread_id.to_string()], + ..Default::default() + }) + .await?; + assert!(logs.is_empty()); + Ok(()) + } + #[tokio::test] async fn list_threads_updated_after_returns_oldest_changes_first() { let codex_home = unique_temp_dir(); diff --git a/codex-rs/thread-store/src/in_memory.rs b/codex-rs/thread-store/src/in_memory.rs index 2ab5103663..07156fe355 100644 --- a/codex-rs/thread-store/src/in_memory.rs +++ b/codex-rs/thread-store/src/in_memory.rs @@ -18,6 +18,7 @@ use codex_protocol::protocol::ThreadMemoryMode; use crate::AppendThreadItemsParams; use crate::ArchiveThreadParams; use crate::CreateThreadParams; +use crate::DeleteThreadParams; use crate::ListThreadsParams; use crate::LoadThreadHistoryParams; use crate::ReadThreadByRolloutPathParams; @@ -115,6 +116,7 @@ pub struct InMemoryThreadStoreCalls { pub update_thread_metadata: usize, pub archive_thread: usize, pub unarchive_thread: usize, + pub delete_thread: usize, } /// In-memory [`ThreadStore`] implementation for tests and debug configs. @@ -333,6 +335,25 @@ impl ThreadStore for InMemoryThreadStore { state.calls.unarchive_thread += 1; stored_thread_from_state(&state, params.thread_id, /*include_history*/ false) } + + async fn delete_thread(&self, params: DeleteThreadParams) -> ThreadStoreResult<()> { + let mut state = self.state.lock().await; + state.calls.delete_thread += 1; + let existed = state.histories.remove(¶ms.thread_id).is_some(); + state.created_threads.remove(¶ms.thread_id); + state.names.remove(¶ms.thread_id); + state.metadata_updates.remove(¶ms.thread_id); + state + .rollout_paths + .retain(|_, thread_id| *thread_id != params.thread_id); + if existed { + Ok(()) + } else { + Err(ThreadStoreError::ThreadNotFound { + thread_id: params.thread_id, + }) + } + } } fn stored_thread_from_state( diff --git a/codex-rs/thread-store/src/lib.rs b/codex-rs/thread-store/src/lib.rs index 84b72f23f0..284614bf19 100644 --- a/codex-rs/thread-store/src/lib.rs +++ b/codex-rs/thread-store/src/lib.rs @@ -25,6 +25,7 @@ pub use types::AppendThreadItemsParams; pub use types::ArchiveThreadParams; pub use types::ClearableField; pub use types::CreateThreadParams; +pub use types::DeleteThreadParams; pub use types::ExtraConfig; pub use types::GitInfoPatch; pub use types::ItemPage; diff --git a/codex-rs/thread-store/src/local/delete_thread.rs b/codex-rs/thread-store/src/local/delete_thread.rs new file mode 100644 index 0000000000..ecd962f647 --- /dev/null +++ b/codex-rs/thread-store/src/local/delete_thread.rs @@ -0,0 +1,210 @@ +//! Local hard-delete support for persisted threads. +//! +//! Existing rollout files are deleted before this operation reports success. A rollout file that +//! vanishes after discovery counts as already deleted. SQLite cleanup happens at the app-server +//! layer after every associated rollout has been removed so failed deletes can be retried. + +use std::io::ErrorKind; +use std::path::Path; + +use codex_rollout::ARCHIVED_SESSIONS_SUBDIR; +use codex_rollout::SESSIONS_SUBDIR; +use codex_rollout::find_archived_thread_path_by_id_str; +use codex_rollout::find_thread_path_by_id_str; +use codex_rollout::remove_thread_name_entries; + +use super::LocalThreadStore; +use super::helpers::matching_rollout_file_name; +use super::helpers::scoped_rollout_path; +use crate::DeleteThreadParams; +use crate::ThreadStoreError; +use crate::ThreadStoreResult; + +pub(super) async fn delete_thread( + store: &LocalThreadStore, + params: DeleteThreadParams, +) -> ThreadStoreResult<()> { + let thread_id = params.thread_id; + let thread_id_str = thread_id.to_string(); + let state_db_ctx = store.state_db().await; + let mut rollout_paths = Vec::new(); + + match find_thread_path_by_id_str( + store.config.codex_home.as_path(), + thread_id_str.as_str(), + state_db_ctx.as_deref(), + ) + .await + { + Ok(Some(path)) => rollout_paths.push(path), + Ok(None) => {} + Err(err) => { + return Err(ThreadStoreError::InvalidRequest { + message: format!("failed to locate thread id {thread_id}: {err}"), + }); + } + } + + match find_archived_thread_path_by_id_str( + store.config.codex_home.as_path(), + thread_id_str.as_str(), + state_db_ctx.as_deref(), + ) + .await + { + Ok(Some(path)) => { + if !rollout_paths.contains(&path) { + rollout_paths.push(path); + } + } + Ok(None) => {} + Err(err) => { + return Err(ThreadStoreError::InvalidRequest { + message: format!("failed to locate archived thread id {thread_id}: {err}"), + }); + } + } + + let found_rollout_path = !rollout_paths.is_empty(); + for rollout_path in rollout_paths { + delete_rollout_file(store, rollout_path.as_path(), thread_id)?; + } + remove_thread_name_entries(store.config.codex_home.as_path(), thread_id) + .await + .map_err(|err| ThreadStoreError::Internal { + message: format!("failed to delete thread name index entries for {thread_id}: {err}"), + })?; + + if !found_rollout_path { + return Err(ThreadStoreError::ThreadNotFound { thread_id }); + } + + store.live_recorders.lock().await.remove(&thread_id); + + Ok(()) +} + +fn delete_rollout_file( + store: &LocalThreadStore, + rollout_path: &Path, + thread_id: codex_protocol::ThreadId, +) -> ThreadStoreResult { + let plain_path = codex_rollout::plain_rollout_path(rollout_path); + let compressed_path = plain_path.with_extension("jsonl.zst"); + let deleted_plain = delete_rollout_path(store, plain_path.as_path(), thread_id)?; + let deleted_compressed = delete_rollout_path(store, compressed_path.as_path(), thread_id)?; + Ok(deleted_plain || deleted_compressed) +} + +fn delete_rollout_path( + store: &LocalThreadStore, + rollout_path: &Path, + thread_id: codex_protocol::ThreadId, +) -> ThreadStoreResult { + let canonical_rollout_path = scoped_rollout_path( + store.config.codex_home.join(SESSIONS_SUBDIR), + rollout_path, + "sessions", + ) + .or_else(|_| { + scoped_rollout_path( + store.config.codex_home.join(ARCHIVED_SESSIONS_SUBDIR), + rollout_path, + "archived sessions", + ) + }) + .or_else(|err| match rollout_path.try_exists() { + Ok(false) => Ok(rollout_path.to_path_buf()), + Ok(true) | Err(_) => Err(err), + })?; + matching_rollout_file_name(&canonical_rollout_path, thread_id, rollout_path)?; + match std::fs::remove_file(&canonical_rollout_path) { + Ok(()) => Ok(true), + Err(err) if err.kind() == ErrorKind::NotFound => Ok(false), + Err(err) => Err(ThreadStoreError::Internal { + message: format!( + "failed to delete rollout file `{}`: {err}", + canonical_rollout_path.display() + ), + }), + } +} + +#[cfg(test)] +mod tests { + use codex_protocol::ThreadId; + use pretty_assertions::assert_eq; + use tempfile::TempDir; + use uuid::Uuid; + + use super::*; + use crate::ThreadStore; + use crate::local::LocalThreadStore; + use crate::local::test_support::test_config; + use crate::local::test_support::write_archived_session_file; + use crate::local::test_support::write_session_file; + + #[tokio::test] + async fn delete_thread_removes_active_and_archived_rollouts() { + let home = TempDir::new().expect("temp dir"); + let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None); + let active_path = + write_session_file(home.path(), "2025-01-03T12-00-00", Uuid::from_u128(301)) + .expect("session file"); + let compressed_path = active_path.with_extension("jsonl.zst"); + std::fs::write(&compressed_path, b"compressed sibling").expect("compressed sibling"); + let cases = [ + (Uuid::from_u128(301), active_path), + ( + Uuid::from_u128(302), + write_archived_session_file( + home.path(), + "2025-01-03T12-00-00", + Uuid::from_u128(302), + ) + .expect("archived session file"), + ), + ]; + + for (uuid, path) in cases { + let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id"); + store + .delete_thread(DeleteThreadParams { thread_id }) + .await + .expect("delete thread"); + + assert!(!path.exists()); + } + assert!(!compressed_path.exists()); + } + + #[tokio::test] + async fn delete_rollout_file_treats_vanished_path_as_already_deleted() { + let home = TempDir::new().expect("temp dir"); + let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None); + let uuid = Uuid::from_u128(305); + let thread_id = ThreadId::from_string(&uuid.to_string()).expect("valid thread id"); + let path = + write_session_file(home.path(), "2025-01-03T12-00-00", uuid).expect("session file"); + std::fs::remove_file(&path).expect("remove session file"); + + assert!(!delete_rollout_file(&store, path.as_path(), thread_id).expect("delete rollout")); + } + + #[tokio::test] + async fn delete_thread_reports_missing_thread() { + let home = TempDir::new().expect("temp dir"); + let store = LocalThreadStore::new(test_config(home.path()), /*state_db*/ None); + let thread_id = + ThreadId::from_string("00000000-0000-0000-0000-000000000304").expect("valid thread id"); + + let err = store + .delete_thread(DeleteThreadParams { thread_id }) + .await + .expect_err("missing thread should fail"); + assert_eq!( + err.to_string(), + "thread 00000000-0000-0000-0000-000000000304 not found" + ); + } +} diff --git a/codex-rs/thread-store/src/local/mod.rs b/codex-rs/thread-store/src/local/mod.rs index cef4496a02..3000fee9d9 100644 --- a/codex-rs/thread-store/src/local/mod.rs +++ b/codex-rs/thread-store/src/local/mod.rs @@ -1,5 +1,6 @@ mod archive_thread; mod create_thread; +mod delete_thread; mod helpers; mod list_threads; mod live_writer; @@ -24,6 +25,7 @@ use tokio::sync::Mutex; use crate::AppendThreadItemsParams; use crate::ArchiveThreadParams; use crate::CreateThreadParams; +use crate::DeleteThreadParams; use crate::ListThreadsParams; use crate::LoadThreadHistoryParams; use crate::ReadThreadByRolloutPathParams; @@ -288,6 +290,10 @@ impl ThreadStore for LocalThreadStore { ) -> ThreadStoreResult { unarchive_thread::unarchive_thread(self, params).await } + + async fn delete_thread(&self, params: DeleteThreadParams) -> ThreadStoreResult<()> { + delete_thread::delete_thread(self, params).await + } } #[cfg(test)] diff --git a/codex-rs/thread-store/src/store.rs b/codex-rs/thread-store/src/store.rs index 7276a980da..6373dd9b18 100644 --- a/codex-rs/thread-store/src/store.rs +++ b/codex-rs/thread-store/src/store.rs @@ -5,6 +5,7 @@ use std::any::Any; use crate::AppendThreadItemsParams; use crate::ArchiveThreadParams; use crate::CreateThreadParams; +use crate::DeleteThreadParams; use crate::ItemPage; use crate::ListItemsParams; use crate::ListThreadsParams; @@ -119,4 +120,7 @@ pub trait ThreadStore: Any + Send + Sync { &self, params: ArchiveThreadParams, ) -> ThreadStoreResult; + + /// Deletes a thread's persisted rollout data and associated metadata. + async fn delete_thread(&self, params: DeleteThreadParams) -> ThreadStoreResult<()>; } diff --git a/codex-rs/thread-store/src/types.rs b/codex-rs/thread-store/src/types.rs index 1f18adf081..853d866bc8 100644 --- a/codex-rs/thread-store/src/types.rs +++ b/codex-rs/thread-store/src/types.rs @@ -653,6 +653,13 @@ pub struct ArchiveThreadParams { pub thread_id: ThreadId, } +/// Parameters for deleting a thread. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct DeleteThreadParams { + /// Thread id to delete. + pub thread_id: ThreadId, +} + #[cfg(test)] mod tests { use pretty_assertions::assert_eq; diff --git a/codex-rs/tui/src/app/app_server_event_targets.rs b/codex-rs/tui/src/app/app_server_event_targets.rs index 800560d520..fa2bab0111 100644 --- a/codex-rs/tui/src/app/app_server_event_targets.rs +++ b/codex-rs/tui/src/app/app_server_event_targets.rs @@ -49,6 +49,7 @@ pub(super) fn server_notification_thread_target( Some(notification.thread_id.as_str()) } ServerNotification::ThreadArchived(notification) => Some(notification.thread_id.as_str()), + ServerNotification::ThreadDeleted(notification) => Some(notification.thread_id.as_str()), ServerNotification::ThreadUnarchived(notification) => Some(notification.thread_id.as_str()), ServerNotification::ThreadClosed(notification) => Some(notification.thread_id.as_str()), ServerNotification::ThreadNameUpdated(notification) => { diff --git a/codex-rs/tui/src/chatwidget/protocol.rs b/codex-rs/tui/src/chatwidget/protocol.rs index 3682f245f5..51e4cedf85 100644 --- a/codex-rs/tui/src/chatwidget/protocol.rs +++ b/codex-rs/tui/src/chatwidget/protocol.rs @@ -224,6 +224,7 @@ impl ChatWidget { | ServerNotification::ThreadStarted(_) | ServerNotification::ThreadStatusChanged(_) | ServerNotification::ThreadArchived(_) + | ServerNotification::ThreadDeleted(_) | ServerNotification::ThreadUnarchived(_) | ServerNotification::RawResponseItemCompleted(_) | ServerNotification::CommandExecOutputDelta(_) From 3691fe5b76ccfa84835936c63efffb053dc8e6c6 Mon Sep 17 00:00:00 2001 From: David de Regt Date: Wed, 10 Jun 2026 11:24:29 -0700 Subject: [PATCH 42/47] fix: Auto-recover from corrupted sqlite databases (#26859) Further investigation of the sqlite incidents showed that the problems are due to corruption from the older version of SQLite that we recently upgraded, and that the data is truly corrupted in the root database -- recovery of all data is not possible. Given that the data is reconstructable from the rollouts on disk, we should just auto-backup the database and let codex rebuild the rollout info from the disk rollouts. The new behavior is that appserver auto-backs-up and rebuilds (with logs reflecting that behavior). The CLI now pops a message letting you know this happened and the paths of the backed-up corrupt db and the new database. There is also context added so that the desktop app can read the rebuild info from it and inform the user with it. --- codex-rs/app-server/src/lib.rs | 130 +++++++++- codex-rs/cli/src/doctor.rs | 5 +- codex-rs/cli/src/main.rs | 17 +- codex-rs/cli/src/state_db_recovery.rs | 194 +++++++-------- codex-rs/rollout/src/state_db.rs | 14 +- codex-rs/state/src/lib.rs | 6 + codex-rs/state/src/runtime.rs | 57 ++++- codex-rs/state/src/runtime/goals.rs | 4 + codex-rs/state/src/runtime/memories.rs | 4 + codex-rs/state/src/runtime/recovery.rs | 243 +++++++++++++++++++ codex-rs/state/src/runtime/recovery_tests.rs | 105 ++++++++ codex-rs/tui/src/lib.rs | 37 ++- codex-rs/tui/src/startup_error.rs | 16 +- 13 files changed, 693 insertions(+), 139 deletions(-) create mode 100644 codex-rs/state/src/runtime/recovery.rs create mode 100644 codex-rs/state/src/runtime/recovery_tests.rs diff --git a/codex-rs/app-server/src/lib.rs b/codex-rs/app-server/src/lib.rs index b6f0b8c859..49b0f9e8ee 100644 --- a/codex-rs/app-server/src/lib.rs +++ b/codex-rs/app-server/src/lib.rs @@ -74,6 +74,8 @@ use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::registry::Registry; use tracing_subscriber::util::SubscriberInitExt; +const SQLITE_RECOVERY_CONFIG_WARNING_SUMMARY: &str = "Codex rebuilt its local database."; + mod analytics_utils; mod app_server_tracing; mod attestation; @@ -533,8 +535,8 @@ pub async fn run_main_with_transport_options( } _ => None, }; - let state_db = match rollout_state_db::try_init(&config).await { - Ok(state_db) => Some(state_db), + let state_db_init = match init_sqlite_state_db_with_fresh_start_on_corruption(&config).await { + Ok(state_db_init) => state_db_init, Err(err) => { return Err(std::io::Error::other(format!( "failed to initialize sqlite state runtime under {}: {err}", @@ -542,6 +544,15 @@ pub async fn run_main_with_transport_options( ))); } }; + let state_db = state_db_init.state_db; + if let Some(recovery_notice) = state_db_init.recovery_notice { + config_warnings.push(ConfigWarningNotification { + summary: SQLITE_RECOVERY_CONFIG_WARNING_SUMMARY.to_string(), + details: Some(recovery_notice.details), + path: None, + range: None, + }); + } if should_run_personality_migration { let effective_toml = config.config_layer_stack.effective_config(); @@ -1099,6 +1110,121 @@ pub async fn run_main_with_transport_options( Ok(()) } +struct SqliteRecoveryNotice { + details: String, +} + +struct RecoveredSqliteDatabase { + database_path: String, + backup_folder: String, +} + +struct StateDbInitResult { + state_db: Option, + recovery_notice: Option, +} + +async fn init_sqlite_state_db_with_fresh_start_on_corruption( + config: &Config, +) -> anyhow::Result { + let mut attempted_backups = HashSet::new(); + let mut recovered_databases = Vec::new(); + loop { + let err = match rollout_state_db::try_init(config).await { + Ok(state_db) => { + let recovery_notice = sqlite_recovery_notice(&recovered_databases); + if recovery_notice.is_some() { + emit_state_db_backup_warning(SQLITE_RECOVERY_CONFIG_WARNING_SUMMARY); + for recovered_database in &recovered_databases { + emit_state_db_backup_warning(&format!( + "Database path: {}", + recovered_database.database_path + )); + emit_state_db_backup_warning(&format!( + "Backup folder: {}", + recovered_database.backup_folder + )); + } + } + return Ok(StateDbInitResult { + state_db: Some(state_db), + recovery_notice, + }); + } + Err(err) => err, + }; + if !codex_state::is_sqlite_corruption_error(&err) { + return Err(err); + } + + let database_path = codex_state::runtime_db_path_for_corruption_error(&err) + .unwrap_or_else(|| codex_state::state_db_path(config.sqlite_home.as_path())); + if !attempted_backups.insert(database_path.clone()) { + return Err(anyhow::anyhow!( + "failed to initialize sqlite state runtime after moving damaged database file into a backup folder: {err}" + )); + } + + let original_error = err.to_string(); + emit_state_db_backup_warning(&format!( + "Codex local database at {} appears damaged. Moving it into a backup folder so the app server can rebuild it from saved data.", + database_path.display() + )); + let backups = codex_state::backup_runtime_db_for_fresh_start(database_path.as_path()) + .await + .map_err(|backup_err| { + anyhow::anyhow!( + "failed to move damaged sqlite state database files into a backup folder: {backup_err}; original error: {original_error}" + ) + })?; + for backup in &backups { + emit_state_db_backup_warning(&format!( + "Moved damaged Codex local database file {} to {}", + backup.original_path.display(), + backup.backup_path.display() + )); + } + if let Some(first_backup) = backups.first() + && let Some(backup_folder) = first_backup.backup_path.parent() + { + recovered_databases.push(RecoveredSqliteDatabase { + database_path: first_backup.original_path.display().to_string(), + backup_folder: backup_folder.display().to_string(), + }); + } + } +} + +fn sqlite_recovery_notice( + recovered_databases: &[RecoveredSqliteDatabase], +) -> Option { + if recovered_databases.is_empty() { + return None; + } + + let details = recovered_databases + .iter() + .map(|recovered_database| { + format!( + "Database path: {}\nBackup folder: {}", + recovered_database.database_path, recovered_database.backup_folder + ) + }) + .collect::>() + .join("\n\n"); + Some(SqliteRecoveryNotice { details }) +} + +fn emit_state_db_backup_warning(message: &str) { + warn!("{message}"); + if !tracing::dispatcher::has_been_set() { + #[allow(clippy::print_stderr)] + { + eprintln!("{message}"); + } + } +} + fn analytics_rpc_transport(transport: &AppServerTransport) -> AppServerRpcTransport { match transport { AppServerTransport::Stdio => AppServerRpcTransport::Stdio, diff --git a/codex-rs/cli/src/doctor.rs b/codex-rs/cli/src/doctor.rs index d817f1a192..abb5daf973 100644 --- a/codex-rs/cli/src/doctor.rs +++ b/codex-rs/cli/src/doctor.rs @@ -2141,8 +2141,9 @@ async fn state_check(config: &Config) -> DoctorCheck { }; let mut check = DoctorCheck::new("state.paths", "state", status, summary).details(details); if status == CheckStatus::Fail { - check = check - .remediation("Back up CODEX_HOME, then remove or repair the affected SQLite database."); + check = check.remediation( + "Move the damaged SQLite database aside, then restart the interactive CLI or app server so it can rebuild that runtime database from saved data. Other entry points may not rebuild automatically.", + ); } check } diff --git a/codex-rs/cli/src/main.rs b/codex-rs/cli/src/main.rs index b194a5ab1b..13acf40d4e 100644 --- a/codex-rs/cli/src/main.rs +++ b/codex-rs/cli/src/main.rs @@ -38,6 +38,7 @@ use codex_utils_cli::ProfileV2Name; use codex_utils_cli::SharedCliOptions; use codex_utils_cli::resume_hint; use owo_colors::OwoColorize; +use std::collections::HashSet; use std::io::IsTerminal; use std::path::PathBuf; use supports_color::Stream; @@ -2188,7 +2189,7 @@ async fn run_interactive_tui( remote_endpoint.clone(), ) }; - let mut attempted_repair = false; + let mut attempted_backups = HashSet::new(); loop { let err = match start_tui().await { Ok(exit_info) => return Ok(exit_info), @@ -2201,25 +2202,25 @@ async fn run_interactive_tui( local_state_db::print_locked_guidance(startup_error); return Ok(AppExitInfo::fatal(startup_error.to_string())); } - if attempted_repair { + if !local_state_db::is_auto_backup_recoverable(startup_error) { local_state_db::print_diagnostic_guidance(startup_error); return Ok(AppExitInfo::fatal(startup_error.to_string())); } - if !local_state_db::confirm_repair(startup_error)? { + if !attempted_backups.insert(startup_error.database_path().to_path_buf()) { local_state_db::print_diagnostic_guidance(startup_error); return Ok(AppExitInfo::fatal(startup_error.to_string())); } - match local_state_db::repair_files(startup_error).await { - Ok(backups) => local_state_db::print_repair_backups(&backups), - Err(repair_err) => { + local_state_db::print_auto_backup_start(startup_error); + match local_state_db::backup_files_for_fresh_start(startup_error).await { + Ok(backups) => local_state_db::confirm_fresh_start_rebuild(startup_error, &backups)?, + Err(backup_err) => { local_state_db::print_diagnostic_guidance(startup_error); return Ok(AppExitInfo::fatal(format!( - "failed to repair Codex local data automatically: {repair_err}" + "failed to move damaged Codex local database files into a backup folder automatically: {backup_err}" ))); } } - attempted_repair = true; } } diff --git a/codex-rs/cli/src/state_db_recovery.rs b/codex-rs/cli/src/state_db_recovery.rs index 7aeffaca3a..89bb16643c 100644 --- a/codex-rs/cli/src/state_db_recovery.rs +++ b/codex-rs/cli/src/state_db_recovery.rs @@ -1,10 +1,12 @@ -//! CLI recovery for local state database startup failures. +//! CLI handling for local state database startup failures. //! -//! This keeps user-facing repair and lock-contention handling out of the main +//! This keeps user-facing backup and lock-contention handling out of the main //! CLI dispatch path while preserving the TUI startup error as the boundary type. +use codex_state::RuntimeDbBackup; use codex_tui::LocalStateDbStartupError; -use std::path::PathBuf; +use std::io::IsTerminal; +use std::path::Path; pub(crate) fn startup_error(err: &std::io::Error) -> Option<&LocalStateDbStartupError> { err.get_ref() @@ -12,66 +14,60 @@ pub(crate) fn startup_error(err: &std::io::Error) -> Option<&LocalStateDbStartup } pub(crate) fn is_locked(detail: &str) -> bool { - let detail = detail.to_ascii_lowercase(); - detail.contains("database is locked") || detail.contains("database is busy") + codex_state::sqlite_error_detail_is_lock(detail) } -pub(crate) fn confirm_repair(startup_error: &LocalStateDbStartupError) -> std::io::Result { +pub(crate) fn is_corruption(detail: &str) -> bool { + codex_state::sqlite_error_detail_is_corruption(detail) +} + +pub(crate) fn is_auto_backup_recoverable(startup_error: &LocalStateDbStartupError) -> bool { + is_corruption(startup_error.detail()) || sqlite_home_is_blocking_file(startup_error) +} + +fn sqlite_home_is_blocking_file(startup_error: &LocalStateDbStartupError) -> bool { + startup_error + .database_path() + .parent() + .and_then(|path| std::fs::metadata(path).ok()) + .is_some_and(|metadata| metadata.is_file()) +} + +pub(crate) fn print_auto_backup_start(startup_error: &LocalStateDbStartupError) { eprintln!("Codex couldn't start because its local database appears to be damaged."); - eprintln!("Codex can try a safe repair by backing up those files and rebuilding them."); + eprintln!("Moving the damaged local database aside so Codex can rebuild it from saved data."); print_technical_details(startup_error); - crate::confirm("Repair Codex local data now? [y/N]: ") } -pub(crate) async fn repair_files( +pub(crate) async fn backup_files_for_fresh_start( startup_error: &LocalStateDbStartupError, -) -> std::io::Result> { - let state_db_path = startup_error.state_db_path(); - let sqlite_home = state_db_path.parent().ok_or_else(|| { - std::io::Error::other("state database path does not have a parent directory") - })?; - let timestamp = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0, |duration| duration.as_secs()); - let repair_suffix = format!("codex-repair-{timestamp}"); - let mut backups = Vec::new(); - - match tokio::fs::metadata(sqlite_home).await { - Ok(metadata) if metadata.is_dir() => {} - Ok(_) => { - backups.push(backup_path(sqlite_home, &repair_suffix).await?); - tokio::fs::create_dir_all(sqlite_home).await?; - } - Err(err) if err.kind() == std::io::ErrorKind::NotFound => { - tokio::fs::create_dir_all(sqlite_home).await?; - } - Err(err) => return Err(err), - } - - for path in codex_state::runtime_db_paths(sqlite_home) - .into_iter() - .flat_map(|db| sqlite_paths(db.path.as_path())) - { - if tokio::fs::try_exists(path.as_path()).await? { - backups.push(backup_path(path.as_path(), &repair_suffix).await?); - } - } - - if backups.is_empty() { - return Err(std::io::Error::other( - "no repairable Codex local data files were found", - )); - } - - Ok(backups) +) -> std::io::Result> { + codex_state::backup_runtime_db_for_fresh_start(startup_error.database_path()).await } -pub(crate) fn print_repair_backups(backups: &[PathBuf]) { - eprintln!("Backed up Codex local data before repair:"); - for backup in backups { - eprintln!(" {}", backup.display()); +pub(crate) fn confirm_fresh_start_rebuild( + startup_error: &LocalStateDbStartupError, + backups: &[RuntimeDbBackup], +) -> std::io::Result<()> { + eprintln!("Codex rebuilt its local database."); + eprintln!( + "Codex detected a damaged local database, moved it into a backup folder, and will continue startup with a fresh database." + ); + eprintln!("Database path: {}", startup_error.database_path().display()); + if let Some(backup_folder) = backup_folder(backups) { + eprintln!("Backup folder: {}", backup_folder.display()); + } else { + eprintln!("Backup folder: unavailable"); } - eprintln!("Retrying startup with rebuilt local data..."); + + if std::io::stdin().is_terminal() && std::io::stderr().is_terminal() { + eprintln!("Press Enter to continue."); + let mut input = String::new(); + std::io::stdin().read_line(&mut input)?; + } else { + eprintln!("Continuing startup with a fresh local database..."); + } + Ok(()) } pub(crate) fn print_diagnostic_guidance(startup_error: &LocalStateDbStartupError) { @@ -87,79 +83,50 @@ pub(crate) fn print_locked_guidance(startup_error: &LocalStateDbStartupError) { print_technical_details(startup_error); } -fn sqlite_paths(db_path: &std::path::Path) -> Vec { - let mut wal_path = db_path.as_os_str().to_os_string(); - wal_path.push("-wal"); - let mut shm_path = db_path.as_os_str().to_os_string(); - shm_path.push("-shm"); - vec![ - db_path.to_path_buf(), - PathBuf::from(wal_path), - PathBuf::from(shm_path), - ] -} - -async fn backup_path(path: &std::path::Path, repair_suffix: &str) -> std::io::Result { - let file_name = path.file_name().ok_or_else(|| { - std::io::Error::other(format!( - "cannot create a repair backup name for {}", - path.display() - )) - })?; - let mut sequence = 0; - loop { - let mut backup_name = file_name.to_os_string(); - backup_name.push(format!(".{repair_suffix}.{sequence}.bak")); - let backup_path = path.with_file_name(backup_name); - if !tokio::fs::try_exists(backup_path.as_path()).await? { - tokio::fs::rename(path, backup_path.as_path()).await?; - return Ok(backup_path); - } - sequence += 1; - } -} - fn print_technical_details(startup_error: &LocalStateDbStartupError) { eprintln!("Technical details:"); - eprintln!(" Location: {}", startup_error.state_db_path().display()); + eprintln!(" Location: {}", startup_error.database_path().display()); eprintln!(" Cause: {}", startup_error.detail()); } +fn backup_folder(backups: &[RuntimeDbBackup]) -> Option<&Path> { + backups.first()?.backup_path.parent() +} + #[cfg(test)] mod tests { use super::*; use pretty_assertions::assert_eq; + use std::path::PathBuf; use tempfile::TempDir; #[tokio::test] - async fn repair_backs_up_owned_database_files() -> std::io::Result<()> { + async fn backup_backs_up_only_failed_database_file() -> std::io::Result<()> { let temp_dir = TempDir::new()?; let state_path = codex_state::state_db_path(temp_dir.path()); - let logs_path = codex_state::logs_db_path(temp_dir.path()); - let goals_path = codex_state::goals_db_path(temp_dir.path()); - let state_sidecars = sqlite_paths(state_path.as_path()); + let failed_db_path = codex_state::logs_db_path(temp_dir.path()); tokio::fs::write(state_path.as_path(), b"state").await?; - tokio::fs::write(state_sidecars[1].as_path(), b"state-wal").await?; - tokio::fs::write(logs_path.as_path(), b"logs").await?; - tokio::fs::write(goals_path.as_path(), b"goals").await?; + tokio::fs::write(failed_db_path.as_path(), b"logs").await?; let startup_error = - LocalStateDbStartupError::new(state_path.clone(), "corrupt".to_string()); - let backups = repair_files(&startup_error).await?; + LocalStateDbStartupError::new(failed_db_path.clone(), "corrupt".to_string()); + let backups = backup_files_for_fresh_start(&startup_error).await?; - assert_eq!(backups.len(), 4); - assert!(!tokio::fs::try_exists(state_path.as_path()).await?); - assert!(!tokio::fs::try_exists(state_sidecars[1].as_path()).await?); - assert!(!tokio::fs::try_exists(logs_path.as_path()).await?); - assert!(!tokio::fs::try_exists(goals_path.as_path()).await?); - for backup in backups { - assert!(tokio::fs::try_exists(backup.as_path()).await?); - } + assert_eq!( + backups + .iter() + .map(|backup| &backup.original_path) + .collect::>(), + vec![&failed_db_path] + ); + assert!(!tokio::fs::try_exists(failed_db_path.as_path()).await?); + assert!(tokio::fs::try_exists(state_path.as_path()).await?); + assert!(tokio::fs::try_exists(backups[0].backup_path.as_path()).await?); Ok(()) } #[tokio::test] - async fn repair_replaces_blocking_sqlite_home_file() -> std::io::Result<()> { + async fn backup_replaces_blocking_sqlite_home_file() -> std::io::Result<()> { let temp_dir = TempDir::new()?; let sqlite_home = temp_dir.path().join("sqlite-home"); tokio::fs::write(sqlite_home.as_path(), b"not-a-directory").await?; @@ -168,18 +135,25 @@ mod tests { "File exists".to_string(), ); - let backups = repair_files(&startup_error).await?; + assert!(is_auto_backup_recoverable(&startup_error)); + let backups = backup_files_for_fresh_start(&startup_error).await?; assert_eq!(backups.len(), 1); assert!(tokio::fs::metadata(sqlite_home.as_path()).await?.is_dir()); - assert!(tokio::fs::try_exists(backups[0].as_path()).await?); + assert!(tokio::fs::try_exists(backups[0].backup_path.as_path()).await?); Ok(()) } #[test] - fn lock_failures_skip_repair() { - assert!(is_locked("database is locked")); - assert!(is_locked("database is busy")); - assert!(!is_locked("database disk image is malformed")); + fn backup_folder_uses_parent_of_first_backup_path() { + let backups = vec![RuntimeDbBackup { + original_path: PathBuf::from("/tmp/state_5.sqlite"), + backup_path: PathBuf::from("/tmp/db-backups/sqlite-1-0/state_5.sqlite"), + }]; + + assert_eq!( + backup_folder(&backups), + Some(Path::new("/tmp/db-backups/sqlite-1-0")) + ); } } diff --git a/codex-rs/rollout/src/state_db.rs b/codex-rs/rollout/src/state_db.rs index fa2905dd4f..c21d7002be 100644 --- a/codex-rs/rollout/src/state_db.rs +++ b/codex-rs/rollout/src/state_db.rs @@ -5,6 +5,7 @@ use crate::list::SortDirection; use crate::list::ThreadSortKey; use crate::metadata; use crate::sqlite_metrics; +use anyhow::Context; use chrono::DateTime; use chrono::Utc; use codex_protocol::ThreadId; @@ -50,7 +51,7 @@ pub async fn init(config: &impl RolloutConfigView) -> Option { { Ok(runtime) => Some(runtime), Err(err) => { - emit_startup_warning(&format!("failed to initialize state runtime: {err}")); + emit_startup_warning(&format!("failed to initialize state runtime: {err:#}")); None } } @@ -109,9 +110,9 @@ async fn try_init_with_roots_inner( let runtime = codex_state::StateRuntime::init(sqlite_home.clone(), default_model_provider_id.clone()) .await - .map_err(|err| { - anyhow::anyhow!( - "failed to initialize state runtime at {}: {err}", + .with_context(|| { + format!( + "failed to initialize state runtime at {}", sqlite_home.display() ) })?; @@ -128,7 +129,10 @@ async fn try_init_with_roots_inner( backfill_gate_started.elapsed(), &backfill_gate_result, ); - backfill_gate_result?; + if let Err(err) = backfill_gate_result { + runtime.close().await; + return Err(err); + } Ok(runtime) } diff --git a/codex-rs/state/src/lib.rs b/codex-rs/state/src/lib.rs index cb8c711e65..264b737e9f 100644 --- a/codex-rs/state/src/lib.rs +++ b/codex-rs/state/src/lib.rs @@ -57,15 +57,21 @@ pub use runtime::GoalStore; pub use runtime::GoalUpdate; pub use runtime::MemoryStore; pub use runtime::RemoteControlEnrollmentRecord; +pub use runtime::RuntimeDbBackup; pub use runtime::RuntimeDbPath; pub use runtime::ThreadFilterOptions; +pub use runtime::backup_runtime_db_for_fresh_start; pub use runtime::goals_db_filename; pub use runtime::goals_db_path; +pub use runtime::is_sqlite_corruption_error; pub use runtime::logs_db_filename; pub use runtime::logs_db_path; pub use runtime::memories_db_filename; pub use runtime::memories_db_path; +pub use runtime::runtime_db_path_for_corruption_error; pub use runtime::runtime_db_paths; +pub use runtime::sqlite_error_detail_is_corruption; +pub use runtime::sqlite_error_detail_is_lock; pub use runtime::sqlite_integrity_check; pub use runtime::state_db_filename; pub use runtime::state_db_path; diff --git a/codex-rs/state/src/runtime.rs b/codex-rs/state/src/runtime.rs index 5ae41bcd8b..2937bf0c2e 100644 --- a/codex-rs/state/src/runtime.rs +++ b/codex-rs/state/src/runtime.rs @@ -62,6 +62,7 @@ mod backfill; mod goals; mod logs; mod memories; +mod recovery; mod remote_control; #[cfg(test)] mod test_support; @@ -72,6 +73,12 @@ pub use goals::GoalAccountingOutcome; pub use goals::GoalStore; pub use goals::GoalUpdate; pub use memories::MemoryStore; +pub use recovery::RuntimeDbBackup; +pub use recovery::backup_runtime_db_for_fresh_start; +pub use recovery::is_sqlite_corruption_error; +pub use recovery::runtime_db_path_for_corruption_error; +pub use recovery::sqlite_error_detail_is_corruption; +pub use recovery::sqlite_error_detail_is_lock; pub use remote_control::RemoteControlEnrollmentRecord; pub use threads::ThreadFilterOptions; @@ -200,6 +207,7 @@ impl StateRuntime { Ok(db) => Arc::new(db), Err(err) => { warn!("failed to open logs db at {}: {err}", logs_path.display()); + close_sqlite_pools(&[pool.as_ref()]).await; return Err(err); } }; @@ -208,6 +216,7 @@ impl StateRuntime { Ok(db) => Arc::new(db), Err(err) => { warn!("failed to open goals db at {}: {err}", goals_path.display()); + close_sqlite_pools(&[pool.as_ref(), logs_pool.as_ref()]).await; return Err(err); } }; @@ -224,6 +233,7 @@ impl StateRuntime { "failed to open memories db at {}: {err}", memories_path.display() ); + close_sqlite_pools(&[pool.as_ref(), logs_pool.as_ref(), goals_pool.as_ref()]).await; return Err(err); } }; @@ -236,7 +246,16 @@ impl StateRuntime { started.elapsed(), &backfill_state_result, ); - backfill_state_result?; + if let Err(err) = backfill_state_result { + close_sqlite_pools(&[ + pool.as_ref(), + logs_pool.as_ref(), + goals_pool.as_ref(), + memories_pool.as_ref(), + ]) + .await; + return Err(err); + } let started = Instant::now(); let thread_updated_at_millis_result: anyhow::Result> = sqlx::query_scalar("SELECT MAX(threads.updated_at_ms) FROM threads") @@ -250,7 +269,19 @@ impl StateRuntime { started.elapsed(), &thread_updated_at_millis_result, ); - let thread_updated_at_millis = thread_updated_at_millis_result?; + let thread_updated_at_millis = match thread_updated_at_millis_result { + Ok(value) => value, + Err(err) => { + close_sqlite_pools(&[ + pool.as_ref(), + logs_pool.as_ref(), + goals_pool.as_ref(), + memories_pool.as_ref(), + ]) + .await; + return Err(err); + } + }; let thread_updated_at_millis = thread_updated_at_millis.unwrap_or(0); let runtime = Arc::new(Self { thread_goals: GoalStore::new(Arc::clone(&goals_pool)), @@ -283,6 +314,14 @@ impl StateRuntime { &self.memories } + /// Close all SQLite pools and wait for outstanding pool workers to exit. + pub async fn close(&self) { + self.memories.close().await; + self.thread_goals.close().await; + self.logs_pool.close().await; + self.pool.close().await; + } + pub async fn clear_memory_data_in_sqlite_home(sqlite_home: &Path) -> anyhow::Result { let memories_path = MEMORIES_DB.path(sqlite_home); if !tokio::fs::try_exists(&memories_path).await? { @@ -302,6 +341,12 @@ impl StateRuntime { } } +async fn close_sqlite_pools(pools: &[&SqlitePool]) { + for pool in pools { + pool.close().await; + } +} + fn base_sqlite_options(path: &Path) -> SqliteConnectOptions { SqliteConnectOptions::new() .filename(path) @@ -367,7 +412,8 @@ async fn open_sqlite( started.elapsed(), &pool_result, ); - let pool = pool_result?; + let pool = pool_result + .map_err(|source| recovery::RuntimeDbInitError::new(spec.label, "open", path, source))?; let started = Instant::now(); let migrate_result = migrator.run(&pool).await.map_err(anyhow::Error::from); crate::telemetry::record_init_result( @@ -377,7 +423,10 @@ async fn open_sqlite( started.elapsed(), &migrate_result, ); - migrate_result?; + if let Err(source) = migrate_result { + pool.close().await; + return Err(recovery::RuntimeDbInitError::new(spec.label, "migrate", path, source).into()); + } Ok(pool) } diff --git a/codex-rs/state/src/runtime/goals.rs b/codex-rs/state/src/runtime/goals.rs index d0fab47d05..e5769ccd1f 100644 --- a/codex-rs/state/src/runtime/goals.rs +++ b/codex-rs/state/src/runtime/goals.rs @@ -11,6 +11,10 @@ impl GoalStore { pub(crate) fn new(pool: Arc) -> Self { Self { pool } } + + pub(crate) async fn close(&self) { + self.pool.close().await; + } } pub struct GoalUpdate { diff --git a/codex-rs/state/src/runtime/memories.rs b/codex-rs/state/src/runtime/memories.rs index 3101500796..895c254350 100644 --- a/codex-rs/state/src/runtime/memories.rs +++ b/codex-rs/state/src/runtime/memories.rs @@ -35,6 +35,10 @@ impl MemoryStore { Self { pool, state_pool } } + pub(crate) async fn close(&self) { + self.pool.close().await; + } + /// Deletes all persisted memory state in one transaction. /// /// This removes every `stage1_outputs` row and all `jobs` rows for the diff --git a/codex-rs/state/src/runtime/recovery.rs b/codex-rs/state/src/runtime/recovery.rs new file mode 100644 index 0000000000..d9cf0bcdc9 --- /dev/null +++ b/codex-rs/state/src/runtime/recovery.rs @@ -0,0 +1,243 @@ +//! Backup-and-rebuild support for Codex runtime SQLite databases. +//! +//! Codex keeps several independent runtime SQLite databases under one SQLite +//! home. When SQLite reports that one of them is corrupt, automatic recovery +//! moves only that database file and its sidecars into a backup folder so the +//! other databases keep their data. + +use std::borrow::Cow; +use std::path::Path; +use std::path::PathBuf; + +const BACKUP_DIR_NAME: &str = "db-backups"; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct RuntimeDbBackup { + /// Path where the runtime database or sidecar lived before it was moved. + pub original_path: PathBuf, + /// Path where the runtime database or sidecar was backed up. + pub backup_path: PathBuf, +} + +#[derive(Debug)] +pub(crate) struct RuntimeDbInitError { + label: &'static str, + operation: &'static str, + path: PathBuf, + source: anyhow::Error, +} + +impl RuntimeDbInitError { + pub(crate) fn new( + label: &'static str, + operation: &'static str, + path: &Path, + source: anyhow::Error, + ) -> Self { + Self { + label, + operation, + path: path.to_path_buf(), + source, + } + } + + fn path(&self) -> &Path { + self.path.as_path() + } +} + +impl std::fmt::Display for RuntimeDbInitError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "failed to {} {} at {}: {}", + self.operation, + self.label, + self.path.display(), + self.source + ) + } +} + +impl std::error::Error for RuntimeDbInitError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(self.source.as_ref()) + } +} + +/// Move one Codex runtime SQLite database out of the way so that database can +/// be recreated without discarding unrelated runtime databases. +pub async fn backup_runtime_db_for_fresh_start( + db_path: &Path, +) -> std::io::Result> { + let sqlite_home = db_path.parent().ok_or_else(|| { + std::io::Error::other(format!( + "database path does not have a parent directory: {}", + db_path.display() + )) + })?; + match tokio::fs::metadata(sqlite_home).await { + Ok(metadata) if metadata.is_dir() => backup_runtime_db_files(db_path).await, + Ok(_) => backup_blocking_sqlite_home(sqlite_home).await, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + tokio::fs::create_dir_all(sqlite_home).await?; + Err(std::io::Error::other(format!( + "no Codex runtime database files were found to back up for {}", + db_path.display() + ))) + } + Err(err) => Err(err), + } +} + +pub fn runtime_db_path_for_corruption_error(err: &anyhow::Error) -> Option { + if !is_sqlite_corruption_error(err) { + return None; + } + err.chain() + .find_map(|source| source.downcast_ref::()) + .map(|err| err.path().to_path_buf()) +} + +pub fn is_sqlite_corruption_error(err: &anyhow::Error) -> bool { + err.chain().any(sqlite_error_source_is_corruption) +} + +fn sqlite_error_source_is_corruption(source: &(dyn std::error::Error + 'static)) -> bool { + let Some(err) = source.downcast_ref::() else { + return false; + }; + let sqlx::Error::Database(database_error) = err else { + return false; + }; + sqlite_error_detail_is_corruption(database_error.message()) + || database_error + .code() + .is_some_and(sqlite_database_code_is_corruption) +} + +fn sqlite_database_code_is_corruption(code: Cow<'_, str>) -> bool { + matches!( + code.as_ref().to_ascii_lowercase().as_str(), + "11" | "26" | "sqlite_corrupt" | "sqlite_notadb" + ) +} + +pub fn sqlite_error_detail_is_corruption(detail: &str) -> bool { + let detail = detail.to_ascii_lowercase(); + detail.contains("database disk image is malformed") + || detail.contains("database schema is malformed") + || detail.contains("database is corrupt") + || detail.contains("file is not a database") + || detail.contains("sqlite_corrupt") + || detail.contains("sqlite_notadb") + || detail.contains("(code: 11)") + || detail.contains("(code: 26)") +} + +pub fn sqlite_error_detail_is_lock(detail: &str) -> bool { + let detail = detail.to_ascii_lowercase(); + detail.contains("database is locked") || detail.contains("database is busy") +} + +async fn backup_runtime_db_files(db_path: &Path) -> std::io::Result> { + let sqlite_home = db_path.parent().ok_or_else(|| { + std::io::Error::other(format!( + "database path does not have a parent directory: {}", + db_path.display() + )) + })?; + backup_sqlite_paths(sqlite_home, sqlite_paths(db_path)).await +} + +async fn backup_sqlite_paths( + sqlite_home: &Path, + paths: impl IntoIterator, +) -> std::io::Result> { + let backup_dir = create_unique_backup_dir(sqlite_home.join(BACKUP_DIR_NAME).as_path()).await?; + let mut backups = Vec::new(); + + for path in paths { + if tokio::fs::try_exists(path.as_path()).await? { + let backup_path = backup_dir.join(file_name(path.as_path())?); + tokio::fs::rename(path.as_path(), backup_path.as_path()).await?; + backups.push(RuntimeDbBackup { + original_path: path, + backup_path, + }); + } + } + + if backups.is_empty() { + let _ = tokio::fs::remove_dir(backup_dir).await; + return Err(std::io::Error::other( + "no Codex runtime database files were found to back up", + )); + } + + Ok(backups) +} + +async fn backup_blocking_sqlite_home(sqlite_home: &Path) -> std::io::Result> { + let parent = sqlite_home.parent().ok_or_else(|| { + std::io::Error::other(format!( + "cannot create a backup folder for {}", + sqlite_home.display() + )) + })?; + let mut backup_dir_name = file_name(sqlite_home)?.to_os_string(); + backup_dir_name.push(format!(".{BACKUP_DIR_NAME}")); + let backup_parent = parent.join(backup_dir_name); + let backup_dir = create_unique_backup_dir(backup_parent.as_path()).await?; + let backup_path = backup_dir.join(file_name(sqlite_home)?); + tokio::fs::rename(sqlite_home, backup_path.as_path()).await?; + tokio::fs::create_dir_all(sqlite_home).await?; + Ok(vec![RuntimeDbBackup { + original_path: sqlite_home.to_path_buf(), + backup_path, + }]) +} + +fn sqlite_paths(db_path: &Path) -> Vec { + let mut wal_path = db_path.as_os_str().to_os_string(); + wal_path.push("-wal"); + let mut shm_path = db_path.as_os_str().to_os_string(); + shm_path.push("-shm"); + vec![ + db_path.to_path_buf(), + PathBuf::from(wal_path), + PathBuf::from(shm_path), + ] +} + +async fn create_unique_backup_dir(backup_parent: &Path) -> std::io::Result { + tokio::fs::create_dir_all(backup_parent).await?; + let timestamp = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |duration| duration.as_secs()); + let mut sequence = 0_u32; + loop { + let backup_dir = backup_parent.join(format!("sqlite-{timestamp}-{sequence}")); + match tokio::fs::create_dir(backup_dir.as_path()).await { + Ok(()) => return Ok(backup_dir), + Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => { + sequence += 1; + } + Err(err) => return Err(err), + } + } +} + +fn file_name(path: &Path) -> std::io::Result<&std::ffi::OsStr> { + path.file_name().ok_or_else(|| { + std::io::Error::other(format!( + "cannot create a backup name for {}", + path.display() + )) + }) +} + +#[cfg(test)] +#[path = "recovery_tests.rs"] +mod tests; diff --git a/codex-rs/state/src/runtime/recovery_tests.rs b/codex-rs/state/src/runtime/recovery_tests.rs new file mode 100644 index 0000000000..7424dbcfe1 --- /dev/null +++ b/codex-rs/state/src/runtime/recovery_tests.rs @@ -0,0 +1,105 @@ +use super::*; +use crate::runtime::test_support::unique_temp_dir; +use pretty_assertions::assert_eq; + +#[tokio::test] +async fn backup_moves_only_requested_runtime_db_files_to_backup_folder() -> std::io::Result<()> { + let sqlite_home = unique_temp_dir(); + tokio::fs::create_dir_all(sqlite_home.as_path()).await?; + let runtime_paths = super::super::runtime_db_paths(sqlite_home.as_path()); + let mut expected_paths = Vec::new(); + for db_path in runtime_paths.iter().map(|db| db.path.as_path()) { + for path in sqlite_paths(db_path) { + tokio::fs::write(path.as_path(), path.display().to_string()).await?; + expected_paths.push(path); + } + } + let failed_db_path = super::super::logs_db_path(sqlite_home.as_path()); + let failed_paths = sqlite_paths(failed_db_path.as_path()); + + let backups = backup_runtime_db_for_fresh_start(failed_db_path.as_path()).await?; + + assert_eq!(backups.len(), failed_paths.len()); + for path in &failed_paths { + assert!(!tokio::fs::try_exists(path.as_path()).await?); + } + for path in expected_paths + .iter() + .filter(|path| !failed_paths.contains(path)) + { + assert!(tokio::fs::try_exists(path.as_path()).await?); + } + for backup in backups { + assert!( + backup + .backup_path + .starts_with(sqlite_home.join(BACKUP_DIR_NAME)) + ); + assert!(tokio::fs::try_exists(backup.backup_path.as_path()).await?); + } + Ok(()) +} + +#[tokio::test] +async fn backup_replaces_blocking_sqlite_home_file() -> std::io::Result<()> { + let temp_dir = unique_temp_dir(); + tokio::fs::create_dir_all(temp_dir.as_path()).await?; + let sqlite_home = temp_dir.join("sqlite-home"); + tokio::fs::write(sqlite_home.as_path(), b"not-a-directory").await?; + + let backups = backup_runtime_db_for_fresh_start( + super::super::state_db_path(sqlite_home.as_path()).as_path(), + ) + .await?; + + assert_eq!(backups.len(), 1); + assert!(tokio::fs::metadata(sqlite_home.as_path()).await?.is_dir()); + assert!( + backups[0] + .backup_path + .starts_with(temp_dir.join(format!("sqlite-home.{BACKUP_DIR_NAME}"))) + ); + assert!(tokio::fs::try_exists(backups[0].backup_path.as_path()).await?); + Ok(()) +} + +#[test] +fn sqlite_error_detail_classifies_corruption_and_lock_errors() { + assert!(sqlite_error_detail_is_corruption("file is not a database")); + assert!(sqlite_error_detail_is_corruption( + "error returned from database: (code: 11) database disk image is malformed" + )); + assert!(!sqlite_error_detail_is_corruption("database is locked")); + assert!(sqlite_error_detail_is_lock("database is locked")); + assert!(sqlite_error_detail_is_lock("database is busy")); +} + +#[tokio::test] +async fn runtime_db_path_for_corruption_error_returns_failed_database_path() -> std::io::Result<()> +{ + let sqlite_home = unique_temp_dir(); + tokio::fs::create_dir_all(sqlite_home.as_path()).await?; + let path = super::super::state_db_path(sqlite_home.as_path()); + tokio::fs::write(path.as_path(), b"not sqlite").await?; + + let err = match super::super::StateRuntime::init(sqlite_home, "openai".to_string()).await { + Ok(_) => panic!("malformed sqlite should fail to initialize"), + Err(err) => err, + }; + + assert_eq!(runtime_db_path_for_corruption_error(&err), Some(path)); + Ok(()) +} + +#[test] +fn runtime_db_path_for_corruption_error_ignores_corrupt_word_in_path() { + let path = PathBuf::from("/tmp/sqlite_corrupt/state_5.sqlite"); + let err = anyhow::Error::new(RuntimeDbInitError::new( + "state DB", + "open", + path.as_path(), + anyhow::anyhow!("permission denied"), + )); + + assert_eq!(runtime_db_path_for_corruption_error(&err), None); +} diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index 281e4a52a0..4c0918387a 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -352,9 +352,11 @@ async fn init_state_db_for_app_server_target( ) -> std::io::Result> { match app_server_target { AppServerTarget::Embedded => state_db::try_init(config).await.map(Some).map_err(|err| { + let database_path = codex_state::runtime_db_path_for_corruption_error(&err) + .unwrap_or_else(|| codex_state::state_db_path(config.sqlite_home.as_path())); std::io::Error::other(LocalStateDbStartupError::new( - codex_state::state_db_path(config.sqlite_home.as_path()), - err.to_string(), + database_path, + format!("{err:#}"), )) }), AppServerTarget::LocalDaemon { .. } | AppServerTarget::Remote { .. } => { @@ -2990,6 +2992,37 @@ mod tests { ); Ok(()) } + + #[tokio::test] + async fn embedded_state_db_corruption_preserves_failed_database_for_cli_recovery() + -> color_eyre::Result<()> { + let temp_dir = TempDir::new()?; + let mut config = build_config(&temp_dir).await?; + let sqlite_home = temp_dir.path().join("sqlite-home"); + std::fs::create_dir_all(&sqlite_home)?; + let logs_db_path = codex_state::logs_db_path(&sqlite_home); + std::fs::write(&logs_db_path, "not a sqlite database")?; + config.sqlite_home = sqlite_home; + + let err = + match init_state_db_for_app_server_target(&config, &AppServerTarget::Embedded).await { + Ok(_) => panic!("embedded startup should surface state db init failures"), + Err(err) => err, + }; + let startup_error = err + .get_ref() + .and_then(|err| err.downcast_ref::()) + .expect("state db startup failure should retain its typed context"); + + assert_eq!(startup_error.database_path(), logs_db_path.as_path()); + assert!( + codex_state::sqlite_error_detail_is_corruption(startup_error.detail()), + "startup error should preserve the SQLite corruption cause, got: {}", + startup_error.detail() + ); + Ok(()) + } + #[tokio::test] #[serial] async fn windows_shows_trust_prompt_with_sandbox() -> std::io::Result<()> { diff --git a/codex-rs/tui/src/startup_error.rs b/codex-rs/tui/src/startup_error.rs index 55b691d836..585cec4b23 100644 --- a/codex-rs/tui/src/startup_error.rs +++ b/codex-rs/tui/src/startup_error.rs @@ -3,24 +3,28 @@ use std::path::PathBuf; #[derive(Debug, thiserror::Error)] #[error( - "failed to initialize sqlite state db at {}: {detail}", - state_db_path.display() + "failed to initialize sqlite local db at {}: {detail}", + database_path.display() )] pub struct LocalStateDbStartupError { - state_db_path: PathBuf, + database_path: PathBuf, detail: String, } impl LocalStateDbStartupError { - pub fn new(state_db_path: PathBuf, detail: String) -> Self { + pub fn new(database_path: PathBuf, detail: String) -> Self { Self { - state_db_path, + database_path, detail, } } + pub fn database_path(&self) -> &Path { + self.database_path.as_path() + } + pub fn state_db_path(&self) -> &Path { - self.state_db_path.as_path() + self.database_path() } pub fn detail(&self) -> &str { From 636cc11398b805a9ad89a4ce2b45ded2feb59567 Mon Sep 17 00:00:00 2001 From: stefanstokic-oai Date: Wed, 10 Jun 2026 14:25:04 -0400 Subject: [PATCH 43/47] [codex] remove blocking external agent migration flow (#27064) ## Why External-agent import should be initiated deliberately instead of interrupting eligible TUI startups. This cleanup removes the blocking startup flow before the replacement import experience is introduced later in the stack. ## What changed - remove the startup-blocking external-agent migration prompt - remove the now-unused external migration feature gate - remove the obsolete TUI app-server migration wrappers - retain the dormant picker behind a module-scoped dead-code allowance until the next stack item wires it back in - keep normal TUI startup focused on entering Codex immediately ## Validation - `bazel build --config=clippy //codex-rs/tui:tui //codex-rs/tui:tui-unit-tests-bin` - `just test -p codex-tui external_agent_config_migration` (8 passed) - `just test -p codex-tui` (2,786 passed, 12 unrelated local environment-sensitive failures, 4 skipped) - `just fix -p codex-tui` - `just fmt` ## Stack 1. [#27064](https://github.com/openai/codex/pull/27064): remove the startup migration flow 2. [#27065](https://github.com/openai/codex/pull/27065): extract the picker renderer 3. [#27070](https://github.com/openai/codex/pull/27070): add the external-agent import picker UX 4. [#27071](https://github.com/openai/codex/pull/27071): expose the flow through `/import` **This PR is stack item 1.** --- codex-rs/features/src/lib.rs | 8 +- codex-rs/features/src/tests.rs | 27 +- codex-rs/tui/src/app.rs | 40 -- codex-rs/tui/src/app_server_session.rs | 30 - ...external_agent_config_migration_startup.rs | 570 ------------------ codex-rs/tui/src/lib.rs | 25 +- 6 files changed, 18 insertions(+), 682 deletions(-) delete mode 100644 codex-rs/tui/src/external_agent_config_migration_startup.rs diff --git a/codex-rs/features/src/lib.rs b/codex-rs/features/src/lib.rs index 1777ceef06..b30c255165 100644 --- a/codex-rs/features/src/lib.rs +++ b/codex-rs/features/src/lib.rs @@ -177,7 +177,7 @@ pub enum Feature { RemotePlugin, /// Enable remote plugin sharing flows. PluginSharing, - /// Show the startup prompt for migrating external agent config into Codex. + /// Removed compatibility flag retained as a no-op. ExternalMigration, /// Allow the model to invoke the built-in image generation tool. ImageGeneration, @@ -1071,11 +1071,7 @@ pub const FEATURES: &[FeatureSpec] = &[ FeatureSpec { id: Feature::ExternalMigration, key: "external_migration", - stage: Stage::Experimental { - name: "External migration", - menu_description: "Show a startup prompt when Codex detects migratable external agent config for this machine or project.", - announcement: "", - }, + stage: Stage::Removed, default_enabled: false, }, FeatureSpec { diff --git a/codex-rs/features/src/tests.rs b/codex-rs/features/src/tests.rs index e230c45352..b7d158ca36 100644 --- a/codex-rs/features/src/tests.rs +++ b/codex-rs/features/src/tests.rs @@ -83,6 +83,16 @@ fn plugin_hooks_is_removed_and_disabled_by_default() { assert_eq!(feature_for_key("plugin_hooks"), Some(Feature::PluginHooks)); } +#[test] +fn external_migration_is_removed_and_disabled_by_default() { + assert_eq!(Feature::ExternalMigration.stage(), Stage::Removed); + assert_eq!(Feature::ExternalMigration.default_enabled(), false); + assert_eq!( + feature_for_key("external_migration"), + Some(Feature::ExternalMigration) + ); +} + #[test] fn removed_apps_mcp_path_override_shapes_are_ignored() { let features = [ @@ -122,23 +132,6 @@ fn guardian_approval_is_stable_and_enabled_by_default() { assert_eq!(Feature::GuardianApproval.default_enabled(), true); } -#[test] -fn external_migration_is_experimental_and_disabled_by_default() { - let spec = Feature::ExternalMigration.info(); - let stage = spec.stage; - - assert!(matches!(stage, Stage::Experimental { .. })); - assert_eq!(stage.experimental_menu_name(), Some("External migration")); - assert_eq!( - stage.experimental_menu_description(), - Some( - "Show a startup prompt when Codex detects migratable external agent config for this machine or project." - ) - ); - assert_eq!(stage.experimental_announcement(), None); - assert_eq!(Feature::ExternalMigration.default_enabled(), false); -} - #[test] fn request_permissions_is_under_development() { assert_eq!( diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index b239ff4895..8cf1414d75 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -37,8 +37,6 @@ use crate::cwd_prompt::CwdPromptAction; use crate::diff_render::DiffSummary; use crate::exec_command::split_command_string; use crate::exec_command::strip_bash_lc_and_escape; -use crate::external_agent_config_migration_startup::ExternalAgentConfigMigrationStartupOutcome; -use crate::external_agent_config_migration_startup::handle_external_agent_config_migration_prompt_if_needed; use crate::external_editor; use crate::file_search::FileSearchManager; use crate::history_cell; @@ -764,7 +762,6 @@ impl App { session_selection: SessionSelection, feedback: codex_feedback::CodexFeedback, is_first_run: bool, - entered_trust_nux: bool, should_prompt_windows_sandbox_nux_at_startup: bool, app_server_target: AppServerTarget, state_db: Option, @@ -786,39 +783,6 @@ impl App { let harness_overrides = normalize_harness_overrides_for_cwd(harness_overrides, &config.cwd)?; - let external_agent_config_migration_outcome = - handle_external_agent_config_migration_prompt_if_needed( - tui, - &mut app_server, - &mut config, - &cli_kv_overrides, - &harness_overrides, - &cloud_config_bundle, - entered_trust_nux, - ) - .await?; - let external_agent_config_migration_message = match external_agent_config_migration_outcome - { - ExternalAgentConfigMigrationStartupOutcome::Continue { success_message } => { - success_message - } - ExternalAgentConfigMigrationStartupOutcome::ExitRequested => { - app_server - .shutdown() - .await - .inspect_err(|err| { - tracing::warn!("app-server shutdown failed: {err}"); - }) - .ok(); - return Ok(AppExitInfo { - token_usage: TokenUsage::default(), - thread_id: None, - thread_name: None, - update_action: None, - exit_reason: ExitReason::UserRequested, - }); - } - }; let bootstrap = match startup_bootstrap { Some(bootstrap) => bootstrap, None => app_server.bootstrap(&config).await?, @@ -1014,10 +978,6 @@ impl App { }; chat_widget.remote_connection = remote_connection; let thread_and_widget_ms = thread_and_widget_started_at.elapsed().as_millis(); - if let Some(message) = external_agent_config_migration_message { - chat_widget.add_info_message(message, /*hint*/ None); - } - chat_widget .maybe_prompt_windows_sandbox_enable(should_prompt_windows_sandbox_nux_at_startup); diff --git a/codex-rs/tui/src/app_server_session.rs b/codex-rs/tui/src/app_server_session.rs index e29b40111c..e249575d41 100644 --- a/codex-rs/tui/src/app_server_session.rs +++ b/codex-rs/tui/src/app_server_session.rs @@ -22,11 +22,6 @@ use codex_app_server_protocol::AuthMode; use codex_app_server_protocol::ClientRequest; use codex_app_server_protocol::ConfigBatchWriteParams; use codex_app_server_protocol::ConfigWriteResponse; -use codex_app_server_protocol::ExternalAgentConfigDetectParams; -use codex_app_server_protocol::ExternalAgentConfigDetectResponse; -use codex_app_server_protocol::ExternalAgentConfigImportParams; -use codex_app_server_protocol::ExternalAgentConfigImportResponse; -use codex_app_server_protocol::ExternalAgentConfigMigrationItem; use codex_app_server_protocol::GetAccountParams; use codex_app_server_protocol::GetAccountRateLimitsResponse; use codex_app_server_protocol::GetAccountResponse; @@ -349,31 +344,6 @@ impl AppServerSession { .map_err(|err| bootstrap_request_error("account/read failed during TUI bootstrap", err)) } - pub(crate) async fn external_agent_config_detect( - &mut self, - params: ExternalAgentConfigDetectParams, - ) -> Result { - let request_id = self.next_request_id(); - self.client - .request_typed(ClientRequest::ExternalAgentConfigDetect { request_id, params }) - .await - .wrap_err("externalAgentConfig/detect failed during TUI startup") - } - - pub(crate) async fn external_agent_config_import( - &mut self, - migration_items: Vec, - ) -> Result { - let request_id = self.next_request_id(); - self.client - .request_typed(ClientRequest::ExternalAgentConfigImport { - request_id, - params: ExternalAgentConfigImportParams { migration_items }, - }) - .await - .wrap_err("externalAgentConfig/import failed during TUI startup") - } - pub(crate) async fn next_event(&mut self) -> Option { self.client.next_event().await } diff --git a/codex-rs/tui/src/external_agent_config_migration_startup.rs b/codex-rs/tui/src/external_agent_config_migration_startup.rs deleted file mode 100644 index b2afbe7718..0000000000 --- a/codex-rs/tui/src/external_agent_config_migration_startup.rs +++ /dev/null @@ -1,570 +0,0 @@ -use crate::app_server_session::AppServerSession; -use crate::external_agent_config_migration::ExternalAgentConfigMigrationOutcome; -use crate::external_agent_config_migration::run_external_agent_config_migration_prompt; -use crate::legacy_core::config::Config; -use crate::legacy_core::config::ConfigBuilder; -use crate::legacy_core::config::ConfigOverrides; -use crate::legacy_core::config::edit::ConfigEdit; -use crate::legacy_core::config::edit::ConfigEditsBuilder; -use crate::tui; -use codex_app_server_protocol::ExternalAgentConfigDetectParams; -use codex_app_server_protocol::ExternalAgentConfigMigrationItem; -use codex_config::CloudConfigBundleLoader; -use codex_features::Feature; -use color_eyre::eyre::Result; -use color_eyre::eyre::WrapErr; -use std::collections::BTreeSet; -use std::path::Path; -use std::time::SystemTime; -use std::time::UNIX_EPOCH; -use toml::Value as TomlValue; - -const EXTERNAL_CONFIG_MIGRATION_PROMPT_COOLDOWN_SECS: i64 = 5 * 24 * 60 * 60; - -pub(crate) enum ExternalAgentConfigMigrationStartupOutcome { - Continue { success_message: Option }, - ExitRequested, -} - -pub(crate) fn should_show_external_agent_config_migration_prompt( - config: &Config, - entered_trust_nux: bool, -) -> bool { - entered_trust_nux && config.features.enabled(Feature::ExternalMigration) -} - -fn external_config_migration_project_key(path: &Path) -> String { - path.display().to_string() -} - -fn is_external_config_migration_scope_hidden(config: &Config, cwd: Option<&Path>) -> bool { - match cwd { - Some(cwd) => config - .notices - .external_config_migration_prompts - .projects - .get(&external_config_migration_project_key(cwd)) - .copied() - .unwrap_or(false), - None => config - .notices - .external_config_migration_prompts - .home - .unwrap_or(false), - } -} - -fn external_config_migration_last_prompted_at(config: &Config, cwd: Option<&Path>) -> Option { - match cwd { - Some(cwd) => config - .notices - .external_config_migration_prompts - .project_last_prompted_at - .get(&external_config_migration_project_key(cwd)) - .copied(), - None => { - config - .notices - .external_config_migration_prompts - .home_last_prompted_at - } - } -} - -fn is_external_config_migration_scope_cooling_down( - config: &Config, - cwd: Option<&Path>, - now_unix_seconds: i64, -) -> bool { - external_config_migration_last_prompted_at(config, cwd).is_some_and(|last_prompted_at| { - last_prompted_at.saturating_add(EXTERNAL_CONFIG_MIGRATION_PROMPT_COOLDOWN_SECS) - > now_unix_seconds - }) -} - -fn visible_external_agent_config_migration_items( - config: &Config, - items: Vec, - now_unix_seconds: i64, -) -> Vec { - items - .into_iter() - .filter(|item| { - !is_external_config_migration_scope_hidden(config, item.cwd.as_deref()) - && !is_external_config_migration_scope_cooling_down( - config, - item.cwd.as_deref(), - now_unix_seconds, - ) - }) - .collect() -} - -fn external_agent_config_migration_success_message( - items: &[ExternalAgentConfigMigrationItem], -) -> String { - if items.iter().any(|item| { - item.item_type == codex_app_server_protocol::ExternalAgentConfigMigrationItemType::Plugins - }) { - "External config migration completed. Plugin migration is still in progress and may take a few minutes." - .to_string() - } else { - "External config migration completed successfully.".to_string() - } -} - -fn unix_seconds_now() -> i64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_secs() as i64 -} - -async fn persist_external_agent_config_migration_prompt_shown( - config: &mut Config, - items: &[ExternalAgentConfigMigrationItem], - now_unix_seconds: i64, -) -> Result<()> { - let mut edits = Vec::new(); - if items.iter().any(|item| item.cwd.is_none()) { - edits.push( - ConfigEdit::SetNoticeExternalConfigMigrationPromptHomeLastPromptedAt(now_unix_seconds), - ); - } - - for project in items - .iter() - .filter_map(|item| item.cwd.as_deref()) - .map(external_config_migration_project_key) - { - edits.push( - ConfigEdit::SetNoticeExternalConfigMigrationPromptProjectLastPromptedAt( - project, - now_unix_seconds, - ), - ); - } - - if edits.is_empty() { - return Ok(()); - } - - ConfigEditsBuilder::for_config(config) - .with_edits(edits) - .apply() - .await - .map_err(|err| color_eyre::eyre::eyre!("{err}")) - .wrap_err("Failed to save external config migration prompt timestamp")?; - - if items.iter().any(|item| item.cwd.is_none()) { - config - .notices - .external_config_migration_prompts - .home_last_prompted_at = Some(now_unix_seconds); - } - for project in items - .iter() - .filter_map(|item| item.cwd.as_deref()) - .map(external_config_migration_project_key) - { - config - .notices - .external_config_migration_prompts - .project_last_prompted_at - .insert(project, now_unix_seconds); - } - - Ok(()) -} - -async fn persist_external_agent_config_migration_prompt_dismissal( - config: &mut Config, - items: &[ExternalAgentConfigMigrationItem], -) -> Result<()> { - let hide_home = items.iter().any(|item| item.cwd.is_none()); - let projects = items - .iter() - .filter_map(|item| item.cwd.as_deref()) - .map(external_config_migration_project_key) - .collect::>(); - - let mut edits = Vec::new(); - if hide_home - && !config - .notices - .external_config_migration_prompts - .home - .unwrap_or(false) - { - edits.push(ConfigEdit::SetNoticeHideExternalConfigMigrationPromptHome( - true, - )); - } - for project in &projects { - if !config - .notices - .external_config_migration_prompts - .projects - .get(project) - .copied() - .unwrap_or(false) - { - edits.push( - ConfigEdit::SetNoticeHideExternalConfigMigrationPromptProject( - project.clone(), - true, - ), - ); - } - } - - if edits.is_empty() { - return Ok(()); - } - - ConfigEditsBuilder::for_config(config) - .with_edits(edits) - .apply() - .await - .map_err(|err| color_eyre::eyre::eyre!("{err}")) - .wrap_err("Failed to save external config migration prompt preference")?; - - if hide_home { - config.notices.external_config_migration_prompts.home = Some(true); - } - for project in projects { - config - .notices - .external_config_migration_prompts - .projects - .insert(project, true); - } - - Ok(()) -} - -pub(crate) async fn handle_external_agent_config_migration_prompt_if_needed( - tui: &mut tui::Tui, - app_server: &mut AppServerSession, - config: &mut Config, - cli_kv_overrides: &[(String, TomlValue)], - harness_overrides: &ConfigOverrides, - cloud_config_bundle: &CloudConfigBundleLoader, - entered_trust_nux: bool, -) -> Result { - if !should_show_external_agent_config_migration_prompt(config, entered_trust_nux) { - return Ok(ExternalAgentConfigMigrationStartupOutcome::Continue { - success_message: None, - }); - } - - let now_unix_seconds = unix_seconds_now(); - let detected_items = match app_server - .external_agent_config_detect(ExternalAgentConfigDetectParams { - include_home: true, - cwds: Some(vec![config.cwd.to_path_buf()]), - }) - .await - { - Ok(response) => { - visible_external_agent_config_migration_items(config, response.items, now_unix_seconds) - } - Err(err) => { - tracing::warn!( - error = %err, - cwd = %config.cwd.display(), - "failed to detect external agent config migrations; continuing startup" - ); - return Ok(ExternalAgentConfigMigrationStartupOutcome::Continue { - success_message: None, - }); - } - }; - - if detected_items.is_empty() { - return Ok(ExternalAgentConfigMigrationStartupOutcome::Continue { - success_message: None, - }); - } - - if let Err(err) = persist_external_agent_config_migration_prompt_shown( - config, - &detected_items, - now_unix_seconds, - ) - .await - { - tracing::warn!( - error = %err, - cwd = %config.cwd.display(), - "failed to persist external config migration prompt timestamp" - ); - } - - let mut selected_items = detected_items.clone(); - let mut error: Option = None; - - loop { - match run_external_agent_config_migration_prompt( - tui, - &detected_items, - &selected_items, - error.as_deref(), - ) - .await - { - ExternalAgentConfigMigrationOutcome::Proceed(items) => { - selected_items = items.clone(); - match app_server.external_agent_config_import(items).await { - Ok(_) => { - let success_message = - external_agent_config_migration_success_message(&selected_items); - *config = ConfigBuilder::default() - .codex_home(config.codex_home.to_path_buf()) - .cli_overrides(cli_kv_overrides.to_vec()) - .harness_overrides(harness_overrides.clone()) - .cloud_config_bundle(cloud_config_bundle.clone()) - .build() - .await - .wrap_err("Failed to reload config after external agent migration")?; - return Ok(ExternalAgentConfigMigrationStartupOutcome::Continue { - success_message: Some(success_message), - }); - } - Err(err) => { - tracing::warn!( - error = %err, - cwd = %config.cwd.display(), - "failed to import external agent config migration items" - ); - error = Some(format!("Migration failed: {err}")); - } - } - } - ExternalAgentConfigMigrationOutcome::Skip => { - return Ok(ExternalAgentConfigMigrationStartupOutcome::Continue { - success_message: None, - }); - } - ExternalAgentConfigMigrationOutcome::SkipForever => { - match persist_external_agent_config_migration_prompt_dismissal( - config, - &detected_items, - ) - .await - { - Ok(()) => { - return Ok(ExternalAgentConfigMigrationStartupOutcome::Continue { - success_message: None, - }); - } - Err(err) => { - tracing::warn!( - error = %err, - cwd = %config.cwd.display(), - "failed to persist external config migration prompt dismissal" - ); - error = Some(format!("Failed to save preference: {err}")); - } - } - } - ExternalAgentConfigMigrationOutcome::Exit => { - return Ok(ExternalAgentConfigMigrationStartupOutcome::ExitRequested); - } - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use codex_app_server_protocol::ExternalAgentConfigMigrationItemType; - use pretty_assertions::assert_eq; - use std::path::PathBuf; - use tempfile::tempdir; - - #[tokio::test] - async fn visible_external_agent_config_migration_items_omits_hidden_scopes() { - let codex_home = tempdir().expect("temp codex home"); - let mut config = ConfigBuilder::default() - .codex_home(codex_home.path().to_path_buf()) - .build() - .await - .expect("config"); - config.notices.external_config_migration_prompts.home = Some(true); - config - .notices - .external_config_migration_prompts - .projects - .insert("/tmp/project".to_string(), true); - - let visible = visible_external_agent_config_migration_items( - &config, - vec![ - ExternalAgentConfigMigrationItem { - item_type: ExternalAgentConfigMigrationItemType::Config, - description: "home".to_string(), - cwd: None, - details: None, - }, - ExternalAgentConfigMigrationItem { - item_type: ExternalAgentConfigMigrationItemType::AgentsMd, - description: "project".to_string(), - cwd: Some(PathBuf::from("/tmp/project")), - details: None, - }, - ExternalAgentConfigMigrationItem { - item_type: ExternalAgentConfigMigrationItemType::Skills, - description: "other project".to_string(), - cwd: Some(PathBuf::from("/tmp/other")), - details: None, - }, - ], - /*now_unix_seconds*/ 1_760_000_000, - ); - - assert_eq!( - visible, - vec![ExternalAgentConfigMigrationItem { - item_type: ExternalAgentConfigMigrationItemType::Skills, - description: "other project".to_string(), - cwd: Some(PathBuf::from("/tmp/other")), - details: None, - }] - ); - } - - #[tokio::test] - async fn visible_external_agent_config_migration_items_omits_recently_prompted_scopes() { - let codex_home = tempdir().expect("temp codex home"); - let mut config = ConfigBuilder::default() - .codex_home(codex_home.path().to_path_buf()) - .build() - .await - .expect("config"); - config - .notices - .external_config_migration_prompts - .home_last_prompted_at = Some(1_760_000_000); - config - .notices - .external_config_migration_prompts - .project_last_prompted_at - .insert("/tmp/project".to_string(), 1_760_000_000); - - let visible = visible_external_agent_config_migration_items( - &config, - vec![ - ExternalAgentConfigMigrationItem { - item_type: ExternalAgentConfigMigrationItemType::Config, - description: "home".to_string(), - cwd: None, - details: None, - }, - ExternalAgentConfigMigrationItem { - item_type: ExternalAgentConfigMigrationItemType::AgentsMd, - description: "project".to_string(), - cwd: Some(PathBuf::from("/tmp/project")), - details: None, - }, - ExternalAgentConfigMigrationItem { - item_type: ExternalAgentConfigMigrationItemType::Skills, - description: "other project".to_string(), - cwd: Some(PathBuf::from("/tmp/other")), - details: None, - }, - ], - /*now_unix_seconds*/ - 1_760_000_000 + EXTERNAL_CONFIG_MIGRATION_PROMPT_COOLDOWN_SECS - 1, - ); - - assert_eq!( - visible, - vec![ExternalAgentConfigMigrationItem { - item_type: ExternalAgentConfigMigrationItemType::Skills, - description: "other project".to_string(), - cwd: Some(PathBuf::from("/tmp/other")), - details: None, - }] - ); - } - - #[tokio::test] - async fn external_config_migration_scope_cooldown_expires_after_five_days() { - let codex_home = tempdir().expect("temp codex home"); - let mut config = ConfigBuilder::default() - .codex_home(codex_home.path().to_path_buf()) - .build() - .await - .expect("config"); - config - .notices - .external_config_migration_prompts - .home_last_prompted_at = Some(1_760_000_000); - - assert!(is_external_config_migration_scope_cooling_down( - &config, - /*cwd*/ None, - 1_760_000_000 + EXTERNAL_CONFIG_MIGRATION_PROMPT_COOLDOWN_SECS - 1, - )); - assert!(!is_external_config_migration_scope_cooling_down( - &config, - /*cwd*/ None, - 1_760_000_000 + EXTERNAL_CONFIG_MIGRATION_PROMPT_COOLDOWN_SECS, - )); - } - - #[test] - fn external_agent_config_migration_success_message_mentions_plugins_when_present() { - let message = external_agent_config_migration_success_message(&[ - ExternalAgentConfigMigrationItem { - item_type: ExternalAgentConfigMigrationItemType::Config, - description: String::new(), - cwd: None, - details: None, - }, - ExternalAgentConfigMigrationItem { - item_type: ExternalAgentConfigMigrationItemType::Plugins, - description: String::new(), - cwd: None, - details: None, - }, - ]); - - assert_eq!( - message, - "External config migration completed. Plugin migration is still in progress and may take a few minutes." - ); - } - - #[test] - fn external_agent_config_migration_success_message_omits_plugins_copy_when_absent() { - let message = - external_agent_config_migration_success_message(&[ExternalAgentConfigMigrationItem { - item_type: ExternalAgentConfigMigrationItemType::AgentsMd, - description: String::new(), - cwd: None, - details: None, - }]); - - assert_eq!(message, "External config migration completed successfully."); - } - - #[tokio::test] - async fn external_agent_config_migration_prompt_requires_trust_nux_entry() { - let codex_home = tempdir().expect("temp codex home"); - let mut config = ConfigBuilder::default() - .codex_home(codex_home.path().to_path_buf()) - .build() - .await - .expect("config"); - let _ = config.features.enable(Feature::ExternalMigration); - - assert!(!should_show_external_agent_config_migration_prompt( - &config, /*entered_trust_nux*/ false, - )); - assert!(should_show_external_agent_config_migration_prompt( - &config, /*entered_trust_nux*/ true, - )); - } -} diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index 4c0918387a..f29fcd62bd 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -130,8 +130,8 @@ mod diff_model; mod diff_render; mod exec_cell; mod exec_command; +#[allow(dead_code)] mod external_agent_config_migration; -mod external_agent_config_migration_startup; mod external_editor; mod file_search; mod frames; @@ -1814,23 +1814,11 @@ async fn run_ratatui_app( let hooks_request_handle = app_server.request_handle(); let hooks_cwd = config.cwd.to_path_buf(); let startup_prefetch_started_at = Instant::now(); - let should_defer_bootstrap = - external_agent_config_migration_startup::should_show_external_agent_config_migration_prompt( - &config, - should_show_trust_screen_flag, - ); - let (startup_bootstrap, startup_hooks_entry) = if should_defer_bootstrap { - ( - None, - load_startup_hooks_review_entry(hooks_request_handle, hooks_cwd).await, - ) - } else { - let (bootstrap, entry) = tokio::join!( - app_server.bootstrap(&config), - load_startup_hooks_review_entry(hooks_request_handle, hooks_cwd), - ); - (Some(bootstrap?), entry) - }; + let (startup_bootstrap, startup_hooks_entry) = tokio::join!( + app_server.bootstrap(&config), + load_startup_hooks_review_entry(hooks_request_handle, hooks_cwd), + ); + let startup_bootstrap = Some(startup_bootstrap?); let startup_elapsed_before_app = startup_prefetch_started_at.elapsed(); let startup_hooks_browser = match maybe_run_startup_hooks_review( &mut app_server, @@ -1858,7 +1846,6 @@ async fn run_ratatui_app( session_selection, feedback, should_show_trust_screen, // Proxy to: is it a first run in this directory? - should_show_trust_screen_flag, // Preserve the startup-time trust NUX signal before onboarding should_prompt_windows_sandbox_nux_at_startup, app_server_target, state_db, From 42415443d036c374eb848caec69b5714e9681b35 Mon Sep 17 00:00:00 2001 From: "Adam Perry @ OpenAI" Date: Wed, 10 Jun 2026 11:37:14 -0700 Subject: [PATCH 44/47] [codex] Raise app-server recursion limit (#27421) ## Summary Unblock Rust release builds after tracing instrumentation increased the async future query depth beyond rustc's default limit. Set the `codex-app-server` crate recursion limit to 256. This changes compilation only; runtime behavior is unchanged. ## Validation - `just test -p codex-app-server` - `cargo build --release --bin codex-app-server` --- codex-rs/app-server/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/codex-rs/app-server/src/lib.rs b/codex-rs/app-server/src/lib.rs index 49b0f9e8ee..d9c75a54e1 100644 --- a/codex-rs/app-server/src/lib.rs +++ b/codex-rs/app-server/src/lib.rs @@ -1,3 +1,4 @@ +#![recursion_limit = "256"] #![deny(clippy::print_stdout, clippy::print_stderr)] use codex_arg0::Arg0DispatchPaths; From ccf1a185180428727cffbdd9bd4eaaab2dc218ef Mon Sep 17 00:00:00 2001 From: kbazzi Date: Wed, 10 Jun 2026 11:46:57 -0700 Subject: [PATCH 45/47] [codex] Retry transient Guardian review failures (#27062) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Background Codex can use **Auto Review** for permission requests. Instead of asking the user immediately, Codex starts a separate locked-down reviewer session called **Guardian**, which returns a structured `allow` or `deny` assessment. The Guardian reviewer is itself a Codex session, so its model request can fail for transient infrastructure reasons such as model overload, HTTP connection failure, or response-stream disconnect. Today, any such failure immediately ends the Auto Review attempt and blocks the action. This PR adds bounded retries for failures that the existing protocol explicitly identifies as transient. Linear context: [CA-539](https://linear.app/openai/issue/CA-539/retry-auto-review-infrastructure-failures-and-fall-back-to-manual) ## What changes A Guardian review can now make at most **three total attempts**: 1. Run the review normally. 2. Retry after a jittered delay of roughly 180–220 ms if the first attempt fails with an eligible error. 3. Retry after a jittered delay of roughly 360–440 ms if the second attempt also fails with an eligible error. All attempts share the original review deadline. Jitter spreads retries from concurrent clients to reduce synchronized load during broader outages. The retries do not reset the user's maximum wait time, and the backoff waits terminate early if the review is cancelled or the deadline expires. Before retrying, the existing Guardian session lifecycle decides whether the session remains usable. Healthy trunks are reused, broken trunks are removed by the existing cleanup path, and ephemeral sessions continue to clean themselves up. The review still emits one logical lifecycle to clients. Recoverable intermediate failures do not produce warnings or terminal events. ## Retry policy ### Retried up to twice - model/server overload - HTTP connection failure - response-stream connection failure - response-stream disconnect - internal server error - a final reviewer message that cannot be parsed as the required Guardian assessment ### Not retried - bad or invalid requests - authentication failures - usage limits - cyber-policy failures - errors without a structured category - a request that already exhausted the lower-level Responses retry budget - a completed Guardian turn with no assessment payload - prompt-construction failures - Guardian review timeout - cancellation or abort - a valid `deny` assessment The session-error classification uses `ErrorEvent.codex_error_info`; it does not inspect error-message strings. ## Implementation notes - `wait_for_guardian_review` preserves the complete `ErrorEvent`, including structured `codex_error_info`. - Guardian session failures preserve the original message and optional structured `CodexErrorInfo`. - The retry policy classifies the explicitly transient `CodexErrorInfo` variants; unknown, absent, and deterministic categories are not retried. - The Guardian session manager receives the caller's deadline rather than creating a new timeout per attempt. - Analytics record the final `attempt_count`. - Retry orchestration does not add a separate session-cleanup protocol; it relies on the existing trunk and ephemeral lifecycle decisions. ## Automated testing Focused Guardian coverage verifies: - every supported transient `CodexErrorInfo` is classified as retryable, while absent and non-transient categories are not; - structured transient session failure -> retry -> approval with the healthy trunk reused; - two invalid Guardian responses -> third attempt -> approval, with exactly three requests; - three invalid responses -> existing fail-closed result, with exactly three requests and one terminal lifecycle; - valid denial, missing payload, invalid request, timeout, cancellation, and prompt/session construction failures are not retried; - retry eligibility ends after the third attempt; - retry delays use the shared exponential backoff helper and remain within the expected jitter bounds; - cancellation and deadline expiry interrupt the backoff wait; - healthy trunks are reused across retryable failures; - broken event streams remove the trunk through the existing lifecycle cleanup; - an ephemeral retry does not disturb a concurrent trunk review. Validation performed: - `just test -p codex-core guardian_review_ guardian_ephemeral_retry_preserves_parallel_trunk_and_fork_history run_review_removes_trunk_when_event_stream_is_broken` — **42 passed**; - `just test -p codex-analytics` — **71 passed**; - scoped Clippy fixes for `codex-core` and `codex-analytics` passed. A prior full `codex-core` run had unrelated environment-sensitive failures outside Guardian coverage. ## Manual QA The focused integration tests use the local mock Responses server to inspect exact request counts and emitted lifecycle events. They confirm that retries are internal, a successful later attempt supplies the final decision, non-retryable failures issue only one request, and exhausted retries emit only one terminal result. --- .../analytics/src/analytics_client_tests.rs | 2 + codex-rs/analytics/src/events.rs | 4 + codex-rs/core/src/guardian/mod.rs | 2 +- codex-rs/core/src/guardian/review.rs | 234 +++++++++++- codex-rs/core/src/guardian/review_session.rs | 102 ++++- codex-rs/core/src/guardian/tests.rs | 356 +++++++++++++++++- 6 files changed, 660 insertions(+), 40 deletions(-) diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs index 02779c6609..b6af117660 100644 --- a/codex-rs/analytics/src/analytics_client_tests.rs +++ b/codex-rs/analytics/src/analytics_client_tests.rs @@ -1937,6 +1937,7 @@ async fn guardian_review_event_ingests_custom_fact_with_optional_target_item() { decision: GuardianReviewDecision::Denied, terminal_status: GuardianReviewTerminalStatus::TimedOut, failure_reason: Some(GuardianReviewFailureReason::Timeout), + attempt_count: 1, risk_level: None, user_authorization: None, outcome: None, @@ -2008,6 +2009,7 @@ async fn guardian_review_event_ingests_custom_fact_with_optional_target_item() { ); assert_eq!(payload[0]["event_params"]["terminal_status"], "timed_out"); assert_eq!(payload[0]["event_params"]["failure_reason"], "timeout"); + assert_eq!(payload[0]["event_params"]["attempt_count"], 1); assert_eq!(payload[0]["event_params"]["review_timeout_ms"], 90_000); } diff --git a/codex-rs/analytics/src/events.rs b/codex-rs/analytics/src/events.rs index 8a25215ec8..4c7277b226 100644 --- a/codex-rs/analytics/src/events.rs +++ b/codex-rs/analytics/src/events.rs @@ -266,6 +266,7 @@ pub struct GuardianReviewEventParams { pub decision: GuardianReviewDecision, pub terminal_status: GuardianReviewTerminalStatus, pub failure_reason: Option, + pub attempt_count: i64, pub risk_level: Option, pub user_authorization: Option, pub outcome: Option, @@ -338,6 +339,7 @@ impl GuardianReviewTrackContext { decision: result.decision, terminal_status: result.terminal_status, failure_reason: result.failure_reason, + attempt_count: result.attempt_count, risk_level: result.risk_level, user_authorization: result.user_authorization, outcome: result.outcome, @@ -373,6 +375,7 @@ pub struct GuardianReviewAnalyticsResult { pub decision: GuardianReviewDecision, pub terminal_status: GuardianReviewTerminalStatus, pub failure_reason: Option, + pub attempt_count: i64, pub risk_level: Option, pub user_authorization: Option, pub outcome: Option, @@ -392,6 +395,7 @@ impl GuardianReviewAnalyticsResult { decision: GuardianReviewDecision::Denied, terminal_status: GuardianReviewTerminalStatus::FailedClosed, failure_reason: None, + attempt_count: 1, risk_level: None, user_authorization: None, outcome: None, diff --git a/codex-rs/core/src/guardian/mod.rs b/codex-rs/core/src/guardian/mod.rs index b4920f1ff6..b06a2daad3 100644 --- a/codex-rs/core/src/guardian/mod.rs +++ b/codex-rs/core/src/guardian/mod.rs @@ -168,7 +168,7 @@ use prompt::render_guardian_transcript_entries; #[cfg(test)] use review::GuardianReviewOutcome; #[cfg(test)] -use review::run_guardian_review_session as run_guardian_review_session_for_test; +use review::run_guardian_review_session_with_retry as run_guardian_review_session_for_test; #[cfg(test)] use review_session::build_guardian_review_session_config as build_guardian_review_session_config_for_test; diff --git a/codex-rs/core/src/guardian/review.rs b/codex-rs/core/src/guardian/review.rs index 934e68042f..99f9c5d999 100644 --- a/codex-rs/core/src/guardian/review.rs +++ b/codex-rs/core/src/guardian/review.rs @@ -7,6 +7,7 @@ use codex_analytics::GuardianReviewTrackContext; use codex_analytics::GuardianReviewedAction; use codex_protocol::config_types::ApprovalsReviewer; use codex_protocol::protocol::AskForApproval; +use codex_protocol::protocol::CodexErrorInfo; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::GuardianAssessmentDecisionSource; use codex_protocol::protocol::GuardianAssessmentEvent; @@ -19,11 +20,14 @@ use codex_protocol::protocol::TurnAbortReason; use codex_protocol::protocol::WarningEvent; use std::sync::Arc; use tokio::sync::oneshot; +use tokio::time::Instant; +use tokio::time::sleep_until; use tokio_util::sync::CancellationToken; use crate::session::session::Session; use crate::session::turn_context::TurnContext; use crate::turn_timing::now_unix_timestamp_ms; +use crate::util::backoff; use super::AUTO_REVIEW_DENIAL_WINDOW_SIZE; use super::GUARDIAN_REVIEW_TIMEOUT; @@ -58,6 +62,8 @@ const GUARDIAN_TIMEOUT_INSTRUCTIONS: &str = concat!( "You may retry once, or ask the user for guidance or explicit approval.", ); +const GUARDIAN_REVIEW_MAX_ATTEMPTS: i64 = 3; + pub(crate) fn new_guardian_review_id() -> String { uuid::Uuid::new_v4().to_string() } @@ -95,9 +101,16 @@ pub(super) enum GuardianReviewOutcome { #[derive(Debug)] pub(super) enum GuardianReviewError { - PromptBuild { message: String }, - Session { message: String }, - Parse { message: String }, + PromptBuild { + message: String, + }, + Session { + message: String, + error_info: Option, + }, + Parse { + message: String, + }, Timeout, Cancelled, } @@ -112,6 +125,14 @@ impl GuardianReviewError { fn session(err: anyhow::Error) -> Self { Self::Session { message: err.to_string(), + error_info: None, + } + } + + fn session_with_error_info(err: anyhow::Error, error_info: CodexErrorInfo) -> Self { + Self::Session { + message: err.to_string(), + error_info: Some(error_info), } } @@ -335,13 +356,14 @@ async fn run_guardian_review( let schema = guardian_output_schema(); let terminal_action = action_summary.clone(); - let (outcome, analytics_result) = Box::pin(run_guardian_review_session( + let (outcome, analytics_result) = Box::pin(run_guardian_review_session_with_retry( session.clone(), turn.clone(), request, retry_reason.clone(), schema, external_cancel, + GUARDIAN_REVIEW_MAX_ATTEMPTS, )) .await; @@ -464,7 +486,7 @@ async fn run_guardian_review( | GuardianReviewError::Parse { .. } => { let message = match &error { GuardianReviewError::PromptBuild { message } - | GuardianReviewError::Session { message } + | GuardianReviewError::Session { message, .. } | GuardianReviewError::Parse { message } => message, GuardianReviewError::Timeout | GuardianReviewError::Cancelled => { "guardian review failed" @@ -657,13 +679,14 @@ pub(crate) fn spawn_approval_request_review( /// context. It may still reuse the parent's managed-network allowlist for /// read-only checks, but it intentionally runs without inherited exec-policy /// rules. -pub(super) async fn run_guardian_review_session( +async fn run_guardian_review_session_before_deadline( session: Arc, turn: Arc, request: GuardianApprovalRequest, retry_reason: Option, schema: serde_json::Value, external_cancel: Option, + deadline: Instant, ) -> (GuardianReviewOutcome, GuardianReviewAnalyticsResult) { let network_proxy = session.services.network_proxy.load_full(); let live_network_config = match network_proxy.as_ref() { @@ -753,6 +776,7 @@ pub(super) async fn run_guardian_review_session( reasoning_summary: turn.reasoning_summary, personality: turn.personality, external_cancel, + deadline, }), ) .await; @@ -787,10 +811,16 @@ pub(super) async fn run_guardian_review_session( GuardianReviewOutcome::Error(GuardianReviewError::prompt_build(err)), session_analytics_result, ), - GuardianReviewSessionOutcome::SessionFailed(err) => ( - GuardianReviewOutcome::Error(GuardianReviewError::session(err)), - session_analytics_result, - ), + GuardianReviewSessionOutcome::SessionFailed { error, error_info } => { + let error = match error_info { + Some(error_info) => GuardianReviewError::session_with_error_info(error, error_info), + None => GuardianReviewError::session(error), + }; + ( + GuardianReviewOutcome::Error(error), + session_analytics_result, + ) + } GuardianReviewSessionOutcome::TimedOut => ( GuardianReviewOutcome::Error(GuardianReviewError::Timeout), session_analytics_result, @@ -802,9 +832,85 @@ pub(super) async fn run_guardian_review_session( } } +pub(super) async fn run_guardian_review_session_with_retry( + session: Arc, + turn: Arc, + request: GuardianApprovalRequest, + retry_reason: Option, + schema: serde_json::Value, + external_cancel: Option, + max_attempts: i64, +) -> (GuardianReviewOutcome, GuardianReviewAnalyticsResult) { + assert!(max_attempts > 0, "guardian review must run at least once"); + let deadline = Instant::now() + GUARDIAN_REVIEW_TIMEOUT; + let mut attempt_count = 1; + loop { + let (outcome, mut analytics_result) = run_guardian_review_session_before_deadline( + Arc::clone(&session), + Arc::clone(&turn), + request.clone(), + retry_reason.clone(), + schema.clone(), + external_cancel.clone(), + deadline, + ) + .await; + analytics_result.attempt_count = attempt_count; + if attempt_count >= max_attempts || !should_retry_guardian_review(&outcome) { + return (outcome, analytics_result); + } + if let Some(error) = + wait_before_guardian_retry(attempt_count, deadline, external_cancel.as_ref()).await + { + return (GuardianReviewOutcome::Error(error), analytics_result); + } + attempt_count += 1; + } +} + +async fn wait_before_guardian_retry( + attempt_count: i64, + deadline: Instant, + external_cancel: Option<&CancellationToken>, +) -> Option { + let retry_delay = backoff(attempt_count as u64); + let retry_at = (Instant::now() + retry_delay).min(deadline); + tokio::select! { + _ = sleep_until(retry_at) => { + (Instant::now() >= deadline).then_some(GuardianReviewError::Timeout) + } + _ = async { + if let Some(cancel_token) = external_cancel { + cancel_token.cancelled().await; + } else { + std::future::pending::<()>().await; + } + } => Some(GuardianReviewError::Cancelled), + } +} + +fn should_retry_guardian_review(outcome: &GuardianReviewOutcome) -> bool { + matches!( + outcome, + GuardianReviewOutcome::Error( + GuardianReviewError::Session { + error_info: Some( + CodexErrorInfo::ServerOverloaded + | CodexErrorInfo::HttpConnectionFailed { .. } + | CodexErrorInfo::ResponseStreamConnectionFailed { .. } + | CodexErrorInfo::InternalServerError + | CodexErrorInfo::ResponseStreamDisconnected { .. } + ), + .. + } | GuardianReviewError::Parse { .. } + ) + ) +} + #[cfg(test)] mod review_tests { use super::*; + use std::time::Duration; #[test] fn guardian_review_error_reason_distinguishes_error_kinds() { @@ -812,6 +918,10 @@ mod review_tests { let prompt_error = GuardianReviewError::prompt_build(anyhow::anyhow!("bad prompt/config")); let session_error = GuardianReviewError::session(anyhow::anyhow!("guardian runtime failed")); + let structured_session_error = GuardianReviewError::session_with_error_info( + anyhow::anyhow!("temporary guardian failure"), + CodexErrorInfo::ServerOverloaded, + ); assert!(matches!( parse_error.failure_reason(), @@ -825,5 +935,109 @@ mod review_tests { session_error.failure_reason(), GuardianReviewFailureReason::SessionError )); + assert!(matches!( + structured_session_error.failure_reason(), + GuardianReviewFailureReason::SessionError + )); + } + + #[test] + fn guardian_review_retry_only_retries_transient_session_and_parse_errors() { + let assessment = GuardianAssessment { + risk_level: GuardianRiskLevel::High, + user_authorization: GuardianUserAuthorization::Unknown, + outcome: GuardianAssessmentOutcome::Deny, + rationale: "deny".to_string(), + }; + let transient_error_info = [ + CodexErrorInfo::ServerOverloaded, + CodexErrorInfo::HttpConnectionFailed { + http_status_code: Some(502), + }, + CodexErrorInfo::ResponseStreamConnectionFailed { + http_status_code: Some(503), + }, + CodexErrorInfo::InternalServerError, + CodexErrorInfo::ResponseStreamDisconnected { + http_status_code: None, + }, + ]; + let mut outcomes = transient_error_info + .into_iter() + .map(|error_info| { + ( + GuardianReviewOutcome::Error(GuardianReviewError::session_with_error_info( + anyhow::anyhow!("transient session"), + error_info, + )), + true, + ) + }) + .collect::>(); + outcomes.extend([ + (GuardianReviewOutcome::Completed(assessment), false), + ( + GuardianReviewOutcome::Error(GuardianReviewError::prompt_build(anyhow::anyhow!( + "prompt" + ))), + false, + ), + ( + GuardianReviewOutcome::Error(GuardianReviewError::session(anyhow::anyhow!( + "session" + ))), + false, + ), + ( + GuardianReviewOutcome::Error(GuardianReviewError::session_with_error_info( + anyhow::anyhow!("bad request"), + CodexErrorInfo::BadRequest, + )), + false, + ), + ( + GuardianReviewOutcome::Error(GuardianReviewError::parse(anyhow::anyhow!("parse"))), + true, + ), + ( + GuardianReviewOutcome::Error(GuardianReviewError::Timeout), + false, + ), + ( + GuardianReviewOutcome::Error(GuardianReviewError::Cancelled), + false, + ), + ]); + + for (outcome, expected) in outcomes { + assert_eq!(should_retry_guardian_review(&outcome), expected); + } + } + + #[tokio::test] + async fn guardian_review_retry_wait_honors_cancellation() { + let cancel_token = CancellationToken::new(); + cancel_token.cancel(); + + let error = wait_before_guardian_retry( + /*attempt_count*/ 1, + Instant::now() + Duration::from_secs(/*secs*/ 1), + Some(&cancel_token), + ) + .await; + + assert!(matches!(error, Some(GuardianReviewError::Cancelled))); + } + + #[tokio::test] + async fn guardian_review_retry_wait_honors_deadline() { + let error = wait_before_guardian_retry( + /*attempt_count*/ 1, + Instant::now(), + /*external_cancel*/ None, + ) + .await; + + assert!(matches!(error, Some(GuardianReviewError::Timeout))); } } diff --git a/codex-rs/core/src/guardian/review_session.rs b/codex-rs/core/src/guardian/review_session.rs index fcda9a5a23..54282c1c0a 100644 --- a/codex-rs/core/src/guardian/review_session.rs +++ b/codex-rs/core/src/guardian/review_session.rs @@ -15,6 +15,8 @@ use codex_protocol::models::PermissionProfile; use codex_protocol::models::ResponseItem; use codex_protocol::openai_models::ReasoningEffort as ReasoningEffortConfig; use codex_protocol::protocol::AskForApproval; +use codex_protocol::protocol::CodexErrorInfo; +use codex_protocol::protocol::ErrorEvent; use codex_protocol::protocol::Event; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::InitialHistory; @@ -46,7 +48,6 @@ use codex_features::Feature; use codex_model_provider_info::ModelProviderInfo; use codex_utils_absolute_path::AbsolutePathBuf; -use super::GUARDIAN_REVIEW_TIMEOUT; use super::GUARDIAN_REVIEWER_NAME; use super::GuardianApprovalRequest; use super::prompt::GuardianPromptMode; @@ -60,7 +61,10 @@ const GUARDIAN_INTERRUPT_DRAIN_TIMEOUT: Duration = Duration::from_secs(5); pub(crate) enum GuardianReviewSessionOutcome { Completed(anyhow::Result>), PromptBuildFailed(anyhow::Error), - SessionFailed(anyhow::Error), + SessionFailed { + error: anyhow::Error, + error_info: Option, + }, TimedOut, Aborted, } @@ -77,6 +81,7 @@ pub(crate) struct GuardianReviewSessionParams { pub(crate) reasoning_summary: ReasoningSummaryConfig, pub(crate) personality: Option, pub(crate) external_cancel: Option, + pub(crate) deadline: tokio::time::Instant, } #[derive(Default)] @@ -312,7 +317,7 @@ impl GuardianReviewSessionManager { &self, params: GuardianReviewSessionParams, ) -> (GuardianReviewSessionOutcome, GuardianReviewAnalyticsResult) { - let deadline = tokio::time::Instant::now() + GUARDIAN_REVIEW_TIMEOUT; + let deadline = params.deadline; let next_reuse_key = GuardianReviewSessionReuseKey::from_spawn_config(¶ms.spawn_config); let mut stale_trunk_to_shutdown = None; let mut spawned_trunk = false; @@ -364,7 +369,9 @@ impl GuardianReviewSessionManager { state.trunk.as_ref().cloned() } - Err(outcome) => return (outcome, GuardianReviewAnalyticsResult::without_session()), + Err(outcome) => { + return (outcome, GuardianReviewAnalyticsResult::without_session()); + } }; if let Some(review_session) = stale_trunk_to_shutdown { @@ -560,7 +567,9 @@ impl GuardianReviewSessionManager { GuardianReviewAnalyticsResult::without_session(), ); } - Err(outcome) => return (outcome, GuardianReviewAnalyticsResult::without_session()), + Err(outcome) => { + return (outcome, GuardianReviewAnalyticsResult::without_session()); + } }; self.register_active_ephemeral(Arc::clone(&review_session)) .await; @@ -764,7 +773,10 @@ async fn run_review_on_session( Ok(Ok(child_turn_id)) => child_turn_id, Ok(Err(err)) => { return ( - GuardianReviewSessionOutcome::SessionFailed(err.into()), + GuardianReviewSessionOutcome::SessionFailed { + error: err.into(), + error_info: None, + }, false, analytics_result, ); @@ -825,7 +837,7 @@ async fn wait_for_guardian_review( ) -> (GuardianReviewSessionOutcome, bool, bool) { let timeout = tokio::time::sleep_until(deadline); tokio::pin!(timeout); - let mut last_error_message: Option = None; + let mut last_error: Option = None; loop { tokio::select! { @@ -862,10 +874,13 @@ async fn wait_for_guardian_review( .time_to_first_token_ms .and_then(|ms| u64::try_from(ms).ok()); if turn_complete.last_agent_message.is_none() - && let Some(error_message) = last_error_message + && let Some(error) = last_error { return ( - GuardianReviewSessionOutcome::Completed(Err(anyhow!(error_message))), + GuardianReviewSessionOutcome::SessionFailed { + error: anyhow!(error.message), + error_info: error.codex_error_info, + }, true, true, ); @@ -877,7 +892,7 @@ async fn wait_for_guardian_review( ); } EventMsg::Error(error) => { - last_error_message = Some(error.message); + last_error = Some(error); } EventMsg::TurnAborted(_) => { return (GuardianReviewSessionOutcome::Aborted, true, false); @@ -920,6 +935,8 @@ pub(crate) fn build_guardian_review_session_config( let mut guardian_config = parent_config.clone(); guardian_config.model = Some(active_model.to_string()); guardian_config.model_reasoning_effort = reasoning_effort; + guardian_config.model_provider.request_max_retries = Some(1); + guardian_config.model_provider.stream_max_retries = Some(1); guardian_config.include_skill_instructions = false; guardian_config.base_instructions = Some( parent_config @@ -1148,6 +1165,7 @@ mod tests { reasoning_summary, personality, external_cancel: None, + deadline: tokio::time::Instant::now() + Duration::from_secs(30), } } @@ -1473,6 +1491,29 @@ mod tests { assert!(keep_review_session); } + #[tokio::test] + async fn run_review_removes_trunk_when_event_stream_is_broken() { + let (mut review_session, tx_event, _rx_sub) = test_review_session().await; + let params = test_review_params().await; + review_session.reuse_key = + GuardianReviewSessionReuseKey::from_spawn_config(¶ms.spawn_config); + let manager = GuardianReviewSessionManager { + state: Arc::new(Mutex::new(GuardianReviewSessionState { + trunk: Some(Arc::new(review_session)), + ephemeral_reviews: Vec::new(), + })), + }; + drop(tx_event); + + let (outcome, _) = manager.run_review(params).await; + + assert!(matches!( + outcome, + GuardianReviewSessionOutcome::Completed(Err(_)) + )); + assert!(manager.state.lock().await.trunk.is_none()); + } + #[tokio::test] async fn wait_for_guardian_review_ignores_prior_turn_completion() { let (review_session, tx_event, _rx_sub) = test_review_session().await; @@ -1545,6 +1586,47 @@ mod tests { assert!(capture_token_usage); } + #[tokio::test] + async fn wait_for_guardian_review_preserves_structured_session_error() { + let (review_session, tx_event, _rx_sub) = test_review_session().await; + tx_event + .send(Event { + id: "current-turn".to_string(), + msg: EventMsg::Error(ErrorEvent { + message: "temporary failure".to_string(), + codex_error_info: Some(CodexErrorInfo::ServerOverloaded), + }), + }) + .await + .expect("queue guardian error"); + tx_event + .send(turn_complete_event( + "current-turn", + /*last_agent_message*/ None, + Some(42), + )) + .await + .expect("queue current turn completion"); + + let mut analytics_result = GuardianReviewAnalyticsResult::without_session(); + let (outcome, keep_review_session, capture_token_usage) = wait_for_guardian_review( + &review_session, + "current-turn", + tokio::time::Instant::now() + Duration::from_secs(1), + /*external_cancel*/ None, + &mut analytics_result, + ) + .await; + + let GuardianReviewSessionOutcome::SessionFailed { error, error_info } = outcome else { + panic!("expected structured session failure"); + }; + assert_eq!(error.to_string(), "temporary failure"); + assert_eq!(error_info, Some(CodexErrorInfo::ServerOverloaded)); + assert!(keep_review_session); + assert!(capture_token_usage); + } + #[tokio::test] async fn wait_for_guardian_review_ignores_prior_turn_aborts() { let (review_session, tx_event, _rx_sub) = test_review_session().await; diff --git a/codex-rs/core/src/guardian/tests.rs b/codex-rs/core/src/guardian/tests.rs index ffadd801ca..8f6e01298c 100644 --- a/codex-rs/core/src/guardian/tests.rs +++ b/codex-rs/core/src/guardian/tests.rs @@ -55,10 +55,11 @@ use core_test_support::context_snapshot::ContextSnapshotOptions; use core_test_support::responses::ev_assistant_message; use core_test_support::responses::ev_completed; use core_test_support::responses::ev_response_created; -use core_test_support::responses::mount_response_once; +use core_test_support::responses::mount_response_sequence; use core_test_support::responses::mount_sse_once; use core_test_support::responses::mount_sse_sequence; use core_test_support::responses::sse; +use core_test_support::responses::sse_failed; use core_test_support::responses::start_mock_server; use core_test_support::skip_if_no_network; use core_test_support::streaming_sse::StreamingSseChunk; @@ -172,6 +173,51 @@ async fn guardian_test_session_and_turn( guardian_test_session_and_turn_with_base_url(server.uri().as_str()).await } +async fn guardian_test_session_turn_and_rx( + server: &wiremock::MockServer, +) -> ( + Arc, + Arc, + async_channel::Receiver, +) { + let (mut session, mut turn, rx) = + crate::session::tests::make_session_and_context_with_rx().await; + Arc::get_mut(&mut session) + .expect("session should be uniquely owned") + .thread_id = fixed_guardian_parent_session_id(); + let mut config = (*turn.config).clone(); + config.model_provider.base_url = Some(format!("{}/v1", server.uri())); + config.user_instructions = None; + let config = Arc::new(config); + let models_manager = test_support::models_manager_with_provider( + config.codex_home.to_path_buf(), + Arc::clone(&session.services.auth_manager), + config.model_provider.clone(), + ); + Arc::get_mut(&mut session) + .expect("session should be uniquely owned") + .services + .models_manager = models_manager; + let turn_mut = Arc::get_mut(&mut turn).expect("turn should be uniquely owned"); + turn_mut.config = Arc::clone(&config); + turn_mut.provider = + create_model_provider(config.model_provider.clone(), turn_mut.auth_manager.clone()); + turn_mut.user_instructions = None; + + (session, turn, rx) +} + +fn guardian_shell_request(id: &str) -> GuardianApprovalRequest { + GuardianApprovalRequest::Shell { + id: id.to_string(), + command: vec!["git".to_string(), "push".to_string()], + cwd: test_path_buf("/repo/codex-rs/core").abs(), + sandbox_permissions: crate::sandboxing::SandboxPermissions::UseDefault, + additional_permissions: None, + justification: Some("Need to push the reviewed docs fix.".to_string()), + } +} + async fn guardian_test_session_and_turn_with_base_url( base_url: &str, ) -> (Arc, Arc) { @@ -1343,6 +1389,7 @@ async fn guardian_request_model_for_auto_review_override( Some("Sandbox denied outbound git push to github.com.".to_string()), guardian_output_schema(), /*external_cancel*/ None, + /*max_attempts*/ 1, ) .await; let (GuardianReviewOutcome::Completed(_), _) = outcome else { @@ -1454,6 +1501,7 @@ async fn guardian_review_request_layout_matches_model_visible_request_snapshot() Some("Sandbox denied outbound git push to github.com.".to_string()), guardian_output_schema(), /*external_cancel*/ None, + /*max_attempts*/ 1, ) .await; let (GuardianReviewOutcome::Completed(assessment), metadata) = outcome else { @@ -1632,6 +1680,7 @@ async fn guardian_reuses_prompt_cache_key_and_appends_prior_reviews() -> anyhow: Some("First retry reason".to_string()), guardian_output_schema(), /*external_cancel*/ None, + /*max_attempts*/ 1, ) .await; session @@ -1676,6 +1725,7 @@ async fn guardian_reuses_prompt_cache_key_and_appends_prior_reviews() -> anyhow: Some("Second retry reason".to_string()), guardian_output_schema(), /*external_cancel*/ None, + /*max_attempts*/ 1, ) .await; session @@ -1716,6 +1766,7 @@ async fn guardian_reuses_prompt_cache_key_and_appends_prior_reviews() -> anyhow: Some("Third retry reason".to_string()), guardian_output_schema(), /*external_cancel*/ None, + /*max_attempts*/ 1, ) .await; @@ -1907,6 +1958,7 @@ async fn guardian_reused_trunk_ignores_stale_prior_turn_completion() -> anyhow:: /*retry_reason*/ None, guardian_output_schema(), /*external_cancel*/ None, + /*max_attempts*/ 1, ) .await; let (GuardianReviewOutcome::Completed(first_assessment), first_metadata) = first_outcome else { @@ -1949,6 +2001,7 @@ async fn guardian_reused_trunk_ignores_stale_prior_turn_completion() -> anyhow:: /*retry_reason*/ None, guardian_output_schema(), /*external_cancel*/ None, + /*max_attempts*/ 1, ) .await; let (GuardianReviewOutcome::Completed(second_assessment), second_metadata) = second_outcome @@ -1978,15 +2031,17 @@ async fn guardian_review_surfaces_responses_api_errors_in_rejection_reason() -> let server = start_mock_server().await; let error_message = "Item 'rs_test' of type 'reasoning' was provided without its required following item."; - let _request_log = mount_response_once( + let request_log = mount_response_sequence( &server, - wiremock::ResponseTemplate::new(400).set_body_json(serde_json::json!({ - "error": { - "message": error_message, - "type": "invalid_request_error", - "param": "input" - } - })), + vec![ + wiremock::ResponseTemplate::new(400).set_body_json(serde_json::json!({ + "error": { + "message": error_message, + "type": "invalid_request_error", + "param": "input" + } + })), + ], ) .await; @@ -2030,6 +2085,7 @@ async fn guardian_review_surfaces_responses_api_errors_in_rejection_reason() -> .await; assert_eq!(decision, ReviewDecision::Denied); + assert_eq!(request_log.requests().len(), 1); let mut warnings = Vec::new(); let mut denial_rationales = Vec::new(); @@ -2080,13 +2136,255 @@ async fn guardian_review_surfaces_responses_api_errors_in_rejection_reason() -> Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn guardian_review_retries_transient_session_failure_then_approves() -> anyhow::Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let approval = serde_json::json!({ + "risk_level": "low", + "user_authorization": "high", + "outcome": "allow", + "rationale": "retry succeeded", + }) + .to_string(); + let request_log = mount_sse_sequence( + &server, + vec![ + sse_failed( + "resp-session-failure", + "server_is_overloaded", + "temporary reviewer overload", + ), + sse(vec![ + ev_response_created("resp-approved"), + ev_assistant_message("msg-approved", &approval), + ev_completed("resp-approved"), + ]), + ], + ) + .await; + let (session, turn) = guardian_test_session_and_turn(&server).await; + seed_guardian_parent_history(&session, &turn).await; + + let (outcome, metadata) = run_guardian_review_session_for_test( + Arc::clone(&session), + Arc::clone(&turn), + guardian_shell_request("shell-session-retry"), + /*retry_reason*/ None, + guardian_output_schema(), + /*external_cancel*/ None, + /*max_attempts*/ 3, + ) + .await; + + let GuardianReviewOutcome::Completed(assessment) = outcome else { + panic!("expected guardian assessment"); + }; + assert_eq!(assessment.outcome, GuardianAssessmentOutcome::Allow); + assert_eq!(assessment.rationale, "retry succeeded"); + assert_eq!(metadata.attempt_count, 2); + assert!(matches!( + metadata.guardian_session_kind, + Some(codex_analytics::GuardianReviewSessionKind::TrunkReused) + )); + assert_eq!(request_log.requests().len(), 2); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn guardian_review_does_not_retry_missing_assessment_payload() -> anyhow::Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let request_log = mount_sse_sequence( + &server, + vec![sse(vec![ + ev_response_created("resp-missing-assessment"), + ev_completed("resp-missing-assessment"), + ])], + ) + .await; + let (session, turn) = guardian_test_session_and_turn(&server).await; + seed_guardian_parent_history(&session, &turn).await; + + let decision = review_approval_request( + &session, + &turn, + "review-missing-assessment".to_string(), + guardian_shell_request("shell-missing-assessment"), + /*retry_reason*/ None, + ) + .await; + + assert_eq!(decision, ReviewDecision::Denied); + assert_eq!(request_log.requests().len(), 1); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn guardian_review_retries_two_parse_failures_then_approves() -> anyhow::Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let approval = serde_json::json!({ + "risk_level": "low", + "user_authorization": "high", + "outcome": "allow", + "rationale": "retry succeeded", + }) + .to_string(); + let request_log = mount_sse_sequence( + &server, + vec![ + sse(vec![ + ev_response_created("resp-parse-failure-1"), + ev_assistant_message("msg-parse-failure-1", "not valid guardian json"), + ev_completed("resp-parse-failure-1"), + ]), + sse(vec![ + ev_response_created("resp-parse-failure-2"), + ev_assistant_message("msg-parse-failure-2", "still not valid guardian json"), + ev_completed("resp-parse-failure-2"), + ]), + sse(vec![ + ev_response_created("resp-approved"), + ev_assistant_message("msg-approved", &approval), + ev_completed("resp-approved"), + ]), + ], + ) + .await; + let (session, turn) = guardian_test_session_and_turn(&server).await; + seed_guardian_parent_history(&session, &turn).await; + + let (outcome, metadata) = run_guardian_review_session_for_test( + Arc::clone(&session), + Arc::clone(&turn), + guardian_shell_request("shell-parse-retry"), + /*retry_reason*/ None, + guardian_output_schema(), + /*external_cancel*/ None, + /*max_attempts*/ 3, + ) + .await; + + let GuardianReviewOutcome::Completed(assessment) = outcome else { + panic!("expected guardian assessment"); + }; + assert_eq!(assessment.outcome, GuardianAssessmentOutcome::Allow); + assert_eq!(assessment.rationale, "retry succeeded"); + assert_eq!(metadata.attempt_count, 3); + assert!(matches!( + metadata.guardian_session_kind, + Some(codex_analytics::GuardianReviewSessionKind::TrunkReused) + )); + assert_eq!(request_log.requests().len(), 3); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn guardian_review_exhausts_three_failures_with_one_terminal_event() -> anyhow::Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let request_log = mount_sse_sequence( + &server, + vec![ + sse(vec![ + ev_response_created("resp-parse-failure-1"), + ev_assistant_message("msg-parse-failure-1", "invalid one"), + ev_completed("resp-parse-failure-1"), + ]), + sse(vec![ + ev_response_created("resp-parse-failure-2"), + ev_assistant_message("msg-parse-failure-2", "invalid two"), + ev_completed("resp-parse-failure-2"), + ]), + sse(vec![ + ev_response_created("resp-parse-failure-3"), + ev_assistant_message("msg-parse-failure-3", "invalid three"), + ev_completed("resp-parse-failure-3"), + ]), + ], + ) + .await; + let (session, turn, rx) = guardian_test_session_turn_and_rx(&server).await; + seed_guardian_parent_history(&session, &turn).await; + + let decision = review_approval_request( + &session, + &turn, + "review-exhausted-retry".to_string(), + guardian_shell_request("shell-exhausted-retry"), + /*retry_reason*/ None, + ) + .await; + + assert_eq!(decision, ReviewDecision::Denied); + assert_eq!(request_log.requests().len(), 3); + let mut statuses = Vec::new(); + while let Ok(event) = rx.try_recv() { + if let EventMsg::GuardianAssessment(event) = event.msg { + statuses.push(event.status); + } + } + assert_eq!( + statuses, + vec![ + GuardianAssessmentStatus::InProgress, + GuardianAssessmentStatus::Denied, + ] + ); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn guardian_review_does_not_retry_valid_denial() -> anyhow::Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let denial = serde_json::json!({ + "risk_level": "high", + "user_authorization": "unknown", + "outcome": "deny", + "rationale": "unsafe", + }) + .to_string(); + let request_log = mount_sse_sequence( + &server, + vec![sse(vec![ + ev_response_created("resp-denied"), + ev_assistant_message("msg-denied", &denial), + ev_completed("resp-denied"), + ])], + ) + .await; + let (session, turn) = guardian_test_session_and_turn(&server).await; + seed_guardian_parent_history(&session, &turn).await; + + let decision = review_approval_request( + &session, + &turn, + "review-valid-denial".to_string(), + guardian_shell_request("shell-valid-denial"), + /*retry_reason*/ None, + ) + .await; + + assert_eq!(decision, ReviewDecision::Denied); + assert_eq!(request_log.requests().len(), 1); + Ok(()) +} + #[tokio::test] -async fn guardian_parallel_reviews_fork_from_last_committed_trunk_history() -> anyhow::Result<()> { +async fn guardian_ephemeral_retry_preserves_parallel_trunk_and_fork_history() -> anyhow::Result<()> +{ const TEST_STACK_SIZE_BYTES: usize = 4 * 1024 * 1024; let handle = std::thread::Builder::new() - .name("guardian_parallel_reviews_fork_from_last_committed_trunk_history".to_string()) + .name("guardian_ephemeral_retry_preserves_parallel_trunk_and_fork_history".to_string()) .stack_size(TEST_STACK_SIZE_BYTES) .spawn(|| -> anyhow::Result<()> { let runtime = tokio::runtime::Builder::new_current_thread() @@ -2141,10 +2439,18 @@ async fn guardian_parallel_reviews_fork_from_last_committed_trunk_history() -> a gate: None, body: sse(vec![ ev_response_created("resp-guardian-3"), - ev_assistant_message("msg-guardian-3", &third_assessment), + ev_assistant_message("msg-guardian-3", "not valid guardian json"), ev_completed("resp-guardian-3"), ]), }], + vec![StreamingSseChunk { + gate: None, + body: sse(vec![ + ev_response_created("resp-guardian-4"), + ev_assistant_message("msg-guardian-4", &third_assessment), + ev_completed("resp-guardian-4"), + ]), + }], ]) .await; @@ -2271,20 +2577,28 @@ async fn guardian_parallel_reviews_fork_from_last_committed_trunk_history() -> a .await; assert_eq!(third_decision, ReviewDecision::Approved); let requests = server.requests().await; - assert_eq!(requests.len(), 3); + assert_eq!(requests.len(), 4); let second_request_body = serde_json::from_slice::(&requests[1])?; - let third_request_body = serde_json::from_slice::(&requests[2])?; + let failed_ephemeral_request_body = + serde_json::from_slice::(&requests[2])?; + let retried_ephemeral_request_body = + serde_json::from_slice::(&requests[3])?; assert_eq!( second_request_body["prompt_cache_key"], - third_request_body["prompt_cache_key"], + failed_ephemeral_request_body["prompt_cache_key"], "forked guardian review should reuse the trunk guardian prompt cache key" ); - let third_request_body_text = third_request_body.to_string(); + assert_eq!( + failed_ephemeral_request_body["prompt_cache_key"], + retried_ephemeral_request_body["prompt_cache_key"], + "retried ephemeral review should preserve the guardian prompt cache key" + ); + let third_request_body_text = retried_ephemeral_request_body.to_string(); assert!( third_request_body_text.contains("first guardian rationale"), "forked guardian review should include the last committed trunk assessment" ); - let third_user_message = last_user_message_text_from_body(&third_request_body); + let third_user_message = last_user_message_text_from_body(&retried_ephemeral_request_body); assert!(third_user_message.contains(">>> TRANSCRIPT DELTA START\n")); assert!( third_user_message.contains("[5] user: Please inspect pending changes before pushing.") @@ -2315,7 +2629,7 @@ async fn guardian_parallel_reviews_fork_from_last_committed_trunk_history() -> a match handle.join() { Ok(result) => result, Err(_) => Err(anyhow::anyhow!( - "guardian_parallel_reviews_fork_from_last_committed_trunk_history thread panicked" + "guardian_ephemeral_retry_preserves_parallel_trunk_and_fork_history thread panicked" )), } } @@ -2541,6 +2855,10 @@ async fn guardian_review_session_config_keeps_bedrock_provider_for_bedrock_gpt_5 ) .expect("guardian config"); + let mut expected_model_provider = + ModelProviderInfo::create_amazon_bedrock_provider(/*aws*/ None); + expected_model_provider.request_max_retries = Some(1); + expected_model_provider.stream_max_retries = Some(1); assert_eq!( ( guardian_config.model, @@ -2550,7 +2868,7 @@ async fn guardian_review_session_config_keeps_bedrock_provider_for_bedrock_gpt_5 ( Some(AMAZON_BEDROCK_GPT_5_4_MODEL_ID.to_string()), AMAZON_BEDROCK_PROVIDER_ID.to_string(), - ModelProviderInfo::create_amazon_bedrock_provider(/*aws*/ None), + expected_model_provider, ) ); } From 72667f4f41ec5515096ea5676b09cd3c01e6c866 Mon Sep 17 00:00:00 2001 From: stefanstokic-oai Date: Wed, 10 Jun 2026 14:48:30 -0400 Subject: [PATCH 46/47] [codex] extract external agent import picker renderer (#27065) ## Why The external-agent import picker is easier to review when its rendering refactor lands separately from new state and interaction behavior. This layer is intended to be behavior-neutral. ## What changed - extract external-agent migration rendering into a dedicated `render` module - preserve existing behavior while separating presentation from interaction logic - establish a smaller foundation for the import picker UX in the next PR ## Validation - `just test -p codex-tui external_agent_config_migration` (10 passed) ## Stack 1. [#27064](https://github.com/openai/codex/pull/27064): remove the startup migration flow 2. [#27065](https://github.com/openai/codex/pull/27065): extract the picker renderer 3. [#27070](https://github.com/openai/codex/pull/27070): add the external-agent import picker UX 4. [#27071](https://github.com/openai/codex/pull/27071): expose the flow through `/import` **This PR is stack item 2.** Draft while the lower stack dependency is reviewed. --- .../src/external_agent_config_migration.rs | 130 +---------------- .../external_agent_config_migration/render.rs | 137 ++++++++++++++++++ 2 files changed, 139 insertions(+), 128 deletions(-) create mode 100644 codex-rs/tui/src/external_agent_config_migration/render.rs diff --git a/codex-rs/tui/src/external_agent_config_migration.rs b/codex-rs/tui/src/external_agent_config_migration.rs index 1ce9d44751..e64507b861 100644 --- a/codex-rs/tui/src/external_agent_config_migration.rs +++ b/codex-rs/tui/src/external_agent_config_migration.rs @@ -1,9 +1,5 @@ use crate::diff_render::display_path_for; -use crate::key_hint; use crate::line_truncation::truncate_line_with_ellipsis_if_overflow; -use crate::render::Insets; -use crate::render::RectExt as _; -use crate::selection_list::selection_option_row_with_dim; use crate::style::accent_style; use crate::tui::FrameRequester; use crate::tui::Tui; @@ -15,18 +11,14 @@ use crossterm::event::KeyEvent; use crossterm::event::KeyEventKind; use crossterm::event::KeyModifiers; use ratatui::buffer::Buffer; -use ratatui::layout::Constraint; -use ratatui::layout::Layout; use ratatui::layout::Rect; use ratatui::prelude::Stylize as _; use ratatui::text::Line; -use ratatui::widgets::Clear; -use ratatui::widgets::Paragraph; use ratatui::widgets::Widget; -use ratatui::widgets::WidgetRef; -use ratatui::widgets::Wrap; use tokio_stream::StreamExt; +mod render; + #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) enum ExternalAgentConfigMigrationOutcome { Proceed(Vec), @@ -633,124 +625,6 @@ impl ExternalAgentConfigMigrationScreen { } } -impl WidgetRef for &ExternalAgentConfigMigrationScreen { - fn render_ref(&self, area: Rect, buf: &mut Buffer) { - Clear.render(area, buf); - - let inner_area = area.inset(Insets::vh(/*v*/ 1, /*h*/ 2)); - let error_height = u16::from(self.error.is_some()); - let fixed_height = 1u16 + 2u16 + error_height + 1u16 + 4u16 + 1u16; - let list_height = - self.render_line_count() - .max(1) - .min(inner_area.height.saturating_sub(fixed_height) as usize) as u16; - let [ - header_area, - intro_area, - error_area, - list_area, - list_gap_area, - actions_area, - footer_area, - _spacer_area, - ] = Layout::vertical([ - Constraint::Length(1), - Constraint::Length(2), - Constraint::Length(error_height), - Constraint::Length(list_height), - Constraint::Length(1), - Constraint::Length(4), - Constraint::Length(1), - Constraint::Fill(1), - ]) - .areas(inner_area); - - let heading = Line::from(vec!["> ".into(), "External agent config detected".bold()]); - heading.render(header_area, buf); - - Paragraph::new(vec![ - Line::from("We found settings from another agent that you can add to this project."), - Line::from("Select what to import"), - ]) - .wrap(Wrap { trim: false }) - .render(intro_area, buf); - - if let Some(error) = &self.error { - Paragraph::new(error.clone().red().to_string()) - .wrap(Wrap { trim: false }) - .render(error_area, buf); - } - - self.render_items(list_area, buf); - Clear.render(list_gap_area, buf); - - let [ - actions_intro_area, - proceed_area, - skip_area, - skip_forever_area, - ] = Layout::vertical([ - Constraint::Length(1), - Constraint::Length(1), - Constraint::Length(1), - Constraint::Length(1), - ]) - .areas(actions_area); - let actions_intro = format!( - "Selected {} of {} item(s).", - self.selected_count(), - self.items.len() - ); - Paragraph::new(actions_intro) - .wrap(Wrap { trim: false }) - .render(actions_intro_area, buf); - selection_option_row_with_dim( - /*index*/ 0, - ActionMenuOption::Proceed.label().to_string(), - self.focus == FocusArea::Actions - && self.highlighted_action == ActionMenuOption::Proceed, - /*dim*/ self.focus != FocusArea::Actions || !self.proceed_enabled(), - ) - .render(proceed_area, buf); - selection_option_row_with_dim( - /*index*/ 1, - ActionMenuOption::Skip.label().to_string(), - self.focus == FocusArea::Actions && self.highlighted_action == ActionMenuOption::Skip, - /*dim*/ self.focus != FocusArea::Actions, - ) - .render(skip_area, buf); - selection_option_row_with_dim( - /*index*/ 2, - ActionMenuOption::SkipForever.label().to_string(), - self.focus == FocusArea::Actions - && self.highlighted_action == ActionMenuOption::SkipForever, - /*dim*/ self.focus != FocusArea::Actions, - ) - .render(skip_forever_area, buf); - - Line::from(vec![ - "Use ".dim(), - key_hint::plain(KeyCode::Up).into(), - "/".dim(), - key_hint::plain(KeyCode::Down).into(), - " to move, ".dim(), - key_hint::plain(KeyCode::Char(' ')).into(), - " to toggle, ".dim(), - "1".cyan(), - "/".dim(), - "2".cyan(), - "/".dim(), - "3".cyan(), - " to choose, ".dim(), - "a".cyan(), - "/".dim(), - "n".cyan(), - " for all/none".dim(), - ]) - .render(footer_area, buf); - } -} - fn is_ctrl_exit_combo(key_event: KeyEvent) -> bool { key_event.modifiers.contains(KeyModifiers::CONTROL) && matches!(key_event.code, KeyCode::Char('c') | KeyCode::Char('d')) diff --git a/codex-rs/tui/src/external_agent_config_migration/render.rs b/codex-rs/tui/src/external_agent_config_migration/render.rs new file mode 100644 index 0000000000..775ac02148 --- /dev/null +++ b/codex-rs/tui/src/external_agent_config_migration/render.rs @@ -0,0 +1,137 @@ +use super::ActionMenuOption; +use super::ExternalAgentConfigMigrationScreen; +use super::FocusArea; +use crate::key_hint; +use crate::render::Insets; +use crate::render::RectExt as _; +use crate::selection_list::selection_option_row_with_dim; +use crossterm::event::KeyCode; +use ratatui::buffer::Buffer; +use ratatui::layout::Constraint; +use ratatui::layout::Layout; +use ratatui::layout::Rect; +use ratatui::prelude::Stylize as _; +use ratatui::text::Line; +use ratatui::widgets::Clear; +use ratatui::widgets::Paragraph; +use ratatui::widgets::Widget; +use ratatui::widgets::WidgetRef; +use ratatui::widgets::Wrap; + +impl WidgetRef for &ExternalAgentConfigMigrationScreen { + fn render_ref(&self, area: Rect, buf: &mut Buffer) { + Clear.render(area, buf); + + let inner_area = area.inset(Insets::vh(/*v*/ 1, /*h*/ 2)); + let error_height = u16::from(self.error.is_some()); + let fixed_height = 1u16 + 2u16 + error_height + 1u16 + 4u16 + 1u16; + let list_height = + self.render_line_count() + .max(1) + .min(inner_area.height.saturating_sub(fixed_height) as usize) as u16; + let [ + header_area, + intro_area, + error_area, + list_area, + list_gap_area, + actions_area, + footer_area, + _spacer_area, + ] = Layout::vertical([ + Constraint::Length(1), + Constraint::Length(2), + Constraint::Length(error_height), + Constraint::Length(list_height), + Constraint::Length(1), + Constraint::Length(4), + Constraint::Length(1), + Constraint::Fill(1), + ]) + .areas(inner_area); + + let heading = Line::from(vec!["> ".into(), "External agent config detected".bold()]); + heading.render(header_area, buf); + + Paragraph::new(vec![ + Line::from("We found settings from another agent that you can add to this project."), + Line::from("Select what to import"), + ]) + .wrap(Wrap { trim: false }) + .render(intro_area, buf); + + if let Some(error) = &self.error { + Paragraph::new(error.clone().red().to_string()) + .wrap(Wrap { trim: false }) + .render(error_area, buf); + } + + self.render_items(list_area, buf); + Clear.render(list_gap_area, buf); + + let [ + actions_intro_area, + proceed_area, + skip_area, + skip_forever_area, + ] = Layout::vertical([ + Constraint::Length(1), + Constraint::Length(1), + Constraint::Length(1), + Constraint::Length(1), + ]) + .areas(actions_area); + let actions_intro = format!( + "Selected {} of {} item(s).", + self.selected_count(), + self.items.len() + ); + Paragraph::new(actions_intro) + .wrap(Wrap { trim: false }) + .render(actions_intro_area, buf); + selection_option_row_with_dim( + /*index*/ 0, + ActionMenuOption::Proceed.label().to_string(), + self.focus == FocusArea::Actions + && self.highlighted_action == ActionMenuOption::Proceed, + /*dim*/ self.focus != FocusArea::Actions || !self.proceed_enabled(), + ) + .render(proceed_area, buf); + selection_option_row_with_dim( + /*index*/ 1, + ActionMenuOption::Skip.label().to_string(), + self.focus == FocusArea::Actions && self.highlighted_action == ActionMenuOption::Skip, + /*dim*/ self.focus != FocusArea::Actions, + ) + .render(skip_area, buf); + selection_option_row_with_dim( + /*index*/ 2, + ActionMenuOption::SkipForever.label().to_string(), + self.focus == FocusArea::Actions + && self.highlighted_action == ActionMenuOption::SkipForever, + /*dim*/ self.focus != FocusArea::Actions, + ) + .render(skip_forever_area, buf); + + Line::from(vec![ + "Use ".dim(), + key_hint::plain(KeyCode::Up).into(), + "/".dim(), + key_hint::plain(KeyCode::Down).into(), + " to move, ".dim(), + key_hint::plain(KeyCode::Char(' ')).into(), + " to toggle, ".dim(), + "1".cyan(), + "/".dim(), + "2".cyan(), + "/".dim(), + "3".cyan(), + " to choose, ".dim(), + "a".cyan(), + "/".dim(), + "n".cyan(), + " for all/none".dim(), + ]) + .render(footer_area, buf); + } +} From 020bf49346efa6c781726c86eda2a59ff415a712 Mon Sep 17 00:00:00 2001 From: Alex Daley Date: Wed, 10 Jun 2026 12:01:03 -0700 Subject: [PATCH 47/47] [plugins] Inject remote_plugin_id into install elicitations (#26409) Summary - Propagate cached remote plugin IDs through Codex plugin discovery. - Inject `remote_plugin_id` and connector IDs into `request_plugin_install` elicitation `_meta` from the resolved plugin. - Keep the remote plugin ID out of the model-facing tool schema, arguments, and result. Validation - `just test -p codex-tools` - `just test -p codex-core-plugins` - `just test -p codex-core list_tool_suggest_discoverable_plugins_includes_cached_remote_global_plugins` - `just fix -p codex-tools` - `just fix -p codex-core-plugins` - `just fix -p codex-core` - `git diff --check` - `just test -p codex-core` was also attempted: 2,581 passed, 55 failed, and 1 timed out across unrelated sandbox/environment-sensitive integration tests. --- codex-rs/core-plugins/src/discoverable.rs | 3 ++ codex-rs/core/src/plugins/discoverable.rs | 1 + .../core/src/plugins/discoverable_tests.rs | 3 ++ .../handlers/request_plugin_install_tests.rs | 1 + codex-rs/core/src/tools/spec_plan_tests.rs | 1 + codex-rs/tools/src/request_plugin_install.rs | 29 +++++++++------ .../tools/src/request_plugin_install_tests.rs | 36 +++++++++++++++---- codex-rs/tools/src/tool_discovery.rs | 1 + codex-rs/tools/src/tool_discovery_tests.rs | 1 + 9 files changed, 58 insertions(+), 18 deletions(-) diff --git a/codex-rs/core-plugins/src/discoverable.rs b/codex-rs/core-plugins/src/discoverable.rs index 28a90c1928..d0742e9807 100644 --- a/codex-rs/core-plugins/src/discoverable.rs +++ b/codex-rs/core-plugins/src/discoverable.rs @@ -68,6 +68,7 @@ pub struct ToolSuggestPluginDiscoveryInput { #[derive(Clone, Debug, PartialEq, Eq)] pub struct ToolSuggestDiscoverablePlugin { pub id: String, + pub remote_plugin_id: Option, pub name: String, pub description: Option, pub has_skills: bool, @@ -162,6 +163,7 @@ impl PluginsManager { discoverable_plugins.push(ToolSuggestDiscoverablePlugin { id: plugin.config_name, + remote_plugin_id: None, name: plugin.display_name, description: plugin.description, has_skills: plugin.has_skills, @@ -217,6 +219,7 @@ impl PluginsManager { discoverable_plugins.push(ToolSuggestDiscoverablePlugin { id: plugin.config_id, + remote_plugin_id: Some(plugin.remote_plugin_id), name: plugin.name, description: plugin.description, has_skills: plugin.has_skills, diff --git a/codex-rs/core/src/plugins/discoverable.rs b/codex-rs/core/src/plugins/discoverable.rs index 8ddec7056e..fc77ae21e6 100644 --- a/codex-rs/core/src/plugins/discoverable.rs +++ b/codex-rs/core/src/plugins/discoverable.rs @@ -41,6 +41,7 @@ pub(crate) async fn list_tool_suggest_discoverable_plugins( .into_iter() .map(|plugin| DiscoverablePluginInfo { id: plugin.id, + remote_plugin_id: plugin.remote_plugin_id, name: plugin.name, description: plugin.description, has_skills: plugin.has_skills, diff --git a/codex-rs/core/src/plugins/discoverable_tests.rs b/codex-rs/core/src/plugins/discoverable_tests.rs index c1dfe8e639..a6da04e77c 100644 --- a/codex-rs/core/src/plugins/discoverable_tests.rs +++ b/codex-rs/core/src/plugins/discoverable_tests.rs @@ -393,6 +393,7 @@ remote_plugin = true remote_plugins, vec![DiscoverablePluginInfo { id: "github@openai-curated-remote".to_string(), + remote_plugin_id: Some("plugins~Plugin_remote_github".to_string()), name: "Remote GitHub".to_string(), description: Some("Remote GitHub short".to_string()), has_skills: true, @@ -680,6 +681,7 @@ async fn list_tool_suggest_discoverable_plugins_normalizes_description() { discoverable_plugins, vec![DiscoverablePluginInfo { id: "slack@openai-curated".to_string(), + remote_plugin_id: None, name: "slack".to_string(), description: Some("Plugin with extra spacing".to_string()), has_skills: true, @@ -816,6 +818,7 @@ discoverables = [{ type = "plugin", id = "sample@openai-curated" }] discoverable_plugins, vec![DiscoverablePluginInfo { id: "sample@openai-curated".to_string(), + remote_plugin_id: None, name: "sample".to_string(), description: Some( "Plugin that includes skills, MCP servers, and app connectors".to_string(), diff --git a/codex-rs/core/src/tools/handlers/request_plugin_install_tests.rs b/codex-rs/core/src/tools/handlers/request_plugin_install_tests.rs index 830e9d408e..cacc25ff95 100644 --- a/codex-rs/core/src/tools/handlers/request_plugin_install_tests.rs +++ b/codex-rs/core/src/tools/handlers/request_plugin_install_tests.rs @@ -130,6 +130,7 @@ async fn persist_disabled_install_request_writes_plugin_config() { let codex_home = tempdir().expect("tempdir should succeed"); let tool = DiscoverableTool::Plugin(Box::new(DiscoverablePluginInfo { id: "slack@openai-curated".to_string(), + remote_plugin_id: None, name: "Slack".to_string(), description: None, has_skills: true, diff --git a/codex-rs/core/src/tools/spec_plan_tests.rs b/codex-rs/core/src/tools/spec_plan_tests.rs index a9eb5012d5..dce1082bb8 100644 --- a/codex-rs/core/src/tools/spec_plan_tests.rs +++ b/codex-rs/core/src/tools/spec_plan_tests.rs @@ -400,6 +400,7 @@ fn dynamic_tool(namespace: Option<&str>, name: &str, defer_loading: bool) -> Dyn fn discoverable_plugin(id: &str, name: &str) -> DiscoverableTool { DiscoverablePluginInfo { id: id.to_string(), + remote_plugin_id: None, name: name.to_string(), description: Some(format!("{name} plugin")), has_skills: false, diff --git a/codex-rs/tools/src/request_plugin_install.rs b/codex-rs/tools/src/request_plugin_install.rs index 70e9cb093d..5d6352b7f8 100644 --- a/codex-rs/tools/src/request_plugin_install.rs +++ b/codex-rs/tools/src/request_plugin_install.rs @@ -47,6 +47,10 @@ pub struct RequestPluginInstallMeta<'a> { pub tool_name: &'a str, #[serde(skip_serializing_if = "Option::is_none")] pub install_url: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + pub remote_plugin_id: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + pub app_connector_ids: Option<&'a [String]>, } pub fn build_request_plugin_install_elicitation_request( @@ -57,8 +61,6 @@ pub fn build_request_plugin_install_elicitation_request( suggest_reason: &str, tool: &DiscoverableTool, ) -> McpServerElicitationRequestParams { - let tool_name = tool.name().to_string(); - let install_url = tool.install_url().map(ToString::to_string); let message = suggest_reason.to_string(); McpServerElicitationRequestParams { @@ -70,9 +72,7 @@ pub fn build_request_plugin_install_elicitation_request( args.tool_type, args.action_type, suggest_reason, - tool.id(), - tool_name.as_str(), - install_url.as_deref(), + tool, ))), message, requested_schema: McpElicitationSchema { @@ -108,19 +108,26 @@ fn build_request_plugin_install_meta<'a>( tool_type: DiscoverableToolType, action_type: DiscoverableToolAction, suggest_reason: &'a str, - tool_id: &'a str, - tool_name: &'a str, - install_url: Option<&'a str>, + tool: &'a DiscoverableTool, ) -> RequestPluginInstallMeta<'a> { + let (remote_plugin_id, app_connector_ids) = match tool { + DiscoverableTool::Connector(_) => (None, None), + DiscoverableTool::Plugin(plugin) => ( + plugin.remote_plugin_id.as_deref(), + Some(plugin.app_connector_ids.as_slice()), + ), + }; RequestPluginInstallMeta { codex_approval_kind: REQUEST_PLUGIN_INSTALL_APPROVAL_KIND_VALUE, persist: REQUEST_PLUGIN_INSTALL_PERSIST_ALWAYS_VALUE, tool_type, suggest_type: action_type, suggest_reason, - tool_id, - tool_name, - install_url, + tool_id: tool.id(), + tool_name: tool.name(), + install_url: tool.install_url(), + remote_plugin_id, + app_connector_ids, } } diff --git a/codex-rs/tools/src/request_plugin_install_tests.rs b/codex-rs/tools/src/request_plugin_install_tests.rs index ff2370ade7..ab8af9be67 100644 --- a/codex-rs/tools/src/request_plugin_install_tests.rs +++ b/codex-rs/tools/src/request_plugin_install_tests.rs @@ -57,6 +57,8 @@ fn build_request_plugin_install_elicitation_request_uses_expected_shape() { install_url: Some( "https://chatgpt.com/apps/google-calendar/connector_2128aebfecb84f64a069897515042a44" ), + remote_plugin_id: None, + app_connector_ids: None, })), message: "Plan and reference events from your calendar".to_string(), requested_schema: McpElicitationSchema { @@ -71,15 +73,16 @@ fn build_request_plugin_install_elicitation_request_uses_expected_shape() { } #[test] -fn build_request_plugin_install_elicitation_request_for_plugin_omits_install_url() { +fn build_request_plugin_install_elicitation_request_injects_plugin_metadata() { let args = RequestPluginInstallArgs { tool_type: DiscoverableToolType::Plugin, action_type: DiscoverableToolAction::Install, - tool_id: "sample@openai-curated".to_string(), + tool_id: "sample@openai-curated-remote".to_string(), suggest_reason: "Use the sample plugin's skills and MCP server".to_string(), }; let plugin = DiscoverableTool::Plugin(Box::new(DiscoverablePluginInfo { - id: "sample@openai-curated".to_string(), + id: "sample@openai-curated-remote".to_string(), + remote_plugin_id: Some("plugins~Plugin_sample".to_string()), name: "Sample Plugin".to_string(), description: Some("Includes skills, MCP servers, and apps.".to_string()), has_skills: true, @@ -109,9 +112,11 @@ fn build_request_plugin_install_elicitation_request_for_plugin_omits_install_url tool_type: DiscoverableToolType::Plugin, suggest_type: DiscoverableToolAction::Install, suggest_reason: "Use the sample plugin's skills and MCP server", - tool_id: "sample@openai-curated", + tool_id: "sample@openai-curated-remote", tool_name: "Sample Plugin", install_url: None, + remote_plugin_id: Some("plugins~Plugin_sample"), + app_connector_ids: Some(&["connector_calendar".to_string()]), })), message: "Use the sample plugin's skills and MCP server".to_string(), requested_schema: McpElicitationSchema { @@ -127,13 +132,28 @@ fn build_request_plugin_install_elicitation_request_for_plugin_omits_install_url #[test] fn build_request_plugin_install_meta_uses_expected_shape() { + let connector = DiscoverableTool::Connector(Box::new(AppInfo { + id: "connector_68df038e0ba48191908c8434991bbac2".to_string(), + name: "Gmail".to_string(), + description: None, + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + branding: None, + app_metadata: None, + labels: None, + install_url: Some( + "https://chatgpt.com/apps/gmail/connector_68df038e0ba48191908c8434991bbac2".to_string(), + ), + is_accessible: false, + is_enabled: true, + plugin_display_names: Vec::new(), + })); let meta = build_request_plugin_install_meta( DiscoverableToolType::Connector, DiscoverableToolAction::Install, "Find and reference emails from your inbox", - "connector_68df038e0ba48191908c8434991bbac2", - "Gmail", - Some("https://chatgpt.com/apps/gmail/connector_68df038e0ba48191908c8434991bbac2"), + &connector, ); assert_eq!( @@ -149,6 +169,8 @@ fn build_request_plugin_install_meta_uses_expected_shape() { install_url: Some( "https://chatgpt.com/apps/gmail/connector_68df038e0ba48191908c8434991bbac2" ), + remote_plugin_id: None, + app_connector_ids: None, }, ); } diff --git a/codex-rs/tools/src/tool_discovery.rs b/codex-rs/tools/src/tool_discovery.rs index ac5aac904b..2584d7f17a 100644 --- a/codex-rs/tools/src/tool_discovery.rs +++ b/codex-rs/tools/src/tool_discovery.rs @@ -93,6 +93,7 @@ pub fn filter_request_plugin_install_discoverable_tools_for_client( #[derive(Clone, Debug, PartialEq, Eq)] pub struct DiscoverablePluginInfo { pub id: String, + pub remote_plugin_id: Option, pub name: String, pub description: Option, pub has_skills: bool, diff --git a/codex-rs/tools/src/tool_discovery_tests.rs b/codex-rs/tools/src/tool_discovery_tests.rs index 6e45260c0e..5a3310057c 100644 --- a/codex-rs/tools/src/tool_discovery_tests.rs +++ b/codex-rs/tools/src/tool_discovery_tests.rs @@ -37,6 +37,7 @@ fn filter_request_plugin_install_discoverable_tools_for_codex_tui_omits_plugins( })), DiscoverableTool::Plugin(Box::new(DiscoverablePluginInfo { id: "slack@openai-curated".to_string(), + remote_plugin_id: None, name: "Slack".to_string(), description: Some("Search Slack messages".to_string()), has_skills: true,