From 77fe33bf72a7a1a0f2687c0487b6558fd2a82d1c Mon Sep 17 00:00:00 2001 From: Matthew Zeng Date: Wed, 15 Apr 2026 22:01:05 -0700 Subject: [PATCH 01/59] Update ToolSearch to be enabled by default (#17854) ## Summary - Promote `Feature::ToolSearch` to `Stable` and enable it in the default feature set - Update feature tests and tool registry coverage to match the new default - Adjust the search-tool integration test to assert the default-on path and explicit disable fallback ## Testing - `just fmt` - `cargo test -p codex-features` - `cargo test -p codex-core --test all search_tool` - `cargo test -p codex-tools` --- codex-rs/core/tests/suite/search_tool.rs | 16 ++++++++++------ codex-rs/features/src/lib.rs | 4 ++-- codex-rs/features/src/tests.rs | 6 +++--- codex-rs/tools/src/tool_registry_plan_tests.rs | 8 ++++---- 4 files changed, 19 insertions(+), 15 deletions(-) diff --git a/codex-rs/core/tests/suite/search_tool.rs b/codex-rs/core/tests/suite/search_tool.rs index 74de499d70..624c439150 100644 --- a/codex-rs/core/tests/suite/search_tool.rs +++ b/codex-rs/core/tests/suite/search_tool.rs @@ -93,7 +93,7 @@ fn tool_search_output_tools(request: &ResponsesRequest, call_id: &str) -> Vec TestCodexBuilder { test_codex() .with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing()) @@ -127,7 +131,7 @@ fn configured_builder(apps_base_url: String) -> TestCodexBuilder { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn search_tool_flag_adds_tool_search() -> Result<()> { +async fn search_tool_enabled_by_default_adds_tool_search() -> Result<()> { skip_if_no_network!(Ok(())); let server = start_mock_server().await; @@ -185,7 +189,7 @@ async fn search_tool_flag_adds_tool_search() -> Result<()> { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn tool_search_disabled_by_default_exposes_apps_tools_directly() -> Result<()> { +async fn tool_search_disabled_exposes_apps_tools_directly() -> Result<()> { skip_if_no_network!(Ok(())); let server = start_mock_server().await; diff --git a/codex-rs/features/src/lib.rs b/codex-rs/features/src/lib.rs index ce7a196362..2d1ac89851 100644 --- a/codex-rs/features/src/lib.rs +++ b/codex-rs/features/src/lib.rs @@ -788,8 +788,8 @@ pub const FEATURES: &[FeatureSpec] = &[ FeatureSpec { id: Feature::ToolSearch, key: "tool_search", - stage: Stage::UnderDevelopment, - default_enabled: false, + stage: Stage::Stable, + default_enabled: true, }, FeatureSpec { id: Feature::UnavailableDummyTools, diff --git a/codex-rs/features/src/tests.rs b/codex-rs/features/src/tests.rs index 85cb01be23..56f92d55b7 100644 --- a/codex-rs/features/src/tests.rs +++ b/codex-rs/features/src/tests.rs @@ -128,9 +128,9 @@ fn tool_suggest_is_stable_and_enabled_by_default() { } #[test] -fn tool_search_is_under_development_and_disabled_by_default() { - assert_eq!(Feature::ToolSearch.stage(), Stage::UnderDevelopment); - assert_eq!(Feature::ToolSearch.default_enabled(), false); +fn tool_search_is_stable_and_enabled_by_default() { + assert_eq!(Feature::ToolSearch.stage(), Stage::Stable); + assert_eq!(Feature::ToolSearch.default_enabled(), true); } #[test] diff --git a/codex-rs/tools/src/tool_registry_plan_tests.rs b/codex-rs/tools/src/tool_registry_plan_tests.rs index a643543ee7..50b38905ee 100644 --- a/codex-rs/tools/src/tool_registry_plan_tests.rs +++ b/codex-rs/tools/src/tool_registry_plan_tests.rs @@ -1334,7 +1334,7 @@ fn search_tool_description_lists_each_mcp_source_once() { } #[test] -fn search_tool_requires_model_capability_and_feature_flag() { +fn search_tool_requires_model_capability_and_enabled_feature() { let model_info = search_capable_model_info(); let deferred_mcp_tools = Some(vec![deferred_mcp_tool( "_create_event", @@ -1367,10 +1367,12 @@ fn search_tool_requires_model_capability_and_feature_flag() { ); assert_lacks_tool_name(&tools, TOOL_SEARCH_TOOL_NAME); + let mut features_without_tool_search = Features::with_defaults(); + features_without_tool_search.disable(Feature::ToolSearch); let tools_config = ToolsConfig::new(&ToolsConfigParams { model_info: &model_info, available_models: &available_models, - features: &features, + features: &features_without_tool_search, image_generation_tool_auth_allowed: true, web_search_mode: Some(WebSearchMode::Cached), session_source: SessionSource::Cli, @@ -1385,8 +1387,6 @@ fn search_tool_requires_model_capability_and_feature_flag() { ); assert_lacks_tool_name(&tools, TOOL_SEARCH_TOOL_NAME); - let mut features = Features::with_defaults(); - features.enable(Feature::ToolSearch); let tools_config = ToolsConfig::new(&ToolsConfigParams { model_info: &model_info, available_models: &available_models, From 224dad41ac1bdf0c8a848b1fd0068262f1f99223 Mon Sep 17 00:00:00 2001 From: Matthew Zeng Date: Wed, 15 Apr 2026 22:09:17 -0700 Subject: [PATCH 02/59] [codex][mcp] Add resource uri meta to tool call item. (#17831) - [x] Add resource uri meta to tool call item so that the app-server client can start prefetching resources immediately without loading mcp server status. --- .../schema/json/ServerNotification.json | 6 +++ .../codex_app_server_protocol.schemas.json | 6 +++ .../codex_app_server_protocol.v2.schemas.json | 6 +++ .../json/v2/ItemCompletedNotification.json | 6 +++ .../json/v2/ItemStartedNotification.json | 6 +++ .../schema/json/v2/ReviewStartResponse.json | 6 +++ .../schema/json/v2/ThreadForkResponse.json | 6 +++ .../schema/json/v2/ThreadListResponse.json | 6 +++ .../json/v2/ThreadMetadataUpdateResponse.json | 6 +++ .../schema/json/v2/ThreadReadResponse.json | 6 +++ .../schema/json/v2/ThreadResumeResponse.json | 6 +++ .../json/v2/ThreadRollbackResponse.json | 6 +++ .../schema/json/v2/ThreadStartResponse.json | 6 +++ .../json/v2/ThreadStartedNotification.json | 6 +++ .../json/v2/ThreadUnarchiveResponse.json | 6 +++ .../json/v2/TurnCompletedNotification.json | 6 +++ .../schema/json/v2/TurnStartResponse.json | 6 +++ .../json/v2/TurnStartedNotification.json | 6 +++ .../schema/typescript/v2/ThreadItem.ts | 2 +- .../src/protocol/common.rs | 1 + .../src/protocol/thread_history.rs | 14 +++++-- .../app-server-protocol/src/protocol/v2.rs | 5 ++- .../app-server/src/bespoke_event_handling.rs | 18 ++++++-- codex-rs/core/src/mcp_tool_call.rs | 37 +++++++++++++++++ codex-rs/core/src/mcp_tool_call_tests.rs | 41 +++++++++++++++++++ .../core/src/tools/handlers/mcp_resource.rs | 2 + .../core/tests/common/apps_test_server.rs | 3 ++ codex-rs/core/tests/suite/rmcp_client.rs | 1 + codex-rs/core/tests/suite/search_tool.rs | 18 ++++++++ .../tests/event_processor_with_json_output.rs | 13 ++++-- codex-rs/protocol/src/protocol.rs | 6 +++ codex-rs/tui/src/chatwidget.rs | 20 ++++++--- 32 files changed, 269 insertions(+), 20 deletions(-) diff --git a/codex-rs/app-server-protocol/schema/json/ServerNotification.json b/codex-rs/app-server-protocol/schema/json/ServerNotification.json index a613590e3f..b84fbe9a02 100644 --- a/codex-rs/app-server-protocol/schema/json/ServerNotification.json +++ b/codex-rs/app-server-protocol/schema/json/ServerNotification.json @@ -2891,6 +2891,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index d0ce368b78..943b8480f9 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -13369,6 +13369,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index aca9b23880..1717dfb393 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -11213,6 +11213,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json index 4e8cfcb621..c5a84c4272 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ItemCompletedNotification.json @@ -787,6 +787,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json index 5defb06618..cb9201f0a6 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ItemStartedNotification.json @@ -787,6 +787,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json index b7490d5e12..05dfaa4b72 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ReviewStartResponse.json @@ -930,6 +930,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json index 6db7d990f0..909c2af4d8 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadForkResponse.json @@ -1444,6 +1444,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json index 7435fbeff7..8ebebd39f9 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadListResponse.json @@ -1206,6 +1206,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json index c6abbab166..40da3390ec 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadMetadataUpdateResponse.json @@ -1206,6 +1206,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json index fb658030f4..400c0b661a 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadReadResponse.json @@ -1206,6 +1206,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json index 9879db55f6..aef0cd737c 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeResponse.json @@ -1444,6 +1444,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json index 84d2768855..6a50d6d978 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadRollbackResponse.json @@ -1206,6 +1206,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json index 42d76d6818..a3d20555f1 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartResponse.json @@ -1444,6 +1444,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json index 4b24a129bb..c4b1e54490 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadStartedNotification.json @@ -1206,6 +1206,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json index bebc8a6bbf..e55c9651fa 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadUnarchiveResponse.json @@ -1206,6 +1206,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json index 381b78ab71..2848940085 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json @@ -930,6 +930,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json index 02727fbadd..c3f31e5322 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartResponse.json @@ -930,6 +930,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json index 25c15d78dc..7e8ca749c0 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json @@ -930,6 +930,12 @@ "id": { "type": "string" }, + "mcpAppResourceUri": { + "type": [ + "string", + "null" + ] + }, "result": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadItem.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadItem.ts index e7fe940aa7..747a70c54d 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadItem.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadItem.ts @@ -53,7 +53,7 @@ exitCode: number | null, /** * The duration of the command execution in milliseconds. */ -durationMs: number | null, } | { "type": "fileChange", id: string, changes: Array, status: PatchApplyStatus, } | { "type": "mcpToolCall", id: string, server: string, tool: string, status: McpToolCallStatus, arguments: JsonValue, result: McpToolCallResult | null, error: McpToolCallError | null, +durationMs: number | null, } | { "type": "fileChange", id: string, changes: Array, status: PatchApplyStatus, } | { "type": "mcpToolCall", id: string, server: string, tool: string, status: McpToolCallStatus, arguments: JsonValue, mcpAppResourceUri?: string, result: McpToolCallResult | null, error: McpToolCallError | null, /** * The duration of the MCP tool call in milliseconds. */ diff --git a/codex-rs/app-server-protocol/src/protocol/common.rs b/codex-rs/app-server-protocol/src/protocol/common.rs index 3d6bf9d872..7f555e0688 100644 --- a/codex-rs/app-server-protocol/src/protocol/common.rs +++ b/codex-rs/app-server-protocol/src/protocol/common.rs @@ -764,6 +764,7 @@ macro_rules! server_notification_definitions { Display, ExperimentalApi, )] + #[allow(clippy::large_enum_variant)] #[serde(tag = "method", content = "params", rename_all = "camelCase")] #[strum(serialize_all = "camelCase")] pub enum ServerNotification { diff --git a/codex-rs/app-server-protocol/src/protocol/thread_history.rs b/codex-rs/app-server-protocol/src/protocol/thread_history.rs index 9e94515dd9..54f882cf29 100644 --- a/codex-rs/app-server-protocol/src/protocol/thread_history.rs +++ b/codex-rs/app-server-protocol/src/protocol/thread_history.rs @@ -502,6 +502,7 @@ impl ThreadHistoryBuilder { .arguments .clone() .unwrap_or(serde_json::Value::Null), + mcp_app_resource_uri: payload.mcp_app_resource_uri.clone(), result: None, error: None, duration_ms: None, @@ -518,11 +519,11 @@ impl ThreadHistoryBuilder { let duration_ms = i64::try_from(payload.duration.as_millis()).ok(); let (result, error) = match &payload.result { Ok(value) => ( - Some(McpToolCallResult { + Some(Box::new(McpToolCallResult { content: value.content.clone(), structured_content: value.structured_content.clone(), meta: value.meta.clone(), - }), + })), None, ), Err(message) => ( @@ -542,6 +543,7 @@ impl ThreadHistoryBuilder { .arguments .clone() .unwrap_or(serde_json::Value::Null), + mcp_app_resource_uri: payload.mcp_app_resource_uri.clone(), result, error, duration_ms, @@ -1809,6 +1811,7 @@ mod tests { tool: "lookup".into(), arguments: Some(serde_json::json!({"id":"123"})), }, + mcp_app_resource_uri: None, duration: Duration::from_millis(8), result: Err("boom".into()), }), @@ -1857,6 +1860,7 @@ mod tests { tool: "lookup".into(), status: McpToolCallStatus::Failed, arguments: serde_json::json!({"id":"123"}), + mcp_app_resource_uri: None, result: None, error: Some(McpToolCallError { message: "boom".into(), @@ -1882,6 +1886,7 @@ mod tests { tool: "lookup".into(), arguments: Some(serde_json::json!({"id":"123"})), }, + mcp_app_resource_uri: Some("ui://widget/lookup.html".into()), duration: Duration::from_millis(8), result: Ok(CallToolResult { content: vec![serde_json::json!({ @@ -1911,7 +1916,8 @@ mod tests { tool: "lookup".into(), status: McpToolCallStatus::Completed, arguments: serde_json::json!({"id":"123"}), - result: Some(McpToolCallResult { + mcp_app_resource_uri: Some("ui://widget/lookup.html".into()), + result: Some(Box::new(McpToolCallResult { content: vec![serde_json::json!({ "type": "text", "text": "result" @@ -1920,7 +1926,7 @@ mod tests { meta: Some(serde_json::json!({ "ui/resourceUri": "ui://widget/lookup.html" })), - }), + })), error: None, duration_ms: Some(8), } diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index 5897625de1..155c67e7bf 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -4585,7 +4585,10 @@ pub enum ThreadItem { tool: String, status: McpToolCallStatus, arguments: JsonValue, - result: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + mcp_app_resource_uri: Option, + result: Option>, error: Option, /// The duration of the MCP tool call in milliseconds. #[ts(type = "number | null")] diff --git a/codex-rs/app-server/src/bespoke_event_handling.rs b/codex-rs/app-server/src/bespoke_event_handling.rs index 4fddee4a11..5d19bde70c 100644 --- a/codex-rs/app-server/src/bespoke_event_handling.rs +++ b/codex-rs/app-server/src/bespoke_event_handling.rs @@ -2910,6 +2910,7 @@ async fn construct_mcp_tool_call_notification( tool: begin_event.invocation.tool, status: McpToolCallStatus::InProgress, arguments: begin_event.invocation.arguments.unwrap_or(JsonValue::Null), + mcp_app_resource_uri: begin_event.mcp_app_resource_uri, result: None, error: None, duration_ms: None, @@ -2936,11 +2937,11 @@ async fn construct_mcp_tool_call_end_notification( let (result, error) = match &end_event.result { Ok(value) => ( - Some(McpToolCallResult { + Some(Box::new(McpToolCallResult { content: value.content.clone(), structured_content: value.structured_content.clone(), meta: value.meta.clone(), - }), + })), None, ), Err(message) => ( @@ -2957,6 +2958,7 @@ async fn construct_mcp_tool_call_end_notification( tool: end_event.invocation.tool, status, arguments: end_event.invocation.arguments.unwrap_or(JsonValue::Null), + mcp_app_resource_uri: end_event.mcp_app_resource_uri, result, error, duration_ms, @@ -4219,6 +4221,7 @@ mod tests { tool: "list_mcp_resources".to_string(), arguments: Some(serde_json::json!({"server": ""})), }, + mcp_app_resource_uri: Some("ui://widget/list-resources.html".to_string()), }; let thread_id = ThreadId::new().to_string(); @@ -4239,6 +4242,7 @@ mod tests { tool: begin_event.invocation.tool, status: McpToolCallStatus::InProgress, arguments: serde_json::json!({"server": ""}), + mcp_app_resource_uri: Some("ui://widget/list-resources.html".to_string()), result: None, error: None, duration_ms: None, @@ -4379,6 +4383,7 @@ mod tests { tool: "list_mcp_resources".to_string(), arguments: None, }, + mcp_app_resource_uri: None, }; let thread_id = ThreadId::new().to_string(); @@ -4399,6 +4404,7 @@ mod tests { tool: begin_event.invocation.tool, status: McpToolCallStatus::InProgress, arguments: JsonValue::Null, + mcp_app_resource_uri: None, result: None, error: None, duration_ms: None, @@ -4430,6 +4436,7 @@ mod tests { tool: "list_mcp_resources".to_string(), arguments: Some(serde_json::json!({"server": ""})), }, + mcp_app_resource_uri: Some("ui://widget/list-resources.html".to_string()), duration: Duration::from_nanos(92708), result: Ok(result), }; @@ -4452,13 +4459,14 @@ mod tests { tool: end_event.invocation.tool, status: McpToolCallStatus::Completed, arguments: serde_json::json!({"server": ""}), - result: Some(McpToolCallResult { + mcp_app_resource_uri: Some("ui://widget/list-resources.html".to_string()), + result: Some(Box::new(McpToolCallResult { content, structured_content: None, meta: Some(serde_json::json!({ "ui/resourceUri": "ui://widget/list-resources.html" })), - }), + })), error: None, duration_ms: Some(0), }, @@ -4476,6 +4484,7 @@ mod tests { tool: "list_mcp_resources".to_string(), arguments: None, }, + mcp_app_resource_uri: None, duration: Duration::from_millis(1), result: Err("boom".to_string()), }; @@ -4498,6 +4507,7 @@ mod tests { tool: end_event.invocation.tool, status: McpToolCallStatus::Failed, arguments: JsonValue::Null, + mcp_app_resource_uri: None, result: None, error: Some(McpToolCallError { message: "boom".to_string(), diff --git a/codex-rs/core/src/mcp_tool_call.rs b/codex-rs/core/src/mcp_tool_call.rs index 0dd5deae73..064585f447 100644 --- a/codex-rs/core/src/mcp_tool_call.rs +++ b/codex-rs/core/src/mcp_tool_call.rs @@ -101,6 +101,9 @@ pub(crate) async fn handle_mcp_tool_call( let metadata = lookup_mcp_tool_metadata(sess.as_ref(), turn_context.as_ref(), &server, &tool_name).await; + let mcp_app_resource_uri = metadata + .as_ref() + .and_then(|metadata| metadata.mcp_app_resource_uri.clone()); let app_tool_policy = if server == CODEX_APPS_MCP_SERVER_NAME { connectors::app_tool_policy( &turn_context.config, @@ -130,6 +133,7 @@ pub(crate) async fn handle_mcp_tool_call( turn_context.as_ref(), &call_id, invocation, + mcp_app_resource_uri.clone(), "MCP tool call blocked by app configuration".to_string(), /*already_started*/ false, ) @@ -161,6 +165,7 @@ pub(crate) async fn handle_mcp_tool_call( let tool_call_begin_event = EventMsg::McpToolCallBegin(McpToolCallBeginEvent { call_id: call_id.clone(), invocation: invocation.clone(), + mcp_app_resource_uri: mcp_app_resource_uri.clone(), }); notify_mcp_tool_call_event(sess.as_ref(), turn_context.as_ref(), tool_call_begin_event).await; @@ -213,6 +218,7 @@ pub(crate) async fn handle_mcp_tool_call( let tool_call_end_event = EventMsg::McpToolCallEnd(McpToolCallEndEvent { call_id: call_id.clone(), invocation, + mcp_app_resource_uri: mcp_app_resource_uri.clone(), duration, result: result.clone(), }); @@ -239,6 +245,7 @@ pub(crate) async fn handle_mcp_tool_call( turn_context.as_ref(), &call_id, invocation, + mcp_app_resource_uri.clone(), message, /*already_started*/ true, ) @@ -254,6 +261,7 @@ pub(crate) async fn handle_mcp_tool_call( turn_context.as_ref(), &call_id, invocation, + mcp_app_resource_uri.clone(), message, /*already_started*/ true, ) @@ -268,6 +276,7 @@ pub(crate) async fn handle_mcp_tool_call( turn_context.as_ref(), &call_id, invocation, + mcp_app_resource_uri.clone(), message, /*already_started*/ true, ) @@ -325,6 +334,7 @@ pub(crate) async fn handle_mcp_tool_call( let tool_call_end_event = EventMsg::McpToolCallEnd(McpToolCallEndEvent { call_id: call_id.clone(), invocation, + mcp_app_resource_uri, duration, result: result.clone(), }); @@ -642,11 +652,14 @@ pub(crate) struct McpToolApprovalMetadata { connector_description: Option, tool_title: Option, tool_description: Option, + mcp_app_resource_uri: Option, codex_apps_meta: Option>, openai_file_input_params: Option>, } const MCP_TOOL_CODEX_APPS_META_KEY: &str = "_codex_apps"; +const MCP_TOOL_OPENAI_OUTPUT_TEMPLATE_META_KEY: &str = "openai/outputTemplate"; +const MCP_TOOL_UI_RESOURCE_URI_META_KEY: &str = "ui/resourceUri"; fn custom_mcp_tool_approval_mode( turn_context: &TurnContext, @@ -1100,6 +1113,7 @@ pub(crate) async fn lookup_mcp_tool_metadata( connector_description, tool_title: tool_info.tool.title, tool_description: tool_info.tool.description.map(std::borrow::Cow::into_owned), + mcp_app_resource_uri: get_mcp_app_resource_uri(tool_info.tool.meta.as_deref()), codex_apps_meta: tool_info .tool .meta @@ -1114,6 +1128,26 @@ pub(crate) async fn lookup_mcp_tool_metadata( }) } +fn get_mcp_app_resource_uri( + meta: Option<&serde_json::Map>, +) -> Option { + meta.and_then(|meta| { + meta.get("ui") + .and_then(serde_json::Value::as_object) + .and_then(|ui| ui.get("resourceUri")) + .and_then(serde_json::Value::as_str) + .or_else(|| { + meta.get(MCP_TOOL_UI_RESOURCE_URI_META_KEY) + .and_then(serde_json::Value::as_str) + }) + .or_else(|| { + meta.get(MCP_TOOL_OPENAI_OUTPUT_TEMPLATE_META_KEY) + .and_then(serde_json::Value::as_str) + }) + .map(str::to_string) + }) +} + async fn lookup_mcp_app_usage_metadata( sess: &Session, server: &str, @@ -1666,6 +1700,7 @@ async fn notify_mcp_tool_call_skip( turn_context: &TurnContext, call_id: &str, invocation: McpInvocation, + mcp_app_resource_uri: Option, message: String, already_started: bool, ) -> Result { @@ -1673,6 +1708,7 @@ async fn notify_mcp_tool_call_skip( let tool_call_begin_event = EventMsg::McpToolCallBegin(McpToolCallBeginEvent { call_id: call_id.to_string(), invocation: invocation.clone(), + mcp_app_resource_uri: mcp_app_resource_uri.clone(), }); notify_mcp_tool_call_event(sess, turn_context, tool_call_begin_event).await; } @@ -1680,6 +1716,7 @@ async fn notify_mcp_tool_call_skip( let tool_call_end_event = EventMsg::McpToolCallEnd(McpToolCallEndEvent { call_id: call_id.to_string(), invocation, + mcp_app_resource_uri, duration: Duration::ZERO, result: Err(message.clone()), }); diff --git a/codex-rs/core/src/mcp_tool_call_tests.rs b/codex-rs/core/src/mcp_tool_call_tests.rs index 201a162ef7..53e838908d 100644 --- a/codex-rs/core/src/mcp_tool_call_tests.rs +++ b/codex-rs/core/src/mcp_tool_call_tests.rs @@ -59,6 +59,7 @@ fn approval_metadata( connector_description: connector_description.map(str::to_string), tool_title: tool_title.map(str::to_string), tool_description: tool_description.map(str::to_string), + mcp_app_resource_uri: None, codex_apps_meta: None, openai_file_input_params: None, } @@ -74,6 +75,35 @@ fn prompt_options( } } +#[test] +fn mcp_app_resource_uri_reads_known_tool_meta_keys() { + let nested = serde_json::json!({ + "ui": { + "resourceUri": "ui://widget/nested.html", + }, + }); + assert_eq!( + get_mcp_app_resource_uri(nested.as_object()), + Some("ui://widget/nested.html".to_string()) + ); + + let flat = serde_json::json!({ + "ui/resourceUri": "ui://widget/flat.html", + }); + assert_eq!( + get_mcp_app_resource_uri(flat.as_object()), + Some("ui://widget/flat.html".to_string()) + ); + + let output_template = serde_json::json!({ + "openai/outputTemplate": "ui://widget/output-template.html", + }); + assert_eq!( + get_mcp_app_resource_uri(output_template.as_object()), + Some("ui://widget/output-template.html".to_string()) + ); +} + #[test] fn approval_required_when_read_only_false_and_destructive() { let annotations = annotations(Some(false), Some(true), /*open_world*/ None); @@ -589,6 +619,7 @@ async fn codex_apps_tool_call_request_meta_includes_turn_metadata_and_codex_apps connector_description: Some("Manage events".to_string()), tool_title: Some("Create Event".to_string()), tool_description: Some("Create a calendar event.".to_string()), + mcp_app_resource_uri: None, codex_apps_meta: Some( serde_json::json!({ "resource_uri": "connector://calendar/tools/calendar_create_event", @@ -746,6 +777,7 @@ fn guardian_mcp_review_request_includes_annotations_when_present() { connector_description: None, tool_title: None, tool_description: None, + mcp_app_resource_uri: None, codex_apps_meta: None, openai_file_input_params: None, }; @@ -1272,6 +1304,7 @@ async fn approve_mode_skips_when_annotations_do_not_require_approval() { connector_description: None, tool_title: Some("Read Only Tool".to_string()), tool_description: None, + mcp_app_resource_uri: None, codex_apps_meta: None, openai_file_input_params: None, }; @@ -1340,6 +1373,7 @@ async fn guardian_mode_skips_auto_when_annotations_do_not_require_approval() { connector_description: None, tool_title: Some("Read Only Tool".to_string()), tool_description: None, + mcp_app_resource_uri: None, codex_apps_meta: None, openai_file_input_params: None, }; @@ -1411,6 +1445,7 @@ async fn guardian_mode_mcp_denial_returns_rationale_message() { connector_description: None, tool_title: Some("Dangerous Tool".to_string()), tool_description: Some("Reads calendar data.".to_string()), + mcp_app_resource_uri: None, codex_apps_meta: None, openai_file_input_params: None, }; @@ -1462,6 +1497,7 @@ async fn prompt_mode_waits_for_approval_when_annotations_do_not_require_approval connector_description: None, tool_title: Some("Read Only Tool".to_string()), tool_description: None, + mcp_app_resource_uri: None, codex_apps_meta: None, openai_file_input_params: None, }; @@ -1539,6 +1575,7 @@ async fn approve_mode_blocks_when_arc_returns_interrupt_for_model() { connector_description: Some("Manage events".to_string()), tool_title: Some("Dangerous Tool".to_string()), tool_description: Some("Performs a risky action.".to_string()), + mcp_app_resource_uri: None, codex_apps_meta: None, openai_file_input_params: None, }; @@ -1609,6 +1646,7 @@ async fn custom_approve_mode_blocks_when_arc_returns_interrupt_for_model() { connector_description: None, tool_title: Some("Dangerous Tool".to_string()), tool_description: Some("Performs a risky action.".to_string()), + mcp_app_resource_uri: None, codex_apps_meta: None, openai_file_input_params: None, }; @@ -1679,6 +1717,7 @@ async fn approve_mode_blocks_when_arc_returns_interrupt_without_annotations() { connector_description: Some("Manage events".to_string()), tool_title: Some("Dangerous Tool".to_string()), tool_description: Some("Performs a risky action.".to_string()), + mcp_app_resource_uri: None, codex_apps_meta: None, openai_file_input_params: None, }; @@ -1757,6 +1796,7 @@ async fn full_access_mode_skips_arc_monitor_for_all_approval_modes() { connector_description: Some("Manage events".to_string()), tool_title: Some("Dangerous Tool".to_string()), tool_description: Some("Performs a risky action.".to_string()), + mcp_app_resource_uri: None, codex_apps_meta: None, openai_file_input_params: None, }; @@ -1859,6 +1899,7 @@ async fn approve_mode_routes_arc_ask_user_to_guardian_when_guardian_reviewer_is_ connector_description: Some("Manage events".to_string()), tool_title: Some("Dangerous Tool".to_string()), tool_description: Some("Performs a risky action.".to_string()), + mcp_app_resource_uri: None, codex_apps_meta: None, openai_file_input_params: None, }; diff --git a/codex-rs/core/src/tools/handlers/mcp_resource.rs b/codex-rs/core/src/tools/handlers/mcp_resource.rs index 69058a6226..d7d0256dda 100644 --- a/codex-rs/core/src/tools/handlers/mcp_resource.rs +++ b/codex-rs/core/src/tools/handlers/mcp_resource.rs @@ -562,6 +562,7 @@ async fn emit_tool_call_begin( EventMsg::McpToolCallBegin(McpToolCallBeginEvent { call_id: call_id.to_string(), invocation, + mcp_app_resource_uri: None, }), ) .await; @@ -581,6 +582,7 @@ async fn emit_tool_call_end( EventMsg::McpToolCallEnd(McpToolCallEndEvent { call_id: call_id.to_string(), invocation, + mcp_app_resource_uri: None, duration, result, }), diff --git a/codex-rs/core/tests/common/apps_test_server.rs b/codex-rs/core/tests/common/apps_test_server.rs index 5b039aa149..757fc146f2 100644 --- a/codex-rs/core/tests/common/apps_test_server.rs +++ b/codex-rs/core/tests/common/apps_test_server.rs @@ -21,6 +21,8 @@ const SERVER_VERSION: &str = "1.0.0"; const SEARCHABLE_TOOL_COUNT: usize = 100; pub const CALENDAR_CREATE_EVENT_RESOURCE_URI: &str = "connector://calendar/tools/calendar_create_event"; +pub const CALENDAR_CREATE_EVENT_MCP_APP_RESOURCE_URI: &str = + "ui://widget/calendar-create-event.html"; const CALENDAR_LIST_EVENTS_RESOURCE_URI: &str = "connector://calendar/tools/calendar_list_events"; pub const DOCUMENT_EXTRACT_TEXT_RESOURCE_URI: &str = "connector://calendar/tools/calendar_extract_text"; @@ -206,6 +208,7 @@ impl Respond for CodexAppsJsonRpcResponder { "connector_id": CONNECTOR_ID, "connector_name": self.connector_name.clone(), "connector_description": self.connector_description.clone(), + "openai/outputTemplate": CALENDAR_CREATE_EVENT_MCP_APP_RESOURCE_URI, "_codex_apps": { "resource_uri": CALENDAR_CREATE_EVENT_RESOURCE_URI, "contains_mcp_source": true, diff --git a/codex-rs/core/tests/suite/rmcp_client.rs b/codex-rs/core/tests/suite/rmcp_client.rs index 6d19e6ac52..b63369ea6f 100644 --- a/codex-rs/core/tests/suite/rmcp_client.rs +++ b/codex-rs/core/tests/suite/rmcp_client.rs @@ -736,6 +736,7 @@ async fn stdio_image_responses_round_trip() -> anyhow::Result<()> { tool: "image".to_string(), arguments: Some(json!({})), }, + mcp_app_resource_uri: None, }, ); diff --git a/codex-rs/core/tests/suite/search_tool.rs b/codex-rs/core/tests/suite/search_tool.rs index 624c439150..67cf8503b3 100644 --- a/codex-rs/core/tests/suite/search_tool.rs +++ b/codex-rs/core/tests/suite/search_tool.rs @@ -15,6 +15,7 @@ use codex_protocol::protocol::Op; use codex_protocol::protocol::SandboxPolicy; use codex_protocol::user_input::UserInput; use core_test_support::apps_test_server::AppsTestServer; +use core_test_support::apps_test_server::CALENDAR_CREATE_EVENT_MCP_APP_RESOURCE_URI; use core_test_support::apps_test_server::CALENDAR_CREATE_EVENT_RESOURCE_URI; use core_test_support::responses::ResponsesRequest; use core_test_support::responses::ev_assistant_message; @@ -457,6 +458,19 @@ async fn tool_search_returns_deferred_tools_without_follow_up_tool_injection() - }) .await?; + let EventMsg::McpToolCallBegin(begin) = wait_for_event(&test.codex, |event| { + matches!(event, EventMsg::McpToolCallBegin(_)) + }) + .await + else { + unreachable!("event guard guarantees McpToolCallBegin"); + }; + assert_eq!(begin.call_id, "calendar-call-1"); + assert_eq!( + begin.mcp_app_resource_uri.as_deref(), + Some(CALENDAR_CREATE_EVENT_MCP_APP_RESOURCE_URI) + ); + let EventMsg::McpToolCallEnd(end) = wait_for_event(&test.codex, |event| { matches!(event, EventMsg::McpToolCallEnd(_)) }) @@ -465,6 +479,10 @@ async fn tool_search_returns_deferred_tools_without_follow_up_tool_injection() - unreachable!("event guard guarantees McpToolCallEnd"); }; assert_eq!(end.call_id, "calendar-call-1"); + assert_eq!( + end.mcp_app_resource_uri.as_deref(), + Some(CALENDAR_CREATE_EVENT_MCP_APP_RESOURCE_URI) + ); assert_eq!( end.invocation, McpInvocation { diff --git a/codex-rs/exec/tests/event_processor_with_json_output.rs b/codex-rs/exec/tests/event_processor_with_json_output.rs index 71b5767348..02e52988cb 100644 --- a/codex-rs/exec/tests/event_processor_with_json_output.rs +++ b/codex-rs/exec/tests/event_processor_with_json_output.rs @@ -464,6 +464,7 @@ fn mcp_tool_call_begin_and_end_emit_item_events() { tool: "tool_x".to_string(), status: ApiMcpToolCallStatus::InProgress, arguments: json!({ "key": "value" }), + mcp_app_resource_uri: None, result: None, error: None, duration_ms: None, @@ -479,11 +480,12 @@ fn mcp_tool_call_begin_and_end_emit_item_events() { tool: "tool_x".to_string(), status: ApiMcpToolCallStatus::Completed, arguments: json!({ "key": "value" }), - result: Some(McpToolCallResult { + mcp_app_resource_uri: None, + result: Some(Box::new(McpToolCallResult { content: Vec::new(), structured_content: None, meta: None, - }), + })), error: None, duration_ms: Some(1_000), }, @@ -547,6 +549,7 @@ fn mcp_tool_call_failure_sets_failed_status() { tool: "tool_y".to_string(), status: ApiMcpToolCallStatus::Failed, arguments: json!({ "param": 42 }), + mcp_app_resource_uri: None, result: None, error: Some(McpToolCallError { message: "tool exploded".to_string(), @@ -593,6 +596,7 @@ fn mcp_tool_call_defaults_arguments_and_preserves_structured_content() { tool: "tool_z".to_string(), status: ApiMcpToolCallStatus::InProgress, arguments: serde_json::Value::Null, + mcp_app_resource_uri: None, result: None, error: None, duration_ms: None, @@ -608,14 +612,15 @@ fn mcp_tool_call_defaults_arguments_and_preserves_structured_content() { tool: "tool_z".to_string(), status: ApiMcpToolCallStatus::Completed, arguments: serde_json::Value::Null, - result: Some(McpToolCallResult { + mcp_app_resource_uri: None, + result: Some(Box::new(McpToolCallResult { content: vec![json!({ "type": "text", "text": "done", })], structured_content: Some(json!({ "status": "ok" })), meta: None, - }), + })), error: None, duration_ms: Some(10), }, diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index 244f748c82..ca113536e0 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -2362,6 +2362,9 @@ pub struct McpToolCallBeginEvent { /// Identifier so this can be paired with the McpToolCallEnd event. pub call_id: String, pub invocation: McpInvocation, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub mcp_app_resource_uri: Option, } #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, TS, PartialEq)] @@ -2369,6 +2372,9 @@ pub struct McpToolCallEndEvent { /// Identifier for the corresponding McpToolCallBegin that finished. pub call_id: String, pub invocation: McpInvocation, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub mcp_app_resource_uri: Option, #[ts(type = "string")] pub duration: Duration, /// Result of the tool call. Note this could be an error. diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 2f5258bf2f..9471c29ca6 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -4759,6 +4759,7 @@ impl ChatWidget { invocation, duration, result, + .. } = ev; let extra_cell = match self @@ -5943,6 +5944,7 @@ impl ChatWidget { server, tool, arguments, + mcp_app_resource_uri, result, error, duration_ms, @@ -5955,15 +5957,19 @@ impl ChatWidget { tool, arguments: Some(arguments), }, + mcp_app_resource_uri, duration: Duration::from_millis(duration_ms.unwrap_or_default().max(0) as u64), result: match (result, error) { (_, Some(error)) => Err(error.message), - (Some(result), None) => Ok(codex_protocol::mcp::CallToolResult { - content: result.content, - structured_content: result.structured_content, - is_error: Some(false), - meta: None, - }), + (Some(result), None) => { + let result = *result; + Ok(codex_protocol::mcp::CallToolResult { + content: result.content, + structured_content: result.structured_content, + is_error: Some(false), + meta: None, + }) + } (None, None) => Err("MCP tool call completed without a result".to_string()), }, }); @@ -6453,6 +6459,7 @@ impl ChatWidget { server, tool, arguments, + mcp_app_resource_uri, .. } => { self.on_mcp_tool_call_begin(McpToolCallBeginEvent { @@ -6462,6 +6469,7 @@ impl ChatWidget { tool, arguments: Some(arguments), }, + mcp_app_resource_uri, }); } ThreadItem::WebSearch { id, .. } => { From 48cf3ed7b03267c42e1a3d39e2b345a01493ad36 Mon Sep 17 00:00:00 2001 From: xl-openai Date: Wed, 15 Apr 2026 23:13:17 -0700 Subject: [PATCH 03/59] Extract plugin loading and marketplace logic into codex-core-plugins (#18070) Split plugin loading, marketplace, and related infrastructure out of core into codex-core-plugins, while keeping the core-facing configuration and orchestration flow in codex-core. --------- Co-authored-by: Codex --- codex-rs/Cargo.lock | 29 + codex-rs/Cargo.toml | 2 + codex-rs/app-server/Cargo.toml | 1 + .../app-server/src/codex_message_processor.rs | 13 +- codex-rs/app-server/src/config_api.rs | 4 +- codex-rs/core-plugins/BUILD.bazel | 15 + codex-rs/core-plugins/Cargo.toml | 39 + codex-rs/core-plugins/src/lib.rs | 6 + codex-rs/core-plugins/src/loader.rs | 807 ++++++++++++++++ .../plugins => core-plugins/src}/manifest.rs | 14 +- .../src}/marketplace.rs | 11 +- .../src}/marketplace_tests.rs | 0 .../plugins => core-plugins/src}/remote.rs | 32 +- .../src/plugins => core-plugins/src}/store.rs | 10 +- .../src}/store_tests.rs | 0 .../plugins => core-plugins/src}/toggles.rs | 0 codex-rs/core/Cargo.toml | 1 + codex-rs/core/src/lib.rs | 2 - .../src/plugins/installed_marketplaces.rs | 2 +- codex-rs/core/src/plugins/manager.rs | 881 ++---------------- codex-rs/core/src/plugins/manager_tests.rs | 22 +- .../src/plugins/marketplace_add/metadata.rs | 2 +- .../src/plugins/marketplace_add/source.rs | 2 +- codex-rs/core/src/plugins/mod.rs | 22 - 24 files changed, 1025 insertions(+), 892 deletions(-) create mode 100644 codex-rs/core-plugins/BUILD.bazel create mode 100644 codex-rs/core-plugins/Cargo.toml create mode 100644 codex-rs/core-plugins/src/lib.rs create mode 100644 codex-rs/core-plugins/src/loader.rs rename codex-rs/{core/src/plugins => core-plugins/src}/manifest.rs (98%) rename codex-rs/{core/src/plugins => core-plugins/src}/marketplace.rs (98%) rename codex-rs/{core/src/plugins => core-plugins/src}/marketplace_tests.rs (100%) rename codex-rs/{core/src/plugins => core-plugins/src}/remote.rs (94%) rename codex-rs/{core/src/plugins => core-plugins/src}/store.rs (97%) rename codex-rs/{core/src/plugins => core-plugins/src}/store_tests.rs (100%) rename codex-rs/{core/src/plugins => core-plugins/src}/toggles.rs (100%) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 2d39c42df4..1be5b45b6a 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1446,6 +1446,7 @@ dependencies = [ "codex-cloud-requirements", "codex-config", "codex-core", + "codex-core-plugins", "codex-exec-server", "codex-features", "codex-feedback", @@ -1913,6 +1914,7 @@ dependencies = [ "codex-code-mode", "codex-config", "codex-connectors", + "codex-core-plugins", "codex-core-skills", "codex-exec-server", "codex-execpolicy", @@ -2013,6 +2015,33 @@ dependencies = [ "zstd 0.13.3", ] +[[package]] +name = "codex-core-plugins" +version = "0.0.0" +dependencies = [ + "codex-app-server-protocol", + "codex-config", + "codex-core-skills", + "codex-exec-server", + "codex-git-utils", + "codex-login", + "codex-plugin", + "codex-protocol", + "codex-utils-absolute-path", + "codex-utils-plugins", + "dirs", + "pretty_assertions", + "reqwest", + "serde", + "serde_json", + "tempfile", + "thiserror 2.0.18", + "tokio", + "toml 0.9.11+spec-1.1.0", + "tracing", + "url", +] + [[package]] name = "codex-core-skills" version = "0.0.0" diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index 47c0d348bf..e7867304e9 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -28,6 +28,7 @@ members = [ "shell-escalation", "skills", "core", + "core-plugins", "core-skills", "hooks", "instructions", @@ -128,6 +129,7 @@ codex-code-mode = { path = "code-mode" } codex-config = { path = "config" } codex-connectors = { path = "connectors" } codex-core = { path = "core" } +codex-core-plugins = { path = "core-plugins" } codex-core-skills = { path = "core-skills" } codex-exec = { path = "exec" } codex-exec-server = { path = "exec-server" } diff --git a/codex-rs/app-server/Cargo.toml b/codex-rs/app-server/Cargo.toml index 408f39fafe..097abdb713 100644 --- a/codex-rs/app-server/Cargo.toml +++ b/codex-rs/app-server/Cargo.toml @@ -34,6 +34,7 @@ codex-arg0 = { workspace = true } codex-cloud-requirements = { workspace = true } codex-config = { workspace = true } codex-core = { workspace = true } +codex-core-plugins = { workspace = true } codex-exec-server = { workspace = true } codex-features = { workspace = true } codex-git-utils = { workspace = true } diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index f805ec3ff6..94c1e56645 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -232,22 +232,23 @@ use codex_core::find_thread_names_by_ids; use codex_core::find_thread_path_by_id_str; use codex_core::path_utils; use codex_core::plugins::MarketplaceAddError; -use codex_core::plugins::MarketplaceError; -use codex_core::plugins::MarketplacePluginSource; use codex_core::plugins::OPENAI_CURATED_MARKETPLACE_NAME; use codex_core::plugins::PluginInstallError as CorePluginInstallError; use codex_core::plugins::PluginInstallRequest; use codex_core::plugins::PluginReadRequest; use codex_core::plugins::PluginUninstallError as CorePluginUninstallError; use codex_core::plugins::add_marketplace as add_marketplace_to_codex_home; -use codex_core::plugins::load_plugin_apps; -use codex_core::plugins::load_plugin_mcp_servers; use codex_core::read_head_for_summary; use codex_core::read_session_meta_line; use codex_core::sandboxing::SandboxPermissions; use codex_core::windows_sandbox::WindowsSandboxLevelExt; use codex_core::windows_sandbox::WindowsSandboxSetupMode as CoreWindowsSandboxSetupMode; use codex_core::windows_sandbox::WindowsSandboxSetupRequest; +use codex_core_plugins::loader::load_plugin_apps; +use codex_core_plugins::loader::load_plugin_mcp_servers; +use codex_core_plugins::manifest::PluginManifestInterface; +use codex_core_plugins::marketplace::MarketplaceError; +use codex_core_plugins::marketplace::MarketplacePluginSource; use codex_exec_server::LOCAL_FS; use codex_features::FEATURES; use codex_features::Feature; @@ -8834,9 +8835,7 @@ fn plugin_skills_to_info( .collect() } -fn plugin_interface_to_info( - interface: codex_core::plugins::PluginManifestInterface, -) -> PluginInterface { +fn plugin_interface_to_info(interface: PluginManifestInterface) -> PluginInterface { PluginInterface { display_name: interface.display_name, short_description: interface.short_description, diff --git a/codex-rs/app-server/src/config_api.rs b/codex-rs/app-server/src/config_api.rs index 4d0b450d73..2bc7447300 100644 --- a/codex-rs/app-server/src/config_api.rs +++ b/codex-rs/app-server/src/config_api.rs @@ -27,8 +27,8 @@ use codex_core::config_loader::LoaderOverrides; use codex_core::config_loader::ResidencyRequirement as CoreResidencyRequirement; use codex_core::config_loader::SandboxModeRequirement as CoreSandboxModeRequirement; use codex_core::plugins::PluginId; -use codex_core::plugins::collect_plugin_enabled_candidates; -use codex_core::plugins::installed_plugin_telemetry_metadata; +use codex_core_plugins::loader::installed_plugin_telemetry_metadata; +use codex_core_plugins::toggles::collect_plugin_enabled_candidates; use codex_features::canonical_feature_for_key; use codex_features::feature_for_key; use codex_protocol::config_types::WebSearchMode; diff --git a/codex-rs/core-plugins/BUILD.bazel b/codex-rs/core-plugins/BUILD.bazel new file mode 100644 index 0000000000..aa19b9f368 --- /dev/null +++ b/codex-rs/core-plugins/BUILD.bazel @@ -0,0 +1,15 @@ +load("//:defs.bzl", "codex_rust_crate") + +codex_rust_crate( + name = "core-plugins", + crate_name = "codex_core_plugins", + compile_data = glob( + include = ["**"], + exclude = [ + "**/* *", + "BUILD.bazel", + "Cargo.toml", + ], + allow_empty = True, + ), +) diff --git a/codex-rs/core-plugins/Cargo.toml b/codex-rs/core-plugins/Cargo.toml new file mode 100644 index 0000000000..02e121f0e6 --- /dev/null +++ b/codex-rs/core-plugins/Cargo.toml @@ -0,0 +1,39 @@ +[package] +edition.workspace = true +license.workspace = true +name = "codex-core-plugins" +version.workspace = true + +[lib] +doctest = false +name = "codex_core_plugins" +path = "src/lib.rs" + +[lints] +workspace = true + +[dependencies] +codex-app-server-protocol = { workspace = true } +codex-config = { workspace = true } +codex-core-skills = { workspace = true } +codex-exec-server = { workspace = true } +codex-git-utils = { workspace = true } +codex-login = { workspace = true } +codex-plugin = { workspace = true } +codex-protocol = { workspace = true } +codex-utils-absolute-path = { workspace = true } +codex-utils-plugins = { workspace = true } +dirs = { workspace = true } +reqwest = { workspace = true } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +tempfile = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["fs"] } +toml = { workspace = true } +tracing = { workspace = true } +url = { workspace = true } + +[dev-dependencies] +pretty_assertions = { workspace = true } +tempfile = { workspace = true } diff --git a/codex-rs/core-plugins/src/lib.rs b/codex-rs/core-plugins/src/lib.rs new file mode 100644 index 0000000000..e075158cb1 --- /dev/null +++ b/codex-rs/core-plugins/src/lib.rs @@ -0,0 +1,6 @@ +pub mod loader; +pub mod manifest; +pub mod marketplace; +pub mod remote; +pub mod store; +pub mod toggles; diff --git a/codex-rs/core-plugins/src/loader.rs b/codex-rs/core-plugins/src/loader.rs new file mode 100644 index 0000000000..206add5b53 --- /dev/null +++ b/codex-rs/core-plugins/src/loader.rs @@ -0,0 +1,807 @@ +use crate::manifest::PluginManifestPaths; +use crate::manifest::load_plugin_manifest; +use crate::marketplace::MarketplacePluginSource; +use crate::marketplace::list_marketplaces; +use crate::marketplace::load_marketplace; +use crate::store::PluginStore; +use crate::store::plugin_version_for_source; +use codex_config::ConfigLayerStack; +use codex_config::types::McpServerConfig; +use codex_config::types::PluginConfig; +use codex_core_skills::SkillMetadata; +use codex_core_skills::config_rules::SkillConfigRules; +use codex_core_skills::config_rules::resolve_disabled_skill_paths; +use codex_core_skills::config_rules::skill_config_rules_from_stack; +use codex_core_skills::loader::SkillRoot; +use codex_core_skills::loader::load_skills_from_roots; +use codex_exec_server::LOCAL_FS; +use codex_plugin::AppConnectorId; +use codex_plugin::LoadedPlugin; +use codex_plugin::PluginCapabilitySummary; +use codex_plugin::PluginId; +use codex_plugin::PluginIdError; +use codex_plugin::PluginLoadOutcome; +use codex_plugin::PluginTelemetryMetadata; +use codex_protocol::protocol::Product; +use codex_protocol::protocol::SkillScope; +use codex_utils_absolute_path::AbsolutePathBuf; +use serde::Deserialize; +use serde_json::Map as JsonMap; +use serde_json::Value as JsonValue; +use std::collections::HashMap; +use std::collections::HashSet; +use std::fs; +use std::path::Path; +use std::sync::Arc; +use tracing::warn; + +const DEFAULT_SKILLS_DIR_NAME: &str = "skills"; +const DEFAULT_MCP_CONFIG_FILE: &str = ".mcp.json"; +const DEFAULT_APP_CONFIG_FILE: &str = ".app.json"; +const OPENAI_CURATED_MARKETPLACE_NAME: &str = "openai-curated"; +const CONFIG_TOML_FILE: &str = "config.toml"; + +pub fn log_plugin_load_errors(outcome: &PluginLoadOutcome) { + for plugin in outcome + .plugins() + .iter() + .filter(|plugin| plugin.error.is_some()) + { + if let Some(error) = plugin.error.as_deref() { + warn!( + plugin = plugin.config_name, + path = %plugin.root.display(), + "failed to load plugin: {error}" + ); + } + } +} + +#[derive(Debug, Default, Deserialize)] +#[serde(rename_all = "camelCase")] +struct PluginMcpFile { + #[serde(default)] + mcp_servers: HashMap, +} + +#[derive(Debug, Default, Deserialize)] +#[serde(rename_all = "camelCase")] +struct PluginAppFile { + #[serde(default)] + apps: HashMap, +} + +#[derive(Debug, Default, Deserialize)] +struct PluginAppConfig { + id: String, +} + +pub async fn load_plugins_from_layer_stack( + config_layer_stack: &ConfigLayerStack, + store: &PluginStore, + restriction_product: Option, +) -> PluginLoadOutcome { + let skill_config_rules = skill_config_rules_from_stack(config_layer_stack); + let mut configured_plugins: Vec<_> = configured_plugins_from_stack(config_layer_stack) + .into_iter() + .collect(); + configured_plugins.sort_unstable_by(|(a, _), (b, _)| a.cmp(b)); + + let mut plugins = Vec::with_capacity(configured_plugins.len()); + let mut seen_mcp_server_names = HashMap::::new(); + for (configured_name, plugin) in configured_plugins { + let loaded_plugin = load_plugin( + configured_name.clone(), + &plugin, + store, + restriction_product, + &skill_config_rules, + ) + .await; + for name in loaded_plugin.mcp_servers.keys() { + if let Some(previous_plugin) = + seen_mcp_server_names.insert(name.clone(), configured_name.clone()) + { + warn!( + plugin = configured_name, + previous_plugin, + server = name, + "skipping duplicate plugin MCP server name" + ); + } + } + plugins.push(loaded_plugin); + } + + PluginLoadOutcome::from_plugins(plugins) +} + +pub fn refresh_curated_plugin_cache( + codex_home: &Path, + plugin_version: &str, + configured_curated_plugin_ids: &[PluginId], +) -> Result { + let store = PluginStore::new(codex_home.to_path_buf()); + let curated_marketplace_path = AbsolutePathBuf::try_from( + codex_home + .join(".tmp/plugins") + .join(".agents/plugins/marketplace.json"), + ) + .map_err(|_| "local curated marketplace is not available".to_string())?; + let curated_marketplace = load_marketplace(&curated_marketplace_path) + .map_err(|err| format!("failed to load curated marketplace for cache refresh: {err}"))?; + + let mut plugin_sources = HashMap::::new(); + for plugin in curated_marketplace.plugins { + let plugin_name = plugin.name; + if plugin_sources.contains_key(&plugin_name) { + warn!( + plugin = plugin_name, + marketplace = OPENAI_CURATED_MARKETPLACE_NAME, + "ignoring duplicate curated plugin entry during cache refresh" + ); + continue; + } + let source_path = match plugin.source { + MarketplacePluginSource::Local { path } => path, + }; + plugin_sources.insert(plugin_name, source_path); + } + + let mut cache_refreshed = false; + for plugin_id in configured_curated_plugin_ids { + if store.active_plugin_version(plugin_id).as_deref() == Some(plugin_version) { + continue; + } + + let Some(source_path) = plugin_sources.get(&plugin_id.plugin_name).cloned() else { + warn!( + plugin = plugin_id.plugin_name, + marketplace = OPENAI_CURATED_MARKETPLACE_NAME, + "configured curated plugin no longer exists in curated marketplace during cache refresh" + ); + continue; + }; + + store + .install_with_version(source_path, plugin_id.clone(), plugin_version.to_string()) + .map_err(|err| { + format!( + "failed to refresh curated plugin cache for {}: {err}", + plugin_id.as_key() + ) + })?; + cache_refreshed = true; + } + + Ok(cache_refreshed) +} + +pub fn refresh_non_curated_plugin_cache( + codex_home: &Path, + additional_roots: &[AbsolutePathBuf], +) -> Result { + let configured_non_curated_plugin_ids = + non_curated_plugin_ids_from_config_keys(configured_plugins_from_codex_home( + codex_home, + "failed to read user config while refreshing non-curated plugin cache", + "failed to parse user config while refreshing non-curated plugin cache", + )); + if configured_non_curated_plugin_ids.is_empty() { + return Ok(false); + } + let configured_non_curated_plugin_keys = configured_non_curated_plugin_ids + .iter() + .map(PluginId::as_key) + .collect::>(); + + let store = PluginStore::new(codex_home.to_path_buf()); + let marketplace_outcome = list_marketplaces(additional_roots) + .map_err(|err| format!("failed to discover marketplaces for cache refresh: {err}"))?; + let mut plugin_sources = HashMap::::new(); + + for marketplace in marketplace_outcome.marketplaces { + if marketplace.name == OPENAI_CURATED_MARKETPLACE_NAME { + continue; + } + + for plugin in marketplace.plugins { + let plugin_id = + PluginId::new(plugin.name.clone(), marketplace.name.clone()).map_err(|err| { + match err { + PluginIdError::Invalid(message) => { + format!("failed to prepare non-curated plugin cache refresh: {message}") + } + } + })?; + let plugin_key = plugin_id.as_key(); + if !configured_non_curated_plugin_keys.contains(&plugin_key) { + continue; + } + if plugin_sources.contains_key(&plugin_key) { + warn!( + plugin = plugin.name, + marketplace = marketplace.name, + "ignoring duplicate non-curated plugin entry during cache refresh" + ); + continue; + } + + let source_path = match plugin.source { + MarketplacePluginSource::Local { path } => path, + }; + let plugin_version = plugin_version_for_source(source_path.as_path()) + .map_err(|err| format!("failed to read plugin version for {plugin_key}: {err}"))?; + plugin_sources.insert(plugin_key, (source_path, plugin_version)); + } + } + + let mut cache_refreshed = false; + for plugin_id in configured_non_curated_plugin_ids { + let plugin_key = plugin_id.as_key(); + let Some((source_path, plugin_version)) = plugin_sources.get(&plugin_key).cloned() else { + warn!( + plugin = plugin_id.plugin_name, + marketplace = plugin_id.marketplace_name, + "configured non-curated plugin no longer exists in discovered marketplaces during cache refresh" + ); + continue; + }; + + if store.active_plugin_version(&plugin_id).as_deref() == Some(plugin_version.as_str()) { + continue; + } + + store + .install_with_version(source_path, plugin_id.clone(), plugin_version) + .map_err(|err| format!("failed to refresh plugin cache for {plugin_key}: {err}"))?; + cache_refreshed = true; + } + + Ok(cache_refreshed) +} + +fn configured_plugins_from_stack( + config_layer_stack: &ConfigLayerStack, +) -> HashMap { + let Some(user_layer) = config_layer_stack.get_user_layer() else { + return HashMap::new(); + }; + configured_plugins_from_user_config_value(&user_layer.config) +} + +fn configured_plugins_from_user_config_value( + user_config: &toml::Value, +) -> HashMap { + let Some(plugins_value) = user_config.get("plugins") else { + return HashMap::new(); + }; + match plugins_value.clone().try_into() { + Ok(plugins) => plugins, + Err(err) => { + warn!("invalid plugins config: {err}"); + HashMap::new() + } + } +} + +fn configured_plugins_from_codex_home( + codex_home: &Path, + read_error_message: &str, + parse_error_message: &str, +) -> HashMap { + let config_path = codex_home.join(CONFIG_TOML_FILE); + let user_config = match fs::read_to_string(&config_path) { + Ok(user_config) => user_config, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return HashMap::new(), + Err(err) => { + warn!( + path = %config_path.display(), + error = %err, + "{read_error_message}" + ); + return HashMap::new(); + } + }; + + let user_config = match toml::from_str::(&user_config) { + Ok(user_config) => user_config, + Err(err) => { + warn!( + path = %config_path.display(), + error = %err, + "{parse_error_message}" + ); + return HashMap::new(); + } + }; + + configured_plugins_from_user_config_value(&user_config) +} + +fn configured_plugin_ids( + configured_plugins: HashMap, + invalid_plugin_key_message: &str, +) -> Vec { + configured_plugins + .into_keys() + .filter_map(|plugin_key| match PluginId::parse(&plugin_key) { + Ok(plugin_id) => Some(plugin_id), + Err(err) => { + warn!( + plugin_key, + error = %err, + "{invalid_plugin_key_message}" + ); + None + } + }) + .collect() +} + +fn curated_plugin_ids_from_config_keys( + configured_plugins: HashMap, +) -> Vec { + let mut configured_curated_plugin_ids = configured_plugin_ids( + configured_plugins, + "ignoring invalid configured plugin key during curated sync setup", + ) + .into_iter() + .filter(|plugin_id| plugin_id.marketplace_name == OPENAI_CURATED_MARKETPLACE_NAME) + .collect::>(); + configured_curated_plugin_ids.sort_unstable_by_key(PluginId::as_key); + configured_curated_plugin_ids +} + +fn non_curated_plugin_ids_from_config_keys( + configured_plugins: HashMap, +) -> Vec { + let mut configured_non_curated_plugin_ids = configured_plugin_ids( + configured_plugins, + "ignoring invalid plugin key during non-curated cache refresh setup", + ) + .into_iter() + .filter(|plugin_id| plugin_id.marketplace_name != OPENAI_CURATED_MARKETPLACE_NAME) + .collect::>(); + configured_non_curated_plugin_ids.sort_unstable_by_key(PluginId::as_key); + configured_non_curated_plugin_ids +} + +pub fn configured_curated_plugin_ids_from_codex_home(codex_home: &Path) -> Vec { + curated_plugin_ids_from_config_keys(configured_plugins_from_codex_home( + codex_home, + "failed to read user config while refreshing curated plugin cache", + "failed to parse user config while refreshing curated plugin cache", + )) +} + +async fn load_plugin( + config_name: String, + plugin: &PluginConfig, + store: &PluginStore, + restriction_product: Option, + skill_config_rules: &SkillConfigRules, +) -> LoadedPlugin { + let plugin_id = PluginId::parse(&config_name); + let active_plugin_root = plugin_id + .as_ref() + .ok() + .and_then(|plugin_id| store.active_plugin_root(plugin_id)); + let root = active_plugin_root + .clone() + .unwrap_or_else(|| match &plugin_id { + Ok(plugin_id) => store.plugin_base_root(plugin_id), + Err(_) => store.root().clone(), + }); + let mut loaded_plugin = LoadedPlugin { + config_name, + manifest_name: None, + manifest_description: None, + root, + enabled: plugin.enabled, + skill_roots: Vec::new(), + disabled_skill_paths: HashSet::new(), + has_enabled_skills: false, + mcp_servers: HashMap::new(), + apps: Vec::new(), + error: None, + }; + + if !plugin.enabled { + return loaded_plugin; + } + + let plugin_root = match plugin_id { + Ok(_) => match active_plugin_root { + Some(plugin_root) => plugin_root, + None => { + loaded_plugin.error = Some("plugin is not installed".to_string()); + return loaded_plugin; + } + }, + Err(err) => { + loaded_plugin.error = Some(err.to_string()); + return loaded_plugin; + } + }; + + if !plugin_root.as_path().is_dir() { + loaded_plugin.error = Some("path does not exist or is not a directory".to_string()); + return loaded_plugin; + } + + let Some(manifest) = load_plugin_manifest(plugin_root.as_path()) else { + loaded_plugin.error = Some("missing or invalid .codex-plugin/plugin.json".to_string()); + return loaded_plugin; + }; + + let manifest_paths = &manifest.paths; + loaded_plugin.manifest_name = manifest + .interface + .as_ref() + .and_then(|interface| interface.display_name.as_deref()) + .map(str::trim) + .filter(|display_name| !display_name.is_empty()) + .map(str::to_string) + .or_else(|| Some(manifest.name.clone())); + loaded_plugin.manifest_description = manifest.description.clone(); + loaded_plugin.skill_roots = plugin_skill_roots(&plugin_root, manifest_paths); + let resolved_skills = load_plugin_skills( + &plugin_root, + manifest_paths, + restriction_product, + skill_config_rules, + ) + .await; + let has_enabled_skills = resolved_skills.has_enabled_skills(); + loaded_plugin.disabled_skill_paths = resolved_skills.disabled_skill_paths; + loaded_plugin.has_enabled_skills = has_enabled_skills; + let mut mcp_servers = HashMap::new(); + for mcp_config_path in plugin_mcp_config_paths(plugin_root.as_path(), manifest_paths) { + let plugin_mcp = load_mcp_servers_from_file(plugin_root.as_path(), &mcp_config_path).await; + for (name, config) in plugin_mcp.mcp_servers { + if mcp_servers.insert(name.clone(), config).is_some() { + warn!( + plugin = %plugin_root.display(), + path = %mcp_config_path.display(), + server = name, + "plugin MCP file overwrote an earlier server definition" + ); + } + } + } + loaded_plugin.mcp_servers = mcp_servers; + loaded_plugin.apps = load_plugin_apps(plugin_root.as_path()).await; + loaded_plugin +} + +#[derive(Debug, Clone)] +pub struct ResolvedPluginSkills { + pub skills: Vec, + pub disabled_skill_paths: HashSet, + pub had_errors: bool, +} + +impl ResolvedPluginSkills { + pub fn has_enabled_skills(&self) -> bool { + self.had_errors + || self + .skills + .iter() + .any(|skill| !self.disabled_skill_paths.contains(&skill.path_to_skills_md)) + } +} + +pub async fn load_plugin_skills( + plugin_root: &AbsolutePathBuf, + manifest_paths: &PluginManifestPaths, + restriction_product: Option, + skill_config_rules: &SkillConfigRules, +) -> ResolvedPluginSkills { + let roots = plugin_skill_roots(plugin_root, manifest_paths) + .into_iter() + .map(|path| SkillRoot { + path, + scope: SkillScope::User, + file_system: Arc::clone(&LOCAL_FS), + }) + .collect::>(); + let outcome = load_skills_from_roots(roots).await; + let had_errors = !outcome.errors.is_empty(); + let skills = outcome + .skills + .into_iter() + .filter(|skill| skill.matches_product_restriction_for_product(restriction_product)) + .collect::>(); + let disabled_skill_paths = resolve_disabled_skill_paths(&skills, skill_config_rules); + + ResolvedPluginSkills { + skills, + disabled_skill_paths, + had_errors, + } +} + +fn plugin_skill_roots( + plugin_root: &AbsolutePathBuf, + manifest_paths: &PluginManifestPaths, +) -> Vec { + let mut paths = default_skill_roots(plugin_root); + if let Some(path) = &manifest_paths.skills { + paths.push(path.clone()); + } + paths.sort_unstable(); + paths.dedup(); + paths +} + +fn default_skill_roots(plugin_root: &AbsolutePathBuf) -> Vec { + let skills_dir = plugin_root.join(DEFAULT_SKILLS_DIR_NAME); + if skills_dir.is_dir() { + vec![skills_dir] + } else { + Vec::new() + } +} + +fn plugin_mcp_config_paths( + plugin_root: &Path, + manifest_paths: &PluginManifestPaths, +) -> Vec { + if let Some(path) = &manifest_paths.mcp_servers { + return vec![path.clone()]; + } + default_mcp_config_paths(plugin_root) +} + +fn default_mcp_config_paths(plugin_root: &Path) -> Vec { + let mut paths = Vec::new(); + let default_path = plugin_root.join(DEFAULT_MCP_CONFIG_FILE); + if default_path.is_file() + && let Ok(default_path) = AbsolutePathBuf::try_from(default_path) + { + paths.push(default_path); + } + paths.sort_unstable_by(|left, right| left.as_path().cmp(right.as_path())); + paths.dedup_by(|left, right| left.as_path() == right.as_path()); + paths +} + +pub async fn load_plugin_apps(plugin_root: &Path) -> Vec { + if let Some(manifest) = load_plugin_manifest(plugin_root) { + return load_apps_from_paths( + plugin_root, + plugin_app_config_paths(plugin_root, &manifest.paths), + ) + .await; + } + load_apps_from_paths(plugin_root, default_app_config_paths(plugin_root)).await +} + +fn plugin_app_config_paths( + plugin_root: &Path, + manifest_paths: &PluginManifestPaths, +) -> Vec { + if let Some(path) = &manifest_paths.apps { + return vec![path.clone()]; + } + default_app_config_paths(plugin_root) +} + +fn default_app_config_paths(plugin_root: &Path) -> Vec { + let mut paths = Vec::new(); + let default_path = plugin_root.join(DEFAULT_APP_CONFIG_FILE); + if default_path.is_file() + && let Ok(default_path) = AbsolutePathBuf::try_from(default_path) + { + paths.push(default_path); + } + paths.sort_unstable_by(|left, right| left.as_path().cmp(right.as_path())); + paths.dedup_by(|left, right| left.as_path() == right.as_path()); + paths +} + +async fn load_apps_from_paths( + plugin_root: &Path, + app_config_paths: Vec, +) -> Vec { + let mut connector_ids = Vec::new(); + for app_config_path in app_config_paths { + let Ok(contents) = tokio::fs::read_to_string(app_config_path.as_path()).await else { + continue; + }; + let parsed = match serde_json::from_str::(&contents) { + Ok(parsed) => parsed, + Err(err) => { + warn!( + path = %app_config_path.display(), + "failed to parse plugin app config: {err}" + ); + continue; + } + }; + + let mut apps: Vec = parsed.apps.into_values().collect(); + apps.sort_unstable_by(|left, right| left.id.cmp(&right.id)); + + connector_ids.extend(apps.into_iter().filter_map(|app| { + if app.id.trim().is_empty() { + warn!( + plugin = %plugin_root.display(), + "plugin app config is missing an app id" + ); + None + } else { + Some(AppConnectorId(app.id)) + } + })); + } + connector_ids.dedup(); + connector_ids +} + +pub async fn plugin_telemetry_metadata_from_root( + plugin_id: &PluginId, + plugin_root: &AbsolutePathBuf, +) -> PluginTelemetryMetadata { + let Some(manifest) = load_plugin_manifest(plugin_root.as_path()) else { + return PluginTelemetryMetadata::from_plugin_id(plugin_id); + }; + + let manifest_paths = &manifest.paths; + let has_skills = !plugin_skill_roots(plugin_root, manifest_paths).is_empty(); + let mut mcp_server_names = Vec::new(); + for path in plugin_mcp_config_paths(plugin_root.as_path(), manifest_paths) { + mcp_server_names.extend( + load_mcp_servers_from_file(plugin_root.as_path(), &path) + .await + .mcp_servers + .into_keys(), + ); + } + mcp_server_names.sort_unstable(); + mcp_server_names.dedup(); + + PluginTelemetryMetadata { + plugin_id: plugin_id.clone(), + capability_summary: Some(PluginCapabilitySummary { + config_name: plugin_id.as_key(), + display_name: plugin_id.plugin_name.clone(), + description: None, + has_skills, + mcp_server_names, + app_connector_ids: load_apps_from_paths( + plugin_root.as_path(), + plugin_app_config_paths(plugin_root.as_path(), manifest_paths), + ) + .await, + }), + } +} + +pub async fn load_plugin_mcp_servers(plugin_root: &Path) -> HashMap { + let Some(manifest) = load_plugin_manifest(plugin_root) else { + return HashMap::new(); + }; + + let mut mcp_servers = HashMap::new(); + for mcp_config_path in plugin_mcp_config_paths(plugin_root, &manifest.paths) { + let plugin_mcp = load_mcp_servers_from_file(plugin_root, &mcp_config_path).await; + for (name, config) in plugin_mcp.mcp_servers { + mcp_servers.entry(name).or_insert(config); + } + } + + mcp_servers +} + +pub async fn installed_plugin_telemetry_metadata( + codex_home: &Path, + plugin_id: &PluginId, +) -> PluginTelemetryMetadata { + let store = PluginStore::new(codex_home.to_path_buf()); + let Some(plugin_root) = store.active_plugin_root(plugin_id) else { + return PluginTelemetryMetadata::from_plugin_id(plugin_id); + }; + + plugin_telemetry_metadata_from_root(plugin_id, &plugin_root).await +} + +async fn load_mcp_servers_from_file( + plugin_root: &Path, + mcp_config_path: &AbsolutePathBuf, +) -> PluginMcpDiscovery { + let Ok(contents) = tokio::fs::read_to_string(mcp_config_path.as_path()).await else { + return PluginMcpDiscovery::default(); + }; + let parsed = match serde_json::from_str::(&contents) { + Ok(parsed) => parsed, + Err(err) => { + warn!( + path = %mcp_config_path.display(), + "failed to parse plugin MCP config: {err}" + ); + return PluginMcpDiscovery::default(); + } + }; + normalize_plugin_mcp_servers( + plugin_root, + parsed.mcp_servers, + mcp_config_path.to_string_lossy().as_ref(), + ) +} + +fn normalize_plugin_mcp_servers( + plugin_root: &Path, + plugin_mcp_servers: HashMap, + source: &str, +) -> PluginMcpDiscovery { + let mut mcp_servers = HashMap::new(); + + for (name, config_value) in plugin_mcp_servers { + let normalized = normalize_plugin_mcp_server_value(plugin_root, config_value); + match serde_json::from_value::(JsonValue::Object(normalized)) { + Ok(config) => { + mcp_servers.insert(name, config); + } + Err(err) => { + warn!( + plugin = %plugin_root.display(), + server = name, + "failed to parse plugin MCP server from {source}: {err}" + ); + } + } + } + + PluginMcpDiscovery { mcp_servers } +} + +fn normalize_plugin_mcp_server_value( + plugin_root: &Path, + value: JsonValue, +) -> JsonMap { + let mut object = match value { + JsonValue::Object(object) => object, + _ => return JsonMap::new(), + }; + + if let Some(JsonValue::String(transport_type)) = object.remove("type") { + match transport_type.as_str() { + "http" | "streamable_http" | "streamable-http" => {} + "stdio" => {} + other => { + warn!( + plugin = %plugin_root.display(), + transport = other, + "plugin MCP server uses an unknown transport type" + ); + } + } + } + + if let Some(JsonValue::Object(oauth)) = object.remove("oauth") + && oauth.contains_key("callbackPort") + { + warn!( + plugin = %plugin_root.display(), + "plugin MCP server OAuth callbackPort is ignored; Codex uses global MCP OAuth callback settings" + ); + } + + if let Some(JsonValue::String(cwd)) = object.get("cwd") + && !Path::new(cwd).is_absolute() + { + object.insert( + "cwd".to_string(), + JsonValue::String(plugin_root.join(cwd).display().to_string()), + ); + } + + object +} + +#[derive(Debug, Default)] +struct PluginMcpDiscovery { + mcp_servers: HashMap, +} diff --git a/codex-rs/core/src/plugins/manifest.rs b/codex-rs/core-plugins/src/manifest.rs similarity index 98% rename from codex-rs/core/src/plugins/manifest.rs rename to codex-rs/core-plugins/src/manifest.rs index f1253441bd..f58b882ca1 100644 --- a/codex-rs/core/src/plugins/manifest.rs +++ b/codex-rs/core-plugins/src/manifest.rs @@ -30,12 +30,12 @@ struct RawPluginManifest { } #[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct PluginManifest { - pub(crate) name: String, - pub(crate) version: Option, - pub(crate) description: Option, - pub(crate) paths: PluginManifestPaths, - pub(crate) interface: Option, +pub struct PluginManifest { + pub name: String, + pub version: Option, + pub description: Option, + pub paths: PluginManifestPaths, + pub interface: Option, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -114,7 +114,7 @@ enum RawPluginManifestDefaultPromptEntry { Invalid(JsonValue), } -pub(crate) fn load_plugin_manifest(plugin_root: &Path) -> Option { +pub fn load_plugin_manifest(plugin_root: &Path) -> Option { let manifest_path = plugin_root.join(PLUGIN_MANIFEST_PATH); if !manifest_path.is_file() { return None; diff --git a/codex-rs/core/src/plugins/marketplace.rs b/codex-rs/core-plugins/src/marketplace.rs similarity index 98% rename from codex-rs/core/src/plugins/marketplace.rs rename to codex-rs/core-plugins/src/marketplace.rs index 96c73e8e19..79e5237ad0 100644 --- a/codex-rs/core/src/plugins/marketplace.rs +++ b/codex-rs/core-plugins/src/marketplace.rs @@ -1,5 +1,5 @@ -use super::PluginManifestInterface; -use super::load_plugin_manifest; +use crate::manifest::PluginManifestInterface; +use crate::manifest::load_plugin_manifest; use codex_app_server_protocol::PluginAuthPolicy; use codex_app_server_protocol::PluginInstallPolicy; use codex_git_utils::get_git_repo_root; @@ -231,7 +231,7 @@ pub fn validate_marketplace_root(root: &Path) -> Result Option { +pub fn find_marketplace_manifest_path(root: &Path) -> Option { MARKETPLACE_MANIFEST_RELATIVE_PATHS .iter() .find_map(|relative_path| { @@ -265,7 +265,7 @@ fn marketplace_root_from_layout(marketplace_path: &Path, relative_path: &str) -> Some(current.to_path_buf()) } -pub(crate) fn load_marketplace(path: &AbsolutePathBuf) -> Result { +pub fn load_marketplace(path: &AbsolutePathBuf) -> Result { let marketplace = load_raw_marketplace_manifest(path)?; let mut plugins = Vec::new(); @@ -311,7 +311,8 @@ pub(crate) fn load_marketplace(path: &AbsolutePathBuf) -> Result, ) -> Result { diff --git a/codex-rs/core/src/plugins/marketplace_tests.rs b/codex-rs/core-plugins/src/marketplace_tests.rs similarity index 100% rename from codex-rs/core/src/plugins/marketplace_tests.rs rename to codex-rs/core-plugins/src/marketplace_tests.rs diff --git a/codex-rs/core/src/plugins/remote.rs b/codex-rs/core-plugins/src/remote.rs similarity index 94% rename from codex-rs/core/src/plugins/remote.rs rename to codex-rs/core-plugins/src/remote.rs index f26bbe8880..34ece0880a 100644 --- a/codex-rs/core/src/plugins/remote.rs +++ b/codex-rs/core-plugins/src/remote.rs @@ -1,4 +1,3 @@ -use crate::config::Config; use codex_login::CodexAuth; use codex_login::default_client::build_reqwest_client; use codex_protocol::protocol::Product; @@ -11,12 +10,17 @@ const REMOTE_PLUGIN_FETCH_TIMEOUT: Duration = Duration::from_secs(30); const REMOTE_FEATURED_PLUGIN_FETCH_TIMEOUT: Duration = Duration::from_secs(10); const REMOTE_PLUGIN_MUTATION_TIMEOUT: Duration = Duration::from_secs(30); +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RemotePluginServiceConfig { + pub chatgpt_base_url: String, +} + #[derive(Debug, Clone, PartialEq, Eq, Deserialize)] -pub(crate) struct RemotePluginStatusSummary { - pub(crate) name: String, +pub struct RemotePluginStatusSummary { + pub name: String, #[serde(default = "default_remote_marketplace_name")] - pub(crate) marketplace_name: String, - pub(crate) enabled: bool, + pub marketplace_name: String, + pub enabled: bool, } #[derive(Debug, Clone, PartialEq, Eq, Deserialize)] @@ -116,8 +120,8 @@ pub enum RemotePluginFetchError { }, } -pub(crate) async fn fetch_remote_plugin_status( - config: &Config, +pub async fn fetch_remote_plugin_status( + config: &RemotePluginServiceConfig, auth: Option<&CodexAuth>, ) -> Result, RemotePluginFetchError> { let Some(auth) = auth else { @@ -161,7 +165,7 @@ pub(crate) async fn fetch_remote_plugin_status( } pub async fn fetch_remote_featured_plugin_ids( - config: &Config, + config: &RemotePluginServiceConfig, auth: Option<&CodexAuth>, product: Option, ) -> Result, RemotePluginFetchError> { @@ -205,8 +209,8 @@ pub async fn fetch_remote_featured_plugin_ids( }) } -pub(crate) async fn enable_remote_plugin( - config: &Config, +pub async fn enable_remote_plugin( + config: &RemotePluginServiceConfig, auth: Option<&CodexAuth>, plugin_id: &str, ) -> Result<(), RemotePluginMutationError> { @@ -214,8 +218,8 @@ pub(crate) async fn enable_remote_plugin( Ok(()) } -pub(crate) async fn uninstall_remote_plugin( - config: &Config, +pub async fn uninstall_remote_plugin( + config: &RemotePluginServiceConfig, auth: Option<&CodexAuth>, plugin_id: &str, ) -> Result<(), RemotePluginMutationError> { @@ -238,7 +242,7 @@ fn default_remote_marketplace_name() -> String { } async fn post_remote_plugin_mutation( - config: &Config, + config: &RemotePluginServiceConfig, auth: Option<&CodexAuth>, plugin_id: &str, action: &str, @@ -294,7 +298,7 @@ async fn post_remote_plugin_mutation( } fn remote_plugin_mutation_url( - config: &Config, + config: &RemotePluginServiceConfig, plugin_id: &str, action: &str, ) -> Result { diff --git a/codex-rs/core/src/plugins/store.rs b/codex-rs/core-plugins/src/store.rs similarity index 97% rename from codex-rs/core/src/plugins/store.rs rename to codex-rs/core-plugins/src/store.rs index 4316f18002..2f530b8648 100644 --- a/codex-rs/core/src/plugins/store.rs +++ b/codex-rs/core-plugins/src/store.rs @@ -1,5 +1,5 @@ -use super::load_plugin_manifest; -use super::manifest::PluginManifest; +use crate::manifest::PluginManifest; +use crate::manifest::load_plugin_manifest; use codex_plugin::PluginId; use codex_plugin::validate_plugin_segment; use codex_utils_absolute_path::AbsolutePathBuf; @@ -11,8 +11,8 @@ use std::io; use std::path::Path; use std::path::PathBuf; -pub(crate) const DEFAULT_PLUGIN_VERSION: &str = "local"; -pub(crate) const PLUGINS_CACHE_DIR: &str = "plugins/cache"; +pub const DEFAULT_PLUGIN_VERSION: &str = "local"; +pub const PLUGINS_CACHE_DIR: &str = "plugins/cache"; #[derive(Debug, Clone, PartialEq, Eq)] pub struct PluginInstallResult { @@ -157,7 +157,7 @@ impl PluginStoreError { } } -pub(crate) fn plugin_version_for_source(source_path: &Path) -> Result { +pub fn plugin_version_for_source(source_path: &Path) -> Result { let plugin_version = plugin_manifest_version_for_source(source_path)? .unwrap_or_else(|| DEFAULT_PLUGIN_VERSION.to_string()); validate_plugin_version_segment(&plugin_version).map_err(PluginStoreError::Invalid)?; diff --git a/codex-rs/core/src/plugins/store_tests.rs b/codex-rs/core-plugins/src/store_tests.rs similarity index 100% rename from codex-rs/core/src/plugins/store_tests.rs rename to codex-rs/core-plugins/src/store_tests.rs diff --git a/codex-rs/core/src/plugins/toggles.rs b/codex-rs/core-plugins/src/toggles.rs similarity index 100% rename from codex-rs/core/src/plugins/toggles.rs rename to codex-rs/core-plugins/src/toggles.rs diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index 78dfd3c070..4ce4192b81 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -33,6 +33,7 @@ codex-async-utils = { workspace = true } codex-code-mode = { workspace = true } codex-connectors = { workspace = true } codex-config = { workspace = true } +codex-core-plugins = { workspace = true } codex-core-skills = { workspace = true } codex-exec-server = { workspace = true } codex-features = { workspace = true } diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index abfe91a0e1..9950f70142 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -98,9 +98,7 @@ pub(crate) use skills::build_skill_injections; pub(crate) use skills::build_skill_name_counts; pub(crate) use skills::collect_env_var_dependencies; pub(crate) use skills::collect_explicit_skill_mentions; -pub(crate) use skills::config_rules; pub(crate) use skills::injection; -pub(crate) use skills::loader; pub(crate) use skills::manager; pub(crate) use skills::maybe_emit_implicit_skill_invocation; pub(crate) use skills::render_skills_section; diff --git a/codex-rs/core/src/plugins/installed_marketplaces.rs b/codex-rs/core/src/plugins/installed_marketplaces.rs index ad1918d51d..a6f80139a9 100644 --- a/codex-rs/core/src/plugins/installed_marketplaces.rs +++ b/codex-rs/core/src/plugins/installed_marketplaces.rs @@ -1,10 +1,10 @@ use crate::config::Config; +use codex_core_plugins::marketplace::find_marketplace_manifest_path; use codex_utils_absolute_path::AbsolutePathBuf; use std::path::Path; use std::path::PathBuf; use tracing::warn; -use super::marketplace::find_marketplace_manifest_path; use super::validate_plugin_segment; pub const INSTALLED_MARKETPLACES_DIR: &str = ".tmp/marketplaces"; diff --git a/codex-rs/core/src/plugins/manager.rs b/codex-rs/core/src/plugins/manager.rs index a735180ab6..50c62d5823 100644 --- a/codex-rs/core/src/plugins/manager.rs +++ b/codex-rs/core/src/plugins/manager.rs @@ -1,52 +1,48 @@ -use super::LoadedPlugin; use super::PluginLoadOutcome; -use super::PluginManifestPaths; use super::curated_plugins_repo_path; use super::installed_marketplaces::installed_marketplace_roots_from_config; -use super::load_plugin_manifest; -use super::manifest::PluginManifestInterface; -use super::marketplace::MarketplaceError; -use super::marketplace::MarketplaceInterface; -use super::marketplace::MarketplaceListError; -use super::marketplace::MarketplacePluginAuthPolicy; -use super::marketplace::MarketplacePluginPolicy; -use super::marketplace::MarketplacePluginSource; -use super::marketplace::ResolvedMarketplacePlugin; -use super::marketplace::list_marketplaces; -use super::marketplace::load_marketplace; -use super::marketplace::resolve_marketplace_plugin; use super::read_curated_plugins_sha; -use super::remote::RemotePluginFetchError; -use super::remote::RemotePluginMutationError; -use super::remote::enable_remote_plugin; -use super::remote::fetch_remote_featured_plugin_ids; -use super::remote::fetch_remote_plugin_status; -use super::remote::uninstall_remote_plugin; use super::startup_sync::start_startup_remote_plugin_sync_once; -use super::store::PluginInstallResult as StorePluginInstallResult; -use super::store::PluginStore; -use super::store::PluginStoreError; -use super::store::plugin_version_for_source; use super::sync_openai_plugins_repo; use crate::SkillMetadata; -use crate::config::CONFIG_TOML_FILE; use crate::config::Config; use crate::config::ConfigService; use crate::config::ConfigServiceError; use crate::config::edit::ConfigEdit; use crate::config::edit::ConfigEditsBuilder; use crate::config_loader::ConfigLayerStack; -use crate::config_rules::SkillConfigRules; -use crate::config_rules::resolve_disabled_skill_paths; -use crate::config_rules::skill_config_rules_from_stack; -use crate::loader::SkillRoot; -use crate::loader::load_skills_from_roots; use codex_analytics::AnalyticsEventsClient; use codex_app_server_protocol::ConfigValueWriteParams; use codex_app_server_protocol::MergeStrategy; -use codex_config::types::McpServerConfig; use codex_config::types::PluginConfig; -use codex_exec_server::LOCAL_FS; +use codex_core_plugins::loader::configured_curated_plugin_ids_from_codex_home; +use codex_core_plugins::loader::installed_plugin_telemetry_metadata; +use codex_core_plugins::loader::load_plugin_apps; +use codex_core_plugins::loader::load_plugin_mcp_servers; +use codex_core_plugins::loader::load_plugin_skills; +use codex_core_plugins::loader::load_plugins_from_layer_stack; +use codex_core_plugins::loader::log_plugin_load_errors; +use codex_core_plugins::loader::plugin_telemetry_metadata_from_root; +use codex_core_plugins::loader::refresh_curated_plugin_cache; +use codex_core_plugins::loader::refresh_non_curated_plugin_cache; +use codex_core_plugins::manifest::PluginManifestInterface; +use codex_core_plugins::manifest::load_plugin_manifest; +use codex_core_plugins::marketplace::MarketplaceError; +use codex_core_plugins::marketplace::MarketplaceInterface; +use codex_core_plugins::marketplace::MarketplaceListError; +use codex_core_plugins::marketplace::MarketplacePluginAuthPolicy; +use codex_core_plugins::marketplace::MarketplacePluginPolicy; +use codex_core_plugins::marketplace::MarketplacePluginSource; +use codex_core_plugins::marketplace::ResolvedMarketplacePlugin; +use codex_core_plugins::marketplace::list_marketplaces; +use codex_core_plugins::marketplace::load_marketplace; +use codex_core_plugins::marketplace::resolve_marketplace_plugin; +use codex_core_plugins::remote::RemotePluginFetchError; +use codex_core_plugins::remote::RemotePluginMutationError; +use codex_core_plugins::remote::RemotePluginServiceConfig; +use codex_core_plugins::store::PluginInstallResult as StorePluginInstallResult; +use codex_core_plugins::store::PluginStore; +use codex_core_plugins::store::PluginStoreError; use codex_features::Feature; use codex_login::AuthManager; use codex_login::CodexAuth; @@ -54,19 +50,12 @@ use codex_plugin::AppConnectorId; use codex_plugin::PluginCapabilitySummary; use codex_plugin::PluginId; use codex_plugin::PluginIdError; -use codex_plugin::PluginTelemetryMetadata; use codex_plugin::prompt_safe_plugin_description; use codex_protocol::protocol::Product; -use codex_protocol::protocol::SkillScope; use codex_utils_absolute_path::AbsolutePathBuf; -use serde::Deserialize; -use serde_json::Map as JsonMap; -use serde_json::Value as JsonValue; use serde_json::json; use std::collections::HashMap; use std::collections::HashSet; -use std::fs; -use std::path::Path; use std::path::PathBuf; use std::sync::Arc; use std::sync::RwLock; @@ -78,9 +67,6 @@ use toml_edit::value; use tracing::info; use tracing::warn; -const DEFAULT_SKILLS_DIR_NAME: &str = "skills"; -const DEFAULT_MCP_CONFIG_FILE: &str = ".mcp.json"; -const DEFAULT_APP_CONFIG_FILE: &str = ".app.json"; pub const OPENAI_CURATED_MARKETPLACE_NAME: &str = "openai-curated"; pub const OPENAI_CURATED_MARKETPLACE_DISPLAY_NAME: &str = "OpenAI Curated"; static CURATED_REPO_SYNC_STARTED: AtomicBool = AtomicBool::new(false); @@ -109,6 +95,12 @@ struct NonCuratedCacheRefreshState { in_flight: bool, } +fn remote_plugin_service_config(config: &Config) -> RemotePluginServiceConfig { + RemotePluginServiceConfig { + chatgpt_base_url: config.chatgpt_base_url.clone(), + } +} + fn featured_plugin_ids_cache_key( config: &Config, auth: Option<&CodexAuth>, @@ -503,8 +495,12 @@ impl PluginsManager { if let Some(featured_plugin_ids) = self.cached_featured_plugin_ids(&cache_key) { return Ok(featured_plugin_ids); } - let featured_plugin_ids = - fetch_remote_featured_plugin_ids(config, auth, self.restriction_product).await?; + let featured_plugin_ids = codex_core_plugins::remote::fetch_remote_featured_plugin_ids( + &remote_plugin_service_config(config), + auth, + self.restriction_product, + ) + .await?; self.write_featured_plugin_ids_cache(cache_key, &featured_plugin_ids); Ok(featured_plugin_ids) } @@ -536,9 +532,13 @@ impl PluginsManager { // This only forwards the backend mutation before the local install flow. We rely on // `plugin/list(forceRemoteSync=true)` to sync local state rather than doing an extra // reconcile pass here. - enable_remote_plugin(config, auth, &plugin_id) - .await - .map_err(PluginInstallError::from)?; + codex_core_plugins::remote::enable_remote_plugin( + &remote_plugin_service_config(config), + auth, + &plugin_id, + ) + .await + .map_err(PluginInstallError::from)?; self.install_resolved_plugin(resolved).await } @@ -619,9 +619,13 @@ impl PluginsManager { // This only forwards the backend mutation before the local uninstall flow. We rely on // `plugin/list(forceRemoteSync=true)` to sync local state rather than doing an extra // reconcile pass here. - uninstall_remote_plugin(config, auth, &plugin_key) - .await - .map_err(PluginUninstallError::from)?; + codex_core_plugins::remote::uninstall_remote_plugin( + &remote_plugin_service_config(config), + auth, + &plugin_key, + ) + .await + .map_err(PluginUninstallError::from)?; self.uninstall_plugin_id(plugin_id).await } @@ -670,9 +674,12 @@ impl PluginsManager { } info!("starting remote plugin sync"); - let remote_plugins = fetch_remote_plugin_status(config, auth) - .await - .map_err(PluginRemoteSyncError::from)?; + let remote_plugins = codex_core_plugins::remote::fetch_remote_plugin_status( + &remote_plugin_service_config(config), + auth, + ) + .await + .map_err(PluginRemoteSyncError::from)?; let configured_plugins = configured_plugins_from_stack(&config.config_layer_stack); let curated_marketplace_root = curated_plugins_repo_path(self.codex_home.as_path()); let curated_marketplace_path = AbsolutePathBuf::try_from( @@ -1026,7 +1033,9 @@ impl PluginsManager { })?; let description = manifest.description.clone(); let manifest_paths = &manifest.paths; - let skill_config_rules = skill_config_rules_from_stack(&config.config_layer_stack); + let skill_config_rules = codex_core_skills::config_rules::skill_config_rules_from_stack( + &config.config_layer_stack, + ); let resolved_skills = load_plugin_skills( &source_path, manifest_paths, @@ -1034,21 +1043,11 @@ impl PluginsManager { &skill_config_rules, ) .await; - let apps = load_apps_from_paths( - source_path.as_path(), - plugin_app_config_paths(source_path.as_path(), manifest_paths), - ) - .await; - let mcp_config_paths = plugin_mcp_config_paths(source_path.as_path(), manifest_paths); - let mut mcp_server_names = Vec::new(); - for mcp_config_path in mcp_config_paths { - mcp_server_names.extend( - load_mcp_servers_from_file(source_path.as_path(), &mcp_config_path) - .await - .mcp_servers - .into_keys(), - ); - } + let apps = load_plugin_apps(source_path.as_path()).await; + let mut mcp_server_names = load_plugin_mcp_servers(source_path.as_path()) + .await + .into_keys() + .collect::>(); mcp_server_names.sort_unstable(); mcp_server_names.dedup(); @@ -1160,13 +1159,8 @@ impl PluginsManager { .spawn( move || match sync_openai_plugins_repo(codex_home.as_path()) { Ok(curated_plugin_version) => { - let configured_curated_plugin_ids = curated_plugin_ids_from_config_keys( - configured_plugins_from_codex_home( - codex_home.as_path(), - "failed to read user config while refreshing curated plugin cache", - "failed to parse user config while refreshing curated plugin cache", - ), - ); + let configured_curated_plugin_ids = + configured_curated_plugin_ids_from_codex_home(codex_home.as_path()); match refresh_curated_plugin_cache( codex_home.as_path(), &curated_plugin_version, @@ -1352,224 +1346,6 @@ impl PluginUninstallError { } } -fn log_plugin_load_errors(outcome: &PluginLoadOutcome) { - for plugin in outcome - .plugins() - .iter() - .filter(|plugin| plugin.error.is_some()) - { - if let Some(error) = plugin.error.as_deref() { - warn!( - plugin = plugin.config_name, - path = %plugin.root.display(), - "failed to load plugin: {error}" - ); - } - } -} - -#[derive(Debug, Default, Deserialize)] -#[serde(rename_all = "camelCase")] -struct PluginMcpFile { - #[serde(default)] - mcp_servers: HashMap, -} - -#[derive(Debug, Default, Deserialize)] -#[serde(rename_all = "camelCase")] -struct PluginAppFile { - #[serde(default)] - apps: HashMap, -} - -#[derive(Debug, Default, Deserialize)] -struct PluginAppConfig { - id: String, -} - -pub(crate) async fn load_plugins_from_layer_stack( - config_layer_stack: &ConfigLayerStack, - store: &PluginStore, - restriction_product: Option, -) -> PluginLoadOutcome { - let skill_config_rules = skill_config_rules_from_stack(config_layer_stack); - let mut configured_plugins: Vec<_> = configured_plugins_from_stack(config_layer_stack) - .into_iter() - .collect(); - configured_plugins.sort_unstable_by(|(a, _), (b, _)| a.cmp(b)); - - let mut plugins = Vec::with_capacity(configured_plugins.len()); - let mut seen_mcp_server_names = HashMap::::new(); - for (configured_name, plugin) in configured_plugins { - let loaded_plugin = load_plugin( - configured_name.clone(), - &plugin, - store, - restriction_product, - &skill_config_rules, - ) - .await; - for name in loaded_plugin.mcp_servers.keys() { - if let Some(previous_plugin) = - seen_mcp_server_names.insert(name.clone(), configured_name.clone()) - { - warn!( - plugin = configured_name, - previous_plugin, - server = name, - "skipping duplicate plugin MCP server name" - ); - } - } - plugins.push(loaded_plugin); - } - - PluginLoadOutcome::from_plugins(plugins) -} - -fn refresh_curated_plugin_cache( - codex_home: &Path, - plugin_version: &str, - configured_curated_plugin_ids: &[PluginId], -) -> Result { - let store = PluginStore::new(codex_home.to_path_buf()); - let curated_marketplace_path = AbsolutePathBuf::try_from( - curated_plugins_repo_path(codex_home).join(".agents/plugins/marketplace.json"), - ) - .map_err(|_| "local curated marketplace is not available".to_string())?; - let curated_marketplace = load_marketplace(&curated_marketplace_path) - .map_err(|err| format!("failed to load curated marketplace for cache refresh: {err}"))?; - - let mut plugin_sources = HashMap::::new(); - for plugin in curated_marketplace.plugins { - let plugin_name = plugin.name; - if plugin_sources.contains_key(&plugin_name) { - warn!( - plugin = plugin_name, - marketplace = OPENAI_CURATED_MARKETPLACE_NAME, - "ignoring duplicate curated plugin entry during cache refresh" - ); - continue; - } - let source_path = match plugin.source { - MarketplacePluginSource::Local { path } => path, - }; - plugin_sources.insert(plugin_name, source_path); - } - - let mut cache_refreshed = false; - for plugin_id in configured_curated_plugin_ids { - if store.active_plugin_version(plugin_id).as_deref() == Some(plugin_version) { - continue; - } - - let Some(source_path) = plugin_sources.get(&plugin_id.plugin_name).cloned() else { - warn!( - plugin = plugin_id.plugin_name, - marketplace = OPENAI_CURATED_MARKETPLACE_NAME, - "configured curated plugin no longer exists in curated marketplace during cache refresh" - ); - continue; - }; - - store - .install_with_version(source_path, plugin_id.clone(), plugin_version.to_string()) - .map_err(|err| { - format!( - "failed to refresh curated plugin cache for {}: {err}", - plugin_id.as_key() - ) - })?; - cache_refreshed = true; - } - - Ok(cache_refreshed) -} - -fn refresh_non_curated_plugin_cache( - codex_home: &Path, - additional_roots: &[AbsolutePathBuf], -) -> Result { - let configured_non_curated_plugin_ids = - non_curated_plugin_ids_from_config_keys(configured_plugins_from_codex_home( - codex_home, - "failed to read user config while refreshing non-curated plugin cache", - "failed to parse user config while refreshing non-curated plugin cache", - )); - if configured_non_curated_plugin_ids.is_empty() { - return Ok(false); - } - let configured_non_curated_plugin_keys = configured_non_curated_plugin_ids - .iter() - .map(PluginId::as_key) - .collect::>(); - - let store = PluginStore::new(codex_home.to_path_buf()); - let marketplace_outcome = list_marketplaces(additional_roots) - .map_err(|err| format!("failed to discover marketplaces for cache refresh: {err}"))?; - let mut plugin_sources = HashMap::::new(); - - for marketplace in marketplace_outcome.marketplaces { - if marketplace.name == OPENAI_CURATED_MARKETPLACE_NAME { - continue; - } - - for plugin in marketplace.plugins { - let plugin_id = - PluginId::new(plugin.name.clone(), marketplace.name.clone()).map_err(|err| { - match err { - PluginIdError::Invalid(message) => { - format!("failed to prepare non-curated plugin cache refresh: {message}") - } - } - })?; - let plugin_key = plugin_id.as_key(); - if !configured_non_curated_plugin_keys.contains(&plugin_key) { - continue; - } - if plugin_sources.contains_key(&plugin_key) { - warn!( - plugin = plugin.name, - marketplace = marketplace.name, - "ignoring duplicate non-curated plugin entry during cache refresh" - ); - continue; - } - - let source_path = match plugin.source { - MarketplacePluginSource::Local { path } => path, - }; - let plugin_version = plugin_version_for_source(source_path.as_path()) - .map_err(|err| format!("failed to read plugin version for {plugin_key}: {err}"))?; - plugin_sources.insert(plugin_key, (source_path, plugin_version)); - } - } - - let mut cache_refreshed = false; - for plugin_id in configured_non_curated_plugin_ids { - let plugin_key = plugin_id.as_key(); - let Some((source_path, plugin_version)) = plugin_sources.get(&plugin_key).cloned() else { - warn!( - plugin = plugin_id.plugin_name, - marketplace = plugin_id.marketplace_name, - "configured non-curated plugin no longer exists in discovered marketplaces during cache refresh" - ); - continue; - }; - - if store.active_plugin_version(&plugin_id).as_deref() == Some(plugin_version.as_str()) { - continue; - } - - store - .install_with_version(source_path, plugin_id.clone(), plugin_version) - .map_err(|err| format!("failed to refresh plugin cache for {plugin_key}: {err}"))?; - cache_refreshed = true; - } - - Ok(cache_refreshed) -} - fn configured_plugins_from_stack( config_layer_stack: &ConfigLayerStack, ) -> HashMap { @@ -1595,523 +1371,6 @@ fn configured_plugins_from_user_config_value( } } -fn configured_plugins_from_codex_home( - codex_home: &Path, - read_error_message: &str, - parse_error_message: &str, -) -> HashMap { - let config_path = codex_home.join(CONFIG_TOML_FILE); - let user_config = match fs::read_to_string(&config_path) { - Ok(user_config) => user_config, - Err(err) if err.kind() == std::io::ErrorKind::NotFound => return HashMap::new(), - Err(err) => { - warn!( - path = %config_path.display(), - error = %err, - "{read_error_message}" - ); - return HashMap::new(); - } - }; - - let user_config = match toml::from_str::(&user_config) { - Ok(user_config) => user_config, - Err(err) => { - warn!( - path = %config_path.display(), - error = %err, - "{parse_error_message}" - ); - return HashMap::new(); - } - }; - - configured_plugins_from_user_config_value(&user_config) -} - -fn configured_plugin_ids( - configured_plugins: HashMap, - invalid_plugin_key_message: &str, -) -> Vec { - configured_plugins - .into_keys() - .filter_map(|plugin_key| match PluginId::parse(&plugin_key) { - Ok(plugin_id) => Some(plugin_id), - Err(err) => { - warn!( - plugin_key, - error = %err, - "{invalid_plugin_key_message}" - ); - None - } - }) - .collect() -} - -fn curated_plugin_ids_from_config_keys( - configured_plugins: HashMap, -) -> Vec { - let mut configured_curated_plugin_ids = configured_plugin_ids( - configured_plugins, - "ignoring invalid configured plugin key during curated sync setup", - ) - .into_iter() - .filter(|plugin_id| plugin_id.marketplace_name == OPENAI_CURATED_MARKETPLACE_NAME) - .collect::>(); - configured_curated_plugin_ids.sort_unstable_by_key(PluginId::as_key); - configured_curated_plugin_ids -} - -fn non_curated_plugin_ids_from_config_keys( - configured_plugins: HashMap, -) -> Vec { - let mut configured_non_curated_plugin_ids = configured_plugin_ids( - configured_plugins, - "ignoring invalid plugin key during non-curated cache refresh setup", - ) - .into_iter() - .filter(|plugin_id| plugin_id.marketplace_name != OPENAI_CURATED_MARKETPLACE_NAME) - .collect::>(); - configured_non_curated_plugin_ids.sort_unstable_by_key(PluginId::as_key); - configured_non_curated_plugin_ids -} - -async fn load_plugin( - config_name: String, - plugin: &PluginConfig, - store: &PluginStore, - restriction_product: Option, - skill_config_rules: &SkillConfigRules, -) -> LoadedPlugin { - let plugin_id = PluginId::parse(&config_name); - let active_plugin_root = plugin_id - .as_ref() - .ok() - .and_then(|plugin_id| store.active_plugin_root(plugin_id)); - let root = active_plugin_root - .clone() - .unwrap_or_else(|| match &plugin_id { - Ok(plugin_id) => store.plugin_base_root(plugin_id), - Err(_) => store.root().clone(), - }); - let mut loaded_plugin = LoadedPlugin { - config_name, - manifest_name: None, - manifest_description: None, - root, - enabled: plugin.enabled, - skill_roots: Vec::new(), - disabled_skill_paths: HashSet::new(), - has_enabled_skills: false, - mcp_servers: HashMap::new(), - apps: Vec::new(), - error: None, - }; - - if !plugin.enabled { - return loaded_plugin; - } - - let plugin_root = match plugin_id { - Ok(_) => match active_plugin_root { - Some(plugin_root) => plugin_root, - None => { - loaded_plugin.error = Some("plugin is not installed".to_string()); - return loaded_plugin; - } - }, - Err(err) => { - loaded_plugin.error = Some(err.to_string()); - return loaded_plugin; - } - }; - - if !plugin_root.as_path().is_dir() { - loaded_plugin.error = Some("path does not exist or is not a directory".to_string()); - return loaded_plugin; - } - - let Some(manifest) = load_plugin_manifest(plugin_root.as_path()) else { - loaded_plugin.error = Some("missing or invalid .codex-plugin/plugin.json".to_string()); - return loaded_plugin; - }; - - let manifest_paths = &manifest.paths; - loaded_plugin.manifest_name = manifest - .interface - .as_ref() - .and_then(|interface| interface.display_name.as_deref()) - .map(str::trim) - .filter(|display_name| !display_name.is_empty()) - .map(str::to_string) - .or_else(|| Some(manifest.name.clone())); - loaded_plugin.manifest_description = manifest.description.clone(); - loaded_plugin.skill_roots = plugin_skill_roots(&plugin_root, manifest_paths); - let resolved_skills = load_plugin_skills( - &plugin_root, - manifest_paths, - restriction_product, - skill_config_rules, - ) - .await; - let has_enabled_skills = resolved_skills.has_enabled_skills(); - loaded_plugin.disabled_skill_paths = resolved_skills.disabled_skill_paths; - loaded_plugin.has_enabled_skills = has_enabled_skills; - let mut mcp_servers = HashMap::new(); - for mcp_config_path in plugin_mcp_config_paths(plugin_root.as_path(), manifest_paths) { - let plugin_mcp = load_mcp_servers_from_file(plugin_root.as_path(), &mcp_config_path).await; - for (name, config) in plugin_mcp.mcp_servers { - if mcp_servers.insert(name.clone(), config).is_some() { - warn!( - plugin = %plugin_root.display(), - path = %mcp_config_path.display(), - server = name, - "plugin MCP file overwrote an earlier server definition" - ); - } - } - } - loaded_plugin.mcp_servers = mcp_servers; - loaded_plugin.apps = load_apps_from_paths( - plugin_root.as_path(), - plugin_app_config_paths(plugin_root.as_path(), manifest_paths), - ) - .await; - loaded_plugin -} - -struct ResolvedPluginSkills { - skills: Vec, - disabled_skill_paths: HashSet, - had_errors: bool, -} - -impl ResolvedPluginSkills { - fn has_enabled_skills(&self) -> bool { - // Keep the plugin visible in capability summaries if skill loading was partial. - self.had_errors - || self - .skills - .iter() - .any(|skill| !self.disabled_skill_paths.contains(&skill.path_to_skills_md)) - } -} - -async fn load_plugin_skills( - plugin_root: &AbsolutePathBuf, - manifest_paths: &PluginManifestPaths, - restriction_product: Option, - skill_config_rules: &SkillConfigRules, -) -> ResolvedPluginSkills { - let roots = plugin_skill_roots(plugin_root, manifest_paths) - .into_iter() - .map(|path| SkillRoot { - path, - scope: SkillScope::User, - file_system: Arc::clone(&LOCAL_FS), - }) - .collect::>(); - let outcome = load_skills_from_roots(roots).await; - let had_errors = !outcome.errors.is_empty(); - let skills = outcome - .skills - .into_iter() - .filter(|skill| skill.matches_product_restriction_for_product(restriction_product)) - .collect::>(); - let disabled_skill_paths = resolve_disabled_skill_paths(&skills, skill_config_rules); - - ResolvedPluginSkills { - skills, - disabled_skill_paths, - had_errors, - } -} - -fn plugin_skill_roots( - plugin_root: &AbsolutePathBuf, - manifest_paths: &PluginManifestPaths, -) -> Vec { - let mut paths = default_skill_roots(plugin_root); - if let Some(path) = &manifest_paths.skills { - paths.push(path.clone()); - } - paths.sort_unstable(); - paths.dedup(); - paths -} - -fn default_skill_roots(plugin_root: &AbsolutePathBuf) -> Vec { - let skills_dir = plugin_root.join(DEFAULT_SKILLS_DIR_NAME); - if skills_dir.is_dir() { - vec![skills_dir] - } else { - Vec::new() - } -} - -fn plugin_mcp_config_paths( - plugin_root: &Path, - manifest_paths: &PluginManifestPaths, -) -> Vec { - if let Some(path) = &manifest_paths.mcp_servers { - return vec![path.clone()]; - } - default_mcp_config_paths(plugin_root) -} - -fn default_mcp_config_paths(plugin_root: &Path) -> Vec { - let mut paths = Vec::new(); - let default_path = plugin_root.join(DEFAULT_MCP_CONFIG_FILE); - if default_path.is_file() - && let Ok(default_path) = AbsolutePathBuf::try_from(default_path) - { - paths.push(default_path); - } - paths.sort_unstable(); - paths.dedup(); - paths -} - -pub async fn load_plugin_apps(plugin_root: &Path) -> Vec { - if let Some(manifest) = load_plugin_manifest(plugin_root) { - return load_apps_from_paths( - plugin_root, - plugin_app_config_paths(plugin_root, &manifest.paths), - ) - .await; - } - load_apps_from_paths(plugin_root, default_app_config_paths(plugin_root)).await -} - -fn plugin_app_config_paths( - plugin_root: &Path, - manifest_paths: &PluginManifestPaths, -) -> Vec { - if let Some(path) = &manifest_paths.apps { - return vec![path.clone()]; - } - default_app_config_paths(plugin_root) -} - -fn default_app_config_paths(plugin_root: &Path) -> Vec { - let mut paths = Vec::new(); - let default_path = plugin_root.join(DEFAULT_APP_CONFIG_FILE); - if default_path.is_file() - && let Ok(default_path) = AbsolutePathBuf::try_from(default_path) - { - paths.push(default_path); - } - paths.sort_unstable(); - paths.dedup(); - paths -} - -async fn load_apps_from_paths( - plugin_root: &Path, - app_config_paths: Vec, -) -> Vec { - let mut connector_ids = Vec::new(); - for app_config_path in app_config_paths { - let Ok(contents) = tokio::fs::read_to_string(app_config_path.as_path()).await else { - continue; - }; - let parsed = match serde_json::from_str::(&contents) { - Ok(parsed) => parsed, - Err(err) => { - warn!( - path = %app_config_path.display(), - "failed to parse plugin app config: {err}" - ); - continue; - } - }; - - let mut apps: Vec = parsed.apps.into_values().collect(); - apps.sort_unstable_by(|left, right| left.id.cmp(&right.id)); - - connector_ids.extend(apps.into_iter().filter_map(|app| { - if app.id.trim().is_empty() { - warn!( - plugin = %plugin_root.display(), - "plugin app config is missing an app id" - ); - None - } else { - Some(AppConnectorId(app.id)) - } - })); - } - connector_ids.dedup(); - connector_ids -} - -pub async fn plugin_telemetry_metadata_from_root( - plugin_id: &PluginId, - plugin_root: &AbsolutePathBuf, -) -> PluginTelemetryMetadata { - let Some(manifest) = load_plugin_manifest(plugin_root.as_path()) else { - return PluginTelemetryMetadata::from_plugin_id(plugin_id); - }; - - let manifest_paths = &manifest.paths; - let has_skills = !plugin_skill_roots(plugin_root, manifest_paths).is_empty(); - let mut mcp_server_names = Vec::new(); - for path in plugin_mcp_config_paths(plugin_root.as_path(), manifest_paths) { - mcp_server_names.extend( - load_mcp_servers_from_file(plugin_root.as_path(), &path) - .await - .mcp_servers - .into_keys(), - ); - } - mcp_server_names.sort_unstable(); - mcp_server_names.dedup(); - - PluginTelemetryMetadata { - plugin_id: plugin_id.clone(), - capability_summary: Some(PluginCapabilitySummary { - config_name: plugin_id.as_key(), - display_name: plugin_id.plugin_name.clone(), - description: None, - has_skills, - mcp_server_names, - app_connector_ids: load_apps_from_paths( - plugin_root.as_path(), - plugin_app_config_paths(plugin_root.as_path(), manifest_paths), - ) - .await, - }), - } -} - -pub async fn load_plugin_mcp_servers(plugin_root: &Path) -> HashMap { - let Some(manifest) = load_plugin_manifest(plugin_root) else { - return HashMap::new(); - }; - - let mut mcp_servers = HashMap::new(); - for mcp_config_path in plugin_mcp_config_paths(plugin_root, &manifest.paths) { - let plugin_mcp = load_mcp_servers_from_file(plugin_root, &mcp_config_path).await; - for (name, config) in plugin_mcp.mcp_servers { - mcp_servers.entry(name).or_insert(config); - } - } - - mcp_servers -} - -pub async fn installed_plugin_telemetry_metadata( - codex_home: &Path, - plugin_id: &PluginId, -) -> PluginTelemetryMetadata { - let store = PluginStore::new(codex_home.to_path_buf()); - let Some(plugin_root) = store.active_plugin_root(plugin_id) else { - return PluginTelemetryMetadata::from_plugin_id(plugin_id); - }; - - plugin_telemetry_metadata_from_root(plugin_id, &plugin_root).await -} - -async fn load_mcp_servers_from_file( - plugin_root: &Path, - mcp_config_path: &AbsolutePathBuf, -) -> PluginMcpDiscovery { - let Ok(contents) = tokio::fs::read_to_string(mcp_config_path.as_path()).await else { - return PluginMcpDiscovery::default(); - }; - let parsed = match serde_json::from_str::(&contents) { - Ok(parsed) => parsed, - Err(err) => { - warn!( - path = %mcp_config_path.display(), - "failed to parse plugin MCP config: {err}" - ); - return PluginMcpDiscovery::default(); - } - }; - normalize_plugin_mcp_servers( - plugin_root, - parsed.mcp_servers, - mcp_config_path.to_string_lossy().as_ref(), - ) -} - -fn normalize_plugin_mcp_servers( - plugin_root: &Path, - plugin_mcp_servers: HashMap, - source: &str, -) -> PluginMcpDiscovery { - let mut mcp_servers = HashMap::new(); - - for (name, config_value) in plugin_mcp_servers { - let normalized = normalize_plugin_mcp_server_value(plugin_root, config_value); - match serde_json::from_value::(JsonValue::Object(normalized)) { - Ok(config) => { - mcp_servers.insert(name, config); - } - Err(err) => { - warn!( - plugin = %plugin_root.display(), - server = name, - "failed to parse plugin MCP server from {source}: {err}" - ); - } - } - } - - PluginMcpDiscovery { mcp_servers } -} - -fn normalize_plugin_mcp_server_value( - plugin_root: &Path, - value: JsonValue, -) -> JsonMap { - let mut object = match value { - JsonValue::Object(object) => object, - _ => return JsonMap::new(), - }; - - if let Some(JsonValue::String(transport_type)) = object.remove("type") { - match transport_type.as_str() { - "http" | "streamable_http" | "streamable-http" => {} - "stdio" => {} - other => { - warn!( - plugin = %plugin_root.display(), - transport = other, - "plugin MCP server uses an unknown transport type" - ); - } - } - } - - if let Some(JsonValue::Object(oauth)) = object.remove("oauth") - && oauth.contains_key("callbackPort") - { - warn!( - plugin = %plugin_root.display(), - "plugin MCP server OAuth callbackPort is ignored; Codex uses global MCP OAuth callback settings" - ); - } - - if let Some(JsonValue::String(cwd)) = object.get("cwd") - && !Path::new(cwd).is_absolute() - { - object.insert( - "cwd".to_string(), - JsonValue::String(plugin_root.join(cwd).display().to_string()), - ); - } - - object -} - -#[derive(Debug, Default)] -struct PluginMcpDiscovery { - mcp_servers: HashMap, -} - #[cfg(test)] #[path = "manager_tests.rs"] mod tests; diff --git a/codex-rs/core/src/plugins/manager_tests.rs b/codex-rs/core/src/plugins/manager_tests.rs index 23b202f1a8..f74b8fd117 100644 --- a/codex-rs/core/src/plugins/manager_tests.rs +++ b/codex-rs/core/src/plugins/manager_tests.rs @@ -6,7 +6,6 @@ use crate::config_loader::ConfigLayerStack; use crate::config_loader::ConfigRequirements; use crate::config_loader::ConfigRequirementsToml; use crate::plugins::LoadedPlugin; -use crate::plugins::MarketplacePluginInstallPolicy; use crate::plugins::PluginLoadOutcome; use crate::plugins::marketplace_install_root; use crate::plugins::test_support::TEST_CURATED_PLUGIN_SHA; @@ -14,12 +13,15 @@ use crate::plugins::test_support::write_curated_plugin_sha_with as write_curated use crate::plugins::test_support::write_file; use crate::plugins::test_support::write_openai_curated_marketplace; use codex_app_server_protocol::ConfigLayerSource; +use codex_config::McpServerConfig; use codex_config::types::McpServerTransportConfig; +use codex_core_plugins::marketplace::MarketplacePluginInstallPolicy; use codex_login::CodexAuth; use codex_protocol::protocol::Product; use codex_utils_absolute_path::test_support::PathBufExt; use pretty_assertions::assert_eq; use std::fs; +use std::path::Path; use tempfile::TempDir; use toml::Value; use wiremock::Mock; @@ -2477,14 +2479,10 @@ enabled = true ); assert_eq!( - curated_plugin_ids_from_config_keys(configured_plugins_from_codex_home( - tmp.path(), - "failed to read user config while refreshing curated plugin cache", - "failed to parse user config while refreshing curated plugin cache", - )) - .into_iter() - .map(|plugin_id| plugin_id.as_key()) - .collect::>(), + configured_curated_plugin_ids_from_codex_home(tmp.path()) + .into_iter() + .map(|plugin_id| plugin_id.as_key()) + .collect::>(), vec!["slack@openai-curated".to_string()] ); @@ -2496,11 +2494,7 @@ plugins = true ); assert_eq!( - curated_plugin_ids_from_config_keys(configured_plugins_from_codex_home( - tmp.path(), - "failed to read user config while refreshing curated plugin cache", - "failed to parse user config while refreshing curated plugin cache", - )), + configured_curated_plugin_ids_from_codex_home(tmp.path()), Vec::::new() ); } diff --git a/codex-rs/core/src/plugins/marketplace_add/metadata.rs b/codex-rs/core/src/plugins/marketplace_add/metadata.rs index b9412e7f0b..f21ed0faa2 100644 --- a/codex-rs/core/src/plugins/marketplace_add/metadata.rs +++ b/codex-rs/core/src/plugins/marketplace_add/metadata.rs @@ -1,10 +1,10 @@ use super::MarketplaceAddError; use super::MarketplaceSource; use crate::plugins::installed_marketplaces::resolve_configured_marketplace_root; -use crate::plugins::validate_marketplace_root; use codex_config::CONFIG_TOML_FILE; use codex_config::MarketplaceConfigUpdate; use codex_config::record_user_marketplace; +use codex_core_plugins::marketplace::validate_marketplace_root; use std::fs; use std::io::ErrorKind; use std::path::Path; diff --git a/codex-rs/core/src/plugins/marketplace_add/source.rs b/codex-rs/core/src/plugins/marketplace_add/source.rs index 53d52058aa..fa2c7dd922 100644 --- a/codex-rs/core/src/plugins/marketplace_add/source.rs +++ b/codex-rs/core/src/plugins/marketplace_add/source.rs @@ -1,6 +1,6 @@ use super::MarketplaceAddError; -use crate::plugins::validate_marketplace_root; use crate::plugins::validate_plugin_segment; +use codex_core_plugins::marketplace::validate_marketplace_root; use std::path::Path; use std::path::PathBuf; diff --git a/codex-rs/core/src/plugins/mod.rs b/codex-rs/core/src/plugins/mod.rs index 7ffb80b449..57997eb134 100644 --- a/codex-rs/core/src/plugins/mod.rs +++ b/codex-rs/core/src/plugins/mod.rs @@ -4,17 +4,12 @@ mod discoverable; mod injection; mod installed_marketplaces; mod manager; -mod manifest; -mod marketplace; mod marketplace_add; mod mentions; -mod remote; mod render; mod startup_sync; -mod store; #[cfg(test)] pub(crate) mod test_support; -mod toggles; pub use codex_plugin::AppConnectorId; pub use codex_plugin::EffectiveSkillRoots; @@ -45,32 +40,15 @@ pub use manager::PluginRemoteSyncError; pub use manager::PluginUninstallError; pub use manager::PluginsManager; pub use manager::RemotePluginSyncResult; -pub use manager::installed_plugin_telemetry_metadata; -pub use manager::load_plugin_apps; -pub use manager::load_plugin_mcp_servers; -pub use manager::plugin_telemetry_metadata_from_root; -pub use manifest::PluginManifestInterface; -pub(crate) use manifest::PluginManifestPaths; -pub(crate) use manifest::load_plugin_manifest; -pub use marketplace::MarketplaceError; -pub use marketplace::MarketplaceListError; -pub use marketplace::MarketplacePluginAuthPolicy; -pub use marketplace::MarketplacePluginInstallPolicy; -pub use marketplace::MarketplacePluginPolicy; -pub use marketplace::MarketplacePluginSource; -pub use marketplace::validate_marketplace_root; pub use marketplace_add::MarketplaceAddError; pub use marketplace_add::MarketplaceAddOutcome; pub use marketplace_add::MarketplaceAddRequest; pub use marketplace_add::add_marketplace; -pub use remote::RemotePluginFetchError; -pub use remote::fetch_remote_featured_plugin_ids; pub(crate) use render::render_explicit_plugin_instructions; pub(crate) use render::render_plugins_section; pub(crate) use startup_sync::curated_plugins_repo_path; pub(crate) use startup_sync::read_curated_plugins_sha; pub(crate) use startup_sync::sync_openai_plugins_repo; -pub use toggles::collect_plugin_enabled_candidates; pub(crate) use mentions::build_connector_slug_counts; pub(crate) use mentions::build_skill_name_counts; From 4cd85b28d2713289d5940b6cb967894277a17419 Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Thu, 16 Apr 2026 00:03:50 -0700 Subject: [PATCH 04/59] Fix MCP startup cancellation through app server (#18078) Addresses https://github.com/openai/codex/issues/17143 Problem: TUI interrupts without an active turn stopped cancelling slow MCP startup after routing through the app-server APIs. Solution: Route no-active-turn interrupts through app-server as startup cancels, acknowledge them immediately, and emit cancelled MCP startup updates. Testing: I manually confirmed that MCP cancellation didn't work prior to this PR and works after the fix was in place. --- .../app-server/src/codex_message_processor.rs | 51 +++++++++++++++---- .../codex-mcp/src/mcp_connection_manager.rs | 5 +- codex-rs/tui/src/app.rs | 18 +++++-- codex-rs/tui/src/app_server_session.rs | 4 ++ 4 files changed, 61 insertions(+), 17 deletions(-) diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index 94c1e56645..2585383cda 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -186,6 +186,7 @@ use codex_app_server_protocol::ThreadUnsubscribeStatus; use codex_app_server_protocol::Turn; use codex_app_server_protocol::TurnError; use codex_app_server_protocol::TurnInterruptParams; +use codex_app_server_protocol::TurnInterruptResponse; use codex_app_server_protocol::TurnStartParams; use codex_app_server_protocol::TurnStartResponse; use codex_app_server_protocol::TurnStatus; @@ -7559,9 +7560,12 @@ impl CodexMessageProcessor { async fn turn_interrupt(&self, request_id: ConnectionRequestId, params: TurnInterruptParams) { let TurnInterruptParams { thread_id, turn_id } = params; - self.outgoing - .record_request_turn_id(&request_id, &turn_id) - .await; + let is_startup_interrupt = turn_id.is_empty(); + if !is_startup_interrupt { + self.outgoing + .record_request_turn_id(&request_id, &turn_id) + .await; + } let (thread_uuid, thread) = match self.load_thread(&thread_id).await { Ok(v) => v, @@ -7571,21 +7575,48 @@ impl CodexMessageProcessor { } }; - let request = request_id.clone(); - - // Record the pending interrupt so we can reply when TurnAborted arrives. - { + // Record turn interrupts so we can reply when TurnAborted arrives. Startup + // interrupts do not have a turn and are acknowledged after submission. + if !is_startup_interrupt { let thread_state = self.thread_state_manager.thread_state(thread_uuid).await; let mut thread_state = thread_state.lock().await; thread_state .pending_interrupts - .push((request, ApiVersion::V2)); + .push((request_id.clone(), ApiVersion::V2)); } - // Submit the interrupt; we'll respond upon TurnAborted. - let _ = self + // Submit the interrupt. Turn interrupts respond upon TurnAborted; startup + // interrupts respond here because startup cancellation has no turn event. + let submit_result = self .submit_core_op(&request_id, thread.as_ref(), Op::Interrupt) .await; + match submit_result { + Ok(_) if is_startup_interrupt => { + self.outgoing + .send_response(request_id, TurnInterruptResponse {}) + .await; + } + Ok(_) => {} + Err(err) => { + if !is_startup_interrupt { + let thread_state = self.thread_state_manager.thread_state(thread_uuid).await; + let mut thread_state = thread_state.lock().await; + thread_state + .pending_interrupts + .retain(|(pending_request_id, _)| pending_request_id != &request_id); + } + let interrupt_target = if is_startup_interrupt { + "startup" + } else { + "turn" + }; + self.send_internal_error( + request_id, + format!("failed to interrupt {interrupt_target}: {err}"), + ) + .await; + } + } } async fn ensure_conversation_listener( diff --git a/codex-rs/codex-mcp/src/mcp_connection_manager.rs b/codex-rs/codex-mcp/src/mcp_connection_manager.rs index 1542c7b30b..d1696b5931 100644 --- a/codex-rs/codex-mcp/src/mcp_connection_manager.rs +++ b/codex-rs/codex-mcp/src/mcp_connection_manager.rs @@ -758,12 +758,13 @@ impl McpConnectionManager { let submit_id = startup_submit_id.clone(); let auth_entry = auth_entries.get(&server_name).cloned(); join_set.spawn(async move { - let outcome = async_managed_client.client().await; + let mut outcome = async_managed_client.client().await; if cancel_token.is_cancelled() { - return (server_name, Err(StartupOutcomeError::Cancelled)); + outcome = Err(StartupOutcomeError::Cancelled); } let status = match &outcome { Ok(_) => McpStartupStatus::Ready, + Err(StartupOutcomeError::Cancelled) => McpStartupStatus::Cancelled, Err(error) => { let error_str = mcp_init_error_display( server_name.as_str(), diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index 03a99e3aa3..a60856b9c4 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -2378,10 +2378,11 @@ impl App { ) -> Result { match op.view() { AppCommandView::Interrupt => { - let Some(turn_id) = self.active_turn_id_for_thread(thread_id).await else { - return Ok(true); - }; - app_server.turn_interrupt(thread_id, turn_id).await?; + if let Some(turn_id) = self.active_turn_id_for_thread(thread_id).await { + app_server.turn_interrupt(thread_id, turn_id).await?; + } else { + app_server.startup_interrupt(thread_id).await?; + } Ok(true) } AppCommandView::UserTurn { @@ -11410,11 +11411,18 @@ guardian_approval = true #[tokio::test] async fn interrupt_without_active_turn_is_treated_as_handled() { let mut app = make_test_app().await; - let thread_id = ThreadId::new(); let mut app_server = crate::start_embedded_app_server_for_picker(app.chat_widget.config_ref()) .await .expect("embedded app server"); + let started = app_server + .start_thread(app.chat_widget.config_ref()) + .await + .expect("thread/start should succeed"); + let thread_id = started.session.thread_id; + app.enqueue_primary_thread_session(started.session, started.turns) + .await + .expect("primary thread should be registered"); let op = AppCommand::interrupt(); let handled = app diff --git a/codex-rs/tui/src/app_server_session.rs b/codex-rs/tui/src/app_server_session.rs index 47a06738e4..5e11a53340 100644 --- a/codex-rs/tui/src/app_server_session.rs +++ b/codex-rs/tui/src/app_server_session.rs @@ -476,6 +476,10 @@ impl AppServerSession { Ok(()) } + pub(crate) async fn startup_interrupt(&mut self, thread_id: ThreadId) -> Result<()> { + self.turn_interrupt(thread_id, String::new()).await + } + pub(crate) async fn turn_steer( &mut self, thread_id: ThreadId, From f97be7dfff18d99f1b6b25b7613a03d1f42d82d0 Mon Sep 17 00:00:00 2001 From: jackz-oai Date: Thu, 16 Apr 2026 00:13:15 -0700 Subject: [PATCH 05/59] [codex] Route Fed ChatGPT auth through Fed edge (#17151) ## Summary - parse chatgpt_account_is_fedramp from signed ChatGPT auth metadata - add _account_is_fedramp=true to ChatGPT backend-api requests only for FedRAMP ChatGPT-auth accounts --- codex-rs/backend-client/src/client.rs | 15 ++++++++++++++ codex-rs/codex-api/src/api_bridge.rs | 5 +++++ codex-rs/codex-api/src/api_bridge_tests.rs | 20 ++++++++++++++++++ codex-rs/codex-api/src/auth.rs | 24 ++++++++++++++++++++++ codex-rs/core/src/agent_identity.rs | 1 + codex-rs/core/src/mcp_openai_file.rs | 1 + codex-rs/login/src/api_bridge.rs | 4 ++++ codex-rs/login/src/auth/auth_tests.rs | 1 + codex-rs/login/src/auth/manager.rs | 6 ++++++ codex-rs/login/src/token_data.rs | 10 +++++++++ codex-rs/login/src/token_data_tests.rs | 16 +++++++++++++++ 11 files changed, 103 insertions(+) diff --git a/codex-rs/backend-client/src/client.rs b/codex-rs/backend-client/src/client.rs index 20baa1cfea..ad461973f9 100644 --- a/codex-rs/backend-client/src/client.rs +++ b/codex-rs/backend-client/src/client.rs @@ -104,6 +104,7 @@ pub struct Client { bearer_token: Option, user_agent: Option, chatgpt_account_id: Option, + chatgpt_account_is_fedramp: bool, path_style: PathStyle, } @@ -129,6 +130,7 @@ impl Client { bearer_token: None, user_agent: None, chatgpt_account_id: None, + chatgpt_account_is_fedramp: false, path_style, }) } @@ -141,6 +143,9 @@ impl Client { if let Some(account_id) = auth.get_account_id() { client = client.with_chatgpt_account_id(account_id); } + if auth.is_fedramp_account() { + client = client.with_fedramp_routing_header(); + } Ok(client) } @@ -161,6 +166,11 @@ impl Client { self } + pub fn with_fedramp_routing_header(mut self) -> Self { + self.chatgpt_account_is_fedramp = true; + self + } + pub fn with_path_style(mut self, style: PathStyle) -> Self { self.path_style = style; self @@ -185,6 +195,11 @@ impl Client { { h.insert(name, hv); } + if self.chatgpt_account_is_fedramp + && let Ok(name) = HeaderName::from_bytes(b"X-OpenAI-Fedramp") + { + h.insert(name, HeaderValue::from_static("true")); + } h } diff --git a/codex-rs/codex-api/src/api_bridge.rs b/codex-rs/codex-api/src/api_bridge.rs index 7c36c67fdd..2677ff3e55 100644 --- a/codex-rs/codex-api/src/api_bridge.rs +++ b/codex-rs/codex-api/src/api_bridge.rs @@ -179,6 +179,7 @@ struct UsageErrorBody { pub struct CoreAuthProvider { pub token: Option, pub account_id: Option, + pub is_fedramp_account: bool, } impl CoreAuthProvider { @@ -196,6 +197,7 @@ impl CoreAuthProvider { Self { token: token.map(str::to_string), account_id: account_id.map(str::to_string), + is_fedramp_account: false, } } } @@ -212,5 +214,8 @@ impl ApiAuthProvider for CoreAuthProvider { { let _ = headers.insert("ChatGPT-Account-ID", header); } + if self.is_fedramp_account { + crate::auth::add_fedramp_routing_header(headers); + } } } diff --git a/codex-rs/codex-api/src/api_bridge_tests.rs b/codex-rs/codex-api/src/api_bridge_tests.rs index 50247c1314..c7d4bbbdab 100644 --- a/codex-rs/codex-api/src/api_bridge_tests.rs +++ b/codex-rs/codex-api/src/api_bridge_tests.rs @@ -136,6 +136,7 @@ fn core_auth_provider_reports_when_auth_header_will_attach() { let auth = CoreAuthProvider { token: Some("access-token".to_string()), account_id: None, + is_fedramp_account: false, }; assert!(auth.auth_header_attached()); @@ -162,3 +163,22 @@ fn core_auth_provider_adds_auth_headers() { Some("workspace-123") ); } + +#[test] +fn core_auth_provider_adds_fedramp_routing_header_for_fedramp_accounts() { + let auth = CoreAuthProvider { + token: Some("access-token".to_string()), + account_id: Some("workspace-123".to_string()), + is_fedramp_account: true, + }; + let mut headers = HeaderMap::new(); + + crate::AuthProvider::add_auth_headers(&auth, &mut headers); + + assert_eq!( + headers + .get("X-OpenAI-Fedramp") + .and_then(|value| value.to_str().ok()), + Some("true") + ); +} diff --git a/codex-rs/codex-api/src/auth.rs b/codex-rs/codex-api/src/auth.rs index a7b1e69d14..efa5fb3288 100644 --- a/codex-rs/codex-api/src/auth.rs +++ b/codex-rs/codex-api/src/auth.rs @@ -1,4 +1,5 @@ use http::HeaderMap; +use http::HeaderValue; /// Adds authentication headers to API requests. /// @@ -8,3 +9,26 @@ use http::HeaderMap; pub trait AuthProvider: Send + Sync { fn add_auth_headers(&self, headers: &mut HeaderMap); } + +pub(crate) fn add_fedramp_routing_header(headers: &mut HeaderMap) { + headers.insert("X-OpenAI-Fedramp", HeaderValue::from_static("true")); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn add_fedramp_routing_header_sets_header() { + let mut headers = HeaderMap::new(); + + add_fedramp_routing_header(&mut headers); + + assert_eq!( + headers + .get("X-OpenAI-Fedramp") + .and_then(|v| v.to_str().ok()), + Some("true") + ); + } +} diff --git a/codex-rs/core/src/agent_identity.rs b/codex-rs/core/src/agent_identity.rs index 13a07cd961..2c897b5456 100644 --- a/codex-rs/core/src/agent_identity.rs +++ b/codex-rs/core/src/agent_identity.rs @@ -737,6 +737,7 @@ mod tests { chatgpt_plan_type: None, chatgpt_user_id: user_id.map(ToOwned::to_owned), chatgpt_account_id: Some(account_id.to_string()), + chatgpt_account_is_fedramp: false, raw_jwt: fake_id_token(account_id, user_id), }, access_token: format!("access-token-{account_id}"), diff --git a/codex-rs/core/src/mcp_openai_file.rs b/codex-rs/core/src/mcp_openai_file.rs index 587dd4b770..33d0a3f1f0 100644 --- a/codex-rs/core/src/mcp_openai_file.rs +++ b/codex-rs/core/src/mcp_openai_file.rs @@ -115,6 +115,7 @@ async fn build_uploaded_local_argument_value( let upload_auth = CoreAuthProvider { token: Some(token_data.access_token), account_id: token_data.account_id, + is_fedramp_account: auth.is_fedramp_account(), }; let uploaded = upload_local_file( turn_context.config.chatgpt_base_url.trim_end_matches('/'), diff --git a/codex-rs/login/src/api_bridge.rs b/codex-rs/login/src/api_bridge.rs index d8b9dbb77c..684c890bde 100644 --- a/codex-rs/login/src/api_bridge.rs +++ b/codex-rs/login/src/api_bridge.rs @@ -11,6 +11,7 @@ pub fn auth_provider_from_auth( return Ok(CoreAuthProvider { token: Some(api_key), account_id: None, + is_fedramp_account: false, }); } @@ -18,6 +19,7 @@ pub fn auth_provider_from_auth( return Ok(CoreAuthProvider { token: Some(token), account_id: None, + is_fedramp_account: false, }); } @@ -26,11 +28,13 @@ pub fn auth_provider_from_auth( Ok(CoreAuthProvider { token: Some(token), account_id: auth.get_account_id(), + is_fedramp_account: auth.is_fedramp_account(), }) } else { Ok(CoreAuthProvider { token: None, account_id: None, + is_fedramp_account: false, }) } } diff --git a/codex-rs/login/src/auth/auth_tests.rs b/codex-rs/login/src/auth/auth_tests.rs index 8a16f5939c..30244ac268 100644 --- a/codex-rs/login/src/auth/auth_tests.rs +++ b/codex-rs/login/src/auth/auth_tests.rs @@ -130,6 +130,7 @@ async fn pro_account_with_no_api_key_uses_chatgpt_auth() { chatgpt_plan_type: Some(InternalPlanType::Known(InternalKnownPlan::Pro)), chatgpt_user_id: Some("user-12345".to_string()), chatgpt_account_id: None, + chatgpt_account_is_fedramp: false, raw_jwt: fake_jwt, }, access_token: "test-access-token".to_string(), diff --git a/codex-rs/login/src/auth/manager.rs b/codex-rs/login/src/auth/manager.rs index 8768171d95..3cb97f58b0 100644 --- a/codex-rs/login/src/auth/manager.rs +++ b/codex-rs/login/src/auth/manager.rs @@ -293,6 +293,12 @@ impl CodexAuth { self.get_current_token_data().and_then(|t| t.account_id) } + /// Returns false if `is_chatgpt_auth()` is false or the token omits the FedRAMP claim. + pub fn is_fedramp_account(&self) -> bool { + self.get_current_token_data() + .is_some_and(|t| t.id_token.is_fedramp_account()) + } + /// Returns `None` if `is_chatgpt_auth()` is false. pub fn get_account_email(&self) -> Option { self.get_current_token_data().and_then(|t| t.id_token.email) diff --git a/codex-rs/login/src/token_data.rs b/codex-rs/login/src/token_data.rs index f70696d5a2..2952ecb2a7 100644 --- a/codex-rs/login/src/token_data.rs +++ b/codex-rs/login/src/token_data.rs @@ -36,6 +36,8 @@ pub struct IdTokenInfo { pub chatgpt_user_id: Option, /// Organization/workspace identifier associated with the token, if present. pub chatgpt_account_id: Option, + /// Whether the selected ChatGPT workspace must route through the FedRAMP edge. + pub chatgpt_account_is_fedramp: bool, pub raw_jwt: String, } @@ -60,6 +62,10 @@ impl IdTokenInfo { Some(PlanType::Known(plan)) if plan.is_workspace_account() ) } + + pub fn is_fedramp_account(&self) -> bool { + self.chatgpt_account_is_fedramp + } } #[derive(Deserialize)] @@ -88,6 +94,8 @@ struct AuthClaims { user_id: Option, #[serde(default)] chatgpt_account_id: Option, + #[serde(default)] + chatgpt_account_is_fedramp: bool, } #[derive(Deserialize)] @@ -139,6 +147,7 @@ pub fn parse_chatgpt_jwt_claims(jwt: &str) -> Result Ok(IdTokenInfo { email, @@ -146,6 +155,7 @@ pub fn parse_chatgpt_jwt_claims(jwt: &str) -> Result Date: Thu, 16 Apr 2026 10:01:45 +0100 Subject: [PATCH 06/59] fix: more flake (#18006) Stabilizes the Responses API proxy header test by splitting the coverage at the right boundary: - Core integration test now verifies parent/subagent identity headers directly from captured `/responses` requests. - Proxy dump unit test now verifies those identity headers are preserved in dumped request JSON. - Removes the flaky real proxy process + temp-file dump polling path from the core test. --- .../suite/responses_api_proxy_headers.rs | 296 +++++------------- codex-rs/responses-api-proxy/src/dump.rs | 18 ++ 2 files changed, 92 insertions(+), 222 deletions(-) diff --git a/codex-rs/core/tests/suite/responses_api_proxy_headers.rs b/codex-rs/core/tests/suite/responses_api_proxy_headers.rs index 8199956ee0..67f6e86ab0 100644 --- a/codex-rs/core/tests/suite/responses_api_proxy_headers.rs +++ b/codex-rs/core/tests/suite/responses_api_proxy_headers.rs @@ -1,6 +1,5 @@ -//! Exercises a real `responses-api-proxy` process with request dumping enabled, then verifies that -//! parent and spawned subagent requests carry the expected window, parent-thread, and subagent -//! identity headers in the dumped Responses API requests. +//! Verifies that parent and spawned subagent Responses API requests carry the expected window, +//! parent-thread, and subagent identity headers. use anyhow::Result; use anyhow::anyhow; @@ -10,6 +9,8 @@ use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::Op; use codex_protocol::protocol::SandboxPolicy; use codex_protocol::user_input::UserInput; +use core_test_support::responses::ResponseMock; +use core_test_support::responses::ResponsesRequest; use core_test_support::responses::ev_assistant_message; use core_test_support::responses::ev_completed; use core_test_support::responses::ev_function_call; @@ -21,119 +22,28 @@ use core_test_support::skip_if_no_network; use core_test_support::test_codex::TestCodex; use core_test_support::test_codex::test_codex; use pretty_assertions::assert_eq; -use serde_json::Value; use serde_json::json; -use std::io::ErrorKind; -use std::io::Write; -use std::path::Path; -use std::process::Child; -use std::process::Command as StdCommand; -use std::process::Stdio; use std::time::Duration; -use std::time::Instant; -use tempfile::TempDir; const PARENT_PROMPT: &str = "spawn a subagent and report when it is started"; const CHILD_PROMPT: &str = "child: say done"; const SPAWN_CALL_ID: &str = "spawn-call-1"; -const PROXY_START_TIMEOUT: Duration = Duration::from_secs(/*secs*/ 30); -const PROXY_POLL_INTERVAL: Duration = Duration::from_millis(/*millis*/ 20); +const REQUEST_POLL_INTERVAL: Duration = Duration::from_millis(/*millis*/ 20); const TURN_TIMEOUT: Duration = Duration::from_secs(/*secs*/ 60); -struct ResponsesApiProxy { - child: Child, - port: u16, -} - -impl ResponsesApiProxy { - fn start(upstream_url: &str, dump_dir: &Path) -> Result { - let server_info = dump_dir.join("server-info.json"); - let (proxy_program, use_codex_multitool) = - match codex_utils_cargo_bin::cargo_bin("codex-responses-api-proxy") { - Ok(path) => (path, false), - Err(_) => (codex_utils_cargo_bin::cargo_bin("codex")?, true), - }; - let mut command = StdCommand::new(proxy_program); - if use_codex_multitool { - command.arg("responses-api-proxy"); - } - let mut child = command - .args(["--server-info"]) - .arg(&server_info) - .args(["--upstream-url", upstream_url, "--dump-dir"]) - .arg(dump_dir) - .stdin(Stdio::piped()) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn()?; - - child - .stdin - .take() - .ok_or_else(|| anyhow!("responses-api-proxy stdin was not piped"))? - .write_all(b"dummy\n")?; - - let deadline = Instant::now() + PROXY_START_TIMEOUT; - loop { - match std::fs::read_to_string(&server_info) { - Ok(info) => { - if !info.trim().is_empty() { - match serde_json::from_str::(&info) { - Ok(info) => { - let port = info - .get("port") - .and_then(Value::as_u64) - .ok_or_else(|| anyhow!("proxy server info missing port"))?; - return Ok(Self { - child, - port: u16::try_from(port)?, - }); - } - Err(err) if err.is_eof() => {} - Err(err) => return Err(err.into()), - } - } - } - Err(err) if err.kind() == ErrorKind::NotFound => {} - Err(err) => return Err(err.into()), - } - if let Some(status) = child.try_wait()? { - return Err(anyhow!( - "responses-api-proxy exited before writing server info: {status}" - )); - } - if Instant::now() >= deadline { - return Err(anyhow!("timed out waiting for responses-api-proxy")); - } - std::thread::sleep(PROXY_POLL_INTERVAL); - } - } - - fn base_url(&self) -> String { - format!("http://127.0.0.1:{}/v1", self.port) - } -} - -impl Drop for ResponsesApiProxy { - fn drop(&mut self) { - let _ = self.child.kill(); - let _ = self.child.wait(); - } -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn responses_api_proxy_dumps_parent_and_subagent_identity_headers() -> Result<()> { +async fn responses_api_parent_and_subagent_requests_include_identity_headers() -> Result<()> { skip_if_no_network!(Ok(())); let server = start_mock_server().await; - let dump_dir = TempDir::new()?; - let proxy = - ResponsesApiProxy::start(&format!("{}/v1/responses", server.uri()), dump_dir.path())?; let spawn_args = serde_json::to_string(&json!({ "message": CHILD_PROMPT }))?; - mount_sse_once_match( + let parent_mock = mount_sse_once_match( &server, - |req: &wiremock::Request| request_body_contains(req, PARENT_PROMPT), + |req: &wiremock::Request| { + request_body_contains(req, PARENT_PROMPT) + && request_header(req, "x-openai-subagent").is_none() + }, sse(vec![ ev_response_created("resp-parent-1"), ev_function_call(SPAWN_CALL_ID, "spawn_agent", &spawn_args), @@ -141,10 +51,12 @@ async fn responses_api_proxy_dumps_parent_and_subagent_identity_headers() -> Res ]), ) .await; - mount_sse_once_match( + let child_mock = mount_sse_once_match( &server, |req: &wiremock::Request| { - request_body_contains(req, CHILD_PROMPT) && !request_body_contains(req, SPAWN_CALL_ID) + request_body_contains(req, CHILD_PROMPT) + && !request_body_contains(req, SPAWN_CALL_ID) + && request_header(req, "x-openai-subagent") == Some("collab_spawn") }, sse(vec![ ev_response_created("resp-child-1"), @@ -155,7 +67,10 @@ async fn responses_api_proxy_dumps_parent_and_subagent_identity_headers() -> Res .await; mount_sse_once_match( &server, - |req: &wiremock::Request| request_body_contains(req, SPAWN_CALL_ID), + |req: &wiremock::Request| { + request_body_contains(req, SPAWN_CALL_ID) + && request_header(req, "x-openai-subagent").is_none() + }, sse(vec![ ev_response_created("resp-parent-2"), ev_assistant_message("msg-parent-2", "parent done"), @@ -164,50 +79,52 @@ async fn responses_api_proxy_dumps_parent_and_subagent_identity_headers() -> Res ) .await; - let proxy_base_url = proxy.base_url(); - let mut builder = test_codex().with_config(move |config| { - config.model_provider.base_url = Some(proxy_base_url); + let mut builder = test_codex().with_config(|config| { config .features .disable(Feature::EnableRequestCompression) .expect("test config should allow feature update"); }); let test = builder.build(&server).await?; - submit_turn_with_timeout(&test, PARENT_PROMPT, dump_dir.path()).await?; + submit_turn_with_timeout(&test, PARENT_PROMPT).await?; - let dumps = wait_for_proxy_request_dumps(dump_dir.path())?; - let parent = dumps - .iter() - .find(|dump| dump_body_contains(dump, PARENT_PROMPT)) - .ok_or_else(|| anyhow!("missing parent request dump"))?; - let child = dumps - .iter() - .find(|dump| { - dump_body_contains(dump, CHILD_PROMPT) && !dump_body_contains(dump, SPAWN_CALL_ID) - }) - .ok_or_else(|| anyhow!("missing child request dump"))?; + let parent = wait_for_matching_request(&parent_mock, "parent request", |request| { + request.body_contains_text(PARENT_PROMPT) && request.header("x-openai-subagent").is_none() + }) + .await?; + let child = wait_for_matching_request(&child_mock, "child request", |request| { + request.body_contains_text(CHILD_PROMPT) + && !request.body_contains_text(SPAWN_CALL_ID) + && request.header("x-openai-subagent").as_deref() == Some("collab_spawn") + }) + .await?; - let parent_window_id = header(parent, "x-codex-window-id") + let parent_window_id = parent + .header("x-codex-window-id") .ok_or_else(|| anyhow!("parent request missing x-codex-window-id"))?; - let child_window_id = header(child, "x-codex-window-id") + let child_window_id = child + .header("x-codex-window-id") .ok_or_else(|| anyhow!("child request missing x-codex-window-id"))?; - let (parent_thread_id, parent_generation) = split_window_id(parent_window_id)?; - let (child_thread_id, child_generation) = split_window_id(child_window_id)?; + let (parent_thread_id, parent_generation) = split_window_id(&parent_window_id)?; + let (child_thread_id, child_generation) = split_window_id(&child_window_id)?; assert_eq!(parent_generation, 0); assert_eq!(child_generation, 0); assert!(child_thread_id != parent_thread_id); - assert_eq!(header(parent, "x-openai-subagent"), None); - assert_eq!(header(child, "x-openai-subagent"), Some("collab_spawn")); + assert_eq!(parent.header("x-openai-subagent"), None); assert_eq!( - header(child, "x-codex-parent-thread-id"), + child.header("x-openai-subagent").as_deref(), + Some("collab_spawn") + ); + assert_eq!( + child.header("x-codex-parent-thread-id").as_deref(), Some(parent_thread_id) ); Ok(()) } -async fn submit_turn_with_timeout(test: &TestCodex, prompt: &str, dump_dir: &Path) -> Result<()> { +async fn submit_turn_with_timeout(test: &TestCodex, prompt: &str) -> Result<()> { let session_model = test.session_configured.model.clone(); test.codex .submit(Op::UserTurn { @@ -235,14 +152,14 @@ async fn submit_turn_with_timeout(test: &TestCodex, prompt: &str, dump_dir: &Pat }) .await?; - let turn_started = wait_for_event_result(test, "turn started", dump_dir, |event| { + let turn_started = wait_for_event_result(test, "turn started", |event| { matches!(event, EventMsg::TurnStarted(_)) }) .await?; let EventMsg::TurnStarted(turn_started) = turn_started else { unreachable!("event predicate only matches turn started events"); }; - wait_for_event_result(test, "turn complete", dump_dir, |event| match event { + wait_for_event_result(test, "turn complete", |event| match event { EventMsg::TurnComplete(event) => event.turn_id == turn_started.turn_id, _ => false, }) @@ -251,10 +168,33 @@ async fn submit_turn_with_timeout(test: &TestCodex, prompt: &str, dump_dir: &Pat Ok(()) } +async fn wait_for_matching_request( + mock: &ResponseMock, + label: &str, + mut predicate: F, +) -> Result +where + F: FnMut(&ResponsesRequest) -> bool, +{ + tokio::time::timeout(TURN_TIMEOUT, async { + loop { + if let Some(request) = mock + .requests() + .into_iter() + .find(|request| predicate(request)) + { + return request; + } + tokio::time::sleep(REQUEST_POLL_INTERVAL).await; + } + }) + .await + .map_err(|_| anyhow!("timed out waiting for {label}")) +} + async fn wait_for_event_result( test: &TestCodex, stage: &str, - dump_dir: &Path, mut predicate: F, ) -> Result where @@ -273,9 +213,8 @@ where .await .map_err(|_| { anyhow!( - "timed out waiting for {stage}; saw events: {}; {}", - seen_events.join(" | "), - proxy_dump_summary(dump_dir) + "timed out waiting for {stage}; saw events: {}", + seen_events.join(" | ") ) })? } @@ -290,95 +229,8 @@ fn request_body_contains(req: &wiremock::Request, text: &str) -> bool { std::str::from_utf8(&req.body).is_ok_and(|body| body.contains(text)) } -fn wait_for_proxy_request_dumps(dump_dir: &Path) -> Result> { - let deadline = Instant::now() + Duration::from_secs(/*secs*/ 2); - loop { - let dumps = read_proxy_request_dumps(dump_dir).unwrap_or_default(); - if dumps.len() >= 3 - && dumps - .iter() - .any(|dump| dump_body_contains(dump, CHILD_PROMPT)) - { - return Ok(dumps); - } - if Instant::now() >= deadline { - return Err(anyhow!( - "timed out waiting for proxy request dumps, got {}", - dumps.len() - )); - } - std::thread::sleep(PROXY_POLL_INTERVAL); - } -} - -fn read_proxy_request_dumps(dump_dir: &Path) -> Result> { - let mut dumps = Vec::new(); - for entry in std::fs::read_dir(dump_dir)? { - let path = entry?.path(); - if path - .file_name() - .and_then(|name| name.to_str()) - .is_some_and(|name| name.ends_with("-request.json")) - { - let contents = std::fs::read_to_string(&path)?; - if contents.trim().is_empty() { - continue; - } - - match serde_json::from_str(&contents) { - Ok(dump) => dumps.push(dump), - Err(err) if err.is_eof() => continue, - Err(err) => return Err(err.into()), - } - } - } - Ok(dumps) -} - -fn proxy_dump_summary(dump_dir: &Path) -> String { - match read_proxy_request_dumps(dump_dir) { - Ok(dumps) => { - let bodies = dumps - .iter() - .filter_map(|dump| dump.get("body")) - .map(Value::to_string) - .collect::>() - .join("; "); - format!("proxy wrote {} request dumps: {bodies}", dumps.len()) - } - Err(err) => format!("failed to read proxy request dumps: {err}"), - } -} - -#[test] -fn read_proxy_request_dumps_ignores_in_progress_files() -> Result<()> { - let dump_dir = TempDir::new()?; - std::fs::write(dump_dir.path().join("empty-request.json"), "")?; - std::fs::write(dump_dir.path().join("partial-request.json"), "{\"body\"")?; - std::fs::write( - dump_dir.path().join("complete-request.json"), - serde_json::to_string(&json!({ "body": "ready" }))?, - )?; - - assert_eq!( - read_proxy_request_dumps(dump_dir.path())?, - vec![json!({ "body": "ready" })] - ); - - Ok(()) -} - -fn dump_body_contains(dump: &Value, text: &str) -> bool { - dump.get("body") - .is_some_and(|body| body.to_string().contains(text)) -} - -fn header<'a>(dump: &'a Value, name: &str) -> Option<&'a str> { - dump.get("headers")?.as_array()?.iter().find_map(|header| { - (header.get("name")?.as_str()?.eq_ignore_ascii_case(name)) - .then(|| header.get("value")?.as_str()) - .flatten() - }) +fn request_header<'a>(req: &'a wiremock::Request, name: &str) -> Option<&'a str> { + req.headers.get(name).and_then(|value| value.to_str().ok()) } fn split_window_id(window_id: &str) -> Result<(&str, u64)> { diff --git a/codex-rs/responses-api-proxy/src/dump.rs b/codex-rs/responses-api-proxy/src/dump.rs index bebd4ed592..1a09c64ae7 100644 --- a/codex-rs/responses-api-proxy/src/dump.rs +++ b/codex-rs/responses-api-proxy/src/dump.rs @@ -231,6 +231,12 @@ mod tests { Header::from_bytes(&b"Cookie"[..], &b"user-session=secret"[..]).expect("cookie header"), Header::from_bytes(&b"Content-Type"[..], &b"application/json"[..]) .expect("content-type header"), + Header::from_bytes(&b"x-codex-window-id"[..], &b"thread-1:0"[..]) + .expect("window id header"), + Header::from_bytes(&b"x-codex-parent-thread-id"[..], &b"parent-thread-1"[..]) + .expect("parent thread id header"), + Header::from_bytes(&b"x-openai-subagent"[..], &b"collab_spawn"[..]) + .expect("subagent header"), ]; let exchange_dump = dumper @@ -262,6 +268,18 @@ mod tests { { "name": "Content-Type", "value": "application/json" + }, + { + "name": "x-codex-window-id", + "value": "thread-1:0" + }, + { + "name": "x-codex-parent-thread-id", + "value": "parent-thread-1" + }, + { + "name": "x-openai-subagent", + "value": "collab_spawn" } ], "body": { From d4223091d07a7647535b46795aa87cf78550e3f4 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Thu, 16 Apr 2026 13:52:21 +0100 Subject: [PATCH 07/59] fix: windows flake (#18127) Fix `sqlite_feedback_logs_match_feedback_formatter_shape` by explicitly flushing the async log DB layer before querying SQLite. --- codex-rs/state/src/log_db.rs | 37 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/codex-rs/state/src/log_db.rs b/codex-rs/state/src/log_db.rs index e33baae573..c28f762d27 100644 --- a/codex-rs/state/src/log_db.rs +++ b/codex-rs/state/src/log_db.rs @@ -384,9 +384,8 @@ mod tests { use std::io; use std::sync::Arc; use std::sync::Mutex; - use std::time::Duration; - use tokio::time::Instant; + use pretty_assertions::assert_eq; use tracing_subscriber::filter::Targets; use tracing_subscriber::fmt::writer::MakeWriter; use tracing_subscriber::layer::SubscriberExt; @@ -442,6 +441,7 @@ mod tests { .await .expect("initialize runtime"); let writer = SharedWriter::default(); + let layer = start(runtime.clone()); let subscriber = tracing_subscriber::registry() .with( @@ -452,7 +452,8 @@ mod tests { .with_filter(Targets::new().with_default(tracing::Level::TRACE)), ) .with( - start(runtime.clone()) + layer + .clone() .with_filter(Targets::new().with_default(tracing::Level::TRACE)), ); let guard = subscriber.set_default(); @@ -463,6 +464,7 @@ mod tests { }); tracing::debug!("threadless-after"); + layer.flush().await; drop(guard); let feedback_logs = writer.snapshot(); @@ -475,24 +477,17 @@ mod tests { .collect::>() .join("\n") }; - let deadline = Instant::now() + Duration::from_secs(2); - loop { - let sqlite_logs = String::from_utf8( - runtime - .query_feedback_logs("thread-1") - .await - .expect("query feedback logs"), - ) - .expect("valid utf-8"); - if without_timestamps(&sqlite_logs) == without_timestamps(&feedback_logs) { - break; - } - assert!( - Instant::now() < deadline, - "sqlite feedback logs did not match feedback formatter output before timeout\nsqlite:\n{sqlite_logs}\nfeedback:\n{feedback_logs}" - ); - tokio::time::sleep(Duration::from_millis(10)).await; - } + let sqlite_logs = String::from_utf8( + runtime + .query_feedback_logs("thread-1") + .await + .expect("query feedback logs"), + ) + .expect("valid utf-8"); + assert_eq!( + without_timestamps(&sqlite_logs), + without_timestamps(&feedback_logs) + ); let _ = tokio::fs::remove_dir_all(codex_home).await; } From 9c326c4cb4febbf5ed38ffec95f9f38e95085c1a Mon Sep 17 00:00:00 2001 From: jif-oai Date: Thu, 16 Apr 2026 14:37:43 +0100 Subject: [PATCH 08/59] nit: add min values for memories (#18137) Just add min values to some memories config fields --- codex-rs/config/src/types.rs | 16 ++++++++++++++-- codex-rs/config/src/types_tests.rs | 18 ++++++++++++++++++ codex-rs/core/config.schema.json | 6 ++++-- 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/codex-rs/config/src/types.rs b/codex-rs/config/src/types.rs index 73a5763fe1..15c1a6fad5 100644 --- a/codex-rs/config/src/types.rs +++ b/codex-rs/config/src/types.rs @@ -31,6 +31,10 @@ pub const DEFAULT_MEMORIES_MAX_ROLLOUT_AGE_DAYS: i64 = 30; pub const DEFAULT_MEMORIES_MIN_ROLLOUT_IDLE_HOURS: i64 = 6; pub const DEFAULT_MEMORIES_MAX_RAW_MEMORIES_FOR_CONSOLIDATION: usize = 256; pub const DEFAULT_MEMORIES_MAX_UNUSED_DAYS: i64 = 30; +const MIN_MEMORIES_MAX_RAW_MEMORIES_FOR_CONSOLIDATION: usize = 1; +const MAX_MEMORIES_MAX_RAW_MEMORIES_FOR_CONSOLIDATION: usize = 4096; +const MIN_MEMORIES_MAX_ROLLOUTS_PER_STARTUP: usize = 1; +const MAX_MEMORIES_MAX_ROLLOUTS_PER_STARTUP: usize = 128; const fn default_enabled() -> bool { true @@ -185,12 +189,14 @@ pub struct MemoriesToml { /// When `false`, skip injecting memory usage instructions into developer prompts. pub use_memories: Option, /// Maximum number of recent raw memories retained for global consolidation. + #[schemars(range(min = 1, max = 4096))] pub max_raw_memories_for_consolidation: Option, /// Maximum number of days since a memory was last used before it becomes ineligible for phase 2 selection. pub max_unused_days: Option, /// Maximum age of the threads used for memories. pub max_rollout_age_days: Option, /// Maximum number of rollout candidates processed per pass. + #[schemars(range(min = 1, max = 128))] pub max_rollouts_per_startup: Option, /// Minimum idle time between last thread activity and memory creation (hours). > 12h recommended. pub min_rollout_idle_hours: Option, @@ -244,7 +250,10 @@ impl From for MemoriesConfig { max_raw_memories_for_consolidation: toml .max_raw_memories_for_consolidation .unwrap_or(defaults.max_raw_memories_for_consolidation) - .min(4096), + .clamp( + MIN_MEMORIES_MAX_RAW_MEMORIES_FOR_CONSOLIDATION, + MAX_MEMORIES_MAX_RAW_MEMORIES_FOR_CONSOLIDATION, + ), max_unused_days: toml .max_unused_days .unwrap_or(defaults.max_unused_days) @@ -256,7 +265,10 @@ impl From for MemoriesConfig { max_rollouts_per_startup: toml .max_rollouts_per_startup .unwrap_or(defaults.max_rollouts_per_startup) - .min(128), + .clamp( + MIN_MEMORIES_MAX_ROLLOUTS_PER_STARTUP, + MAX_MEMORIES_MAX_ROLLOUTS_PER_STARTUP, + ), min_rollout_idle_hours: toml .min_rollout_idle_hours .unwrap_or(defaults.min_rollout_idle_hours) diff --git a/codex-rs/config/src/types_tests.rs b/codex-rs/config/src/types_tests.rs index fbcd2bdc64..b18c1cc645 100644 --- a/codex-rs/config/src/types_tests.rs +++ b/codex-rs/config/src/types_tests.rs @@ -41,3 +41,21 @@ fn deserialize_skill_config_with_path_selector() { } ); } + +#[test] +fn memories_config_clamps_count_limits_to_nonzero_values() { + let config = MemoriesConfig::from(MemoriesToml { + max_raw_memories_for_consolidation: Some(0), + max_rollouts_per_startup: Some(0), + ..Default::default() + }); + + assert_eq!( + config, + MemoriesConfig { + max_raw_memories_for_consolidation: 1, + max_rollouts_per_startup: 1, + ..MemoriesConfig::default() + } + ); +} diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.json index c61daa8b56..39c50bcd0d 100644 --- a/codex-rs/core/config.schema.json +++ b/codex-rs/core/config.schema.json @@ -855,7 +855,8 @@ "max_raw_memories_for_consolidation": { "description": "Maximum number of recent raw memories retained for global consolidation.", "format": "uint", - "minimum": 0.0, + "maximum": 4096.0, + "minimum": 1.0, "type": "integer" }, "max_rollout_age_days": { @@ -866,7 +867,8 @@ "max_rollouts_per_startup": { "description": "Maximum number of rollout candidates processed per pass.", "format": "uint", - "minimum": 0.0, + "maximum": 128.0, + "minimum": 1.0, "type": "integer" }, "max_unused_days": { From de98b1d3e8b9f36a63e4b872d138d8401b4f50a3 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Thu, 16 Apr 2026 14:51:47 +0100 Subject: [PATCH 09/59] debug: windows flake (#18135) Make sure Bazel logs shows every errors so that we can debug flakes + fix a small flake on Windows by updating the sleep command to a `Start-Sleep` instead of a PowerShell nested command (otherwise we had double nesting which is absurdely slow) --- .github/scripts/run-bazel-ci.sh | 5 +++++ codex-rs/app-server/tests/suite/v2/thread_unsubscribe.rs | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/scripts/run-bazel-ci.sh b/.github/scripts/run-bazel-ci.sh index fefca90b3a..c8c09f406a 100755 --- a/.github/scripts/run-bazel-ci.sh +++ b/.github/scripts/run-bazel-ci.sh @@ -94,6 +94,11 @@ print_bazel_test_log_tails() { local rel_path="${target#//}" rel_path="${rel_path/://}" local test_log="${testlogs_dir}/${rel_path}/test.log" + local reported_test_log + reported_test_log="$(grep -F "FAIL: ${target} " "$console_log" | sed -nE 's#.* \(see ([^)]+/test\.log)\).*#\1#p' | head -n 1 || true)" + if [[ -n "$reported_test_log" ]]; then + test_log="$reported_test_log" + fi echo "::group::Bazel test log tail for ${target}" if [[ -f "$test_log" ]]; then diff --git a/codex-rs/app-server/tests/suite/v2/thread_unsubscribe.rs b/codex-rs/app-server/tests/suite/v2/thread_unsubscribe.rs index 6aab3d186f..1650e55818 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_unsubscribe.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_unsubscribe.rs @@ -130,9 +130,9 @@ async fn thread_unsubscribe_keeps_thread_loaded_until_idle_timeout() -> Result<( async fn thread_unsubscribe_during_turn_keeps_turn_running() -> Result<()> { #[cfg(target_os = "windows")] let shell_command = vec![ - "powershell".to_string(), - "-Command".to_string(), - "Start-Sleep -Seconds 1".to_string(), + "Start-Sleep".to_string(), + "-Seconds".to_string(), + "1".to_string(), ]; #[cfg(not(target_os = "windows"))] let shell_command = vec!["sleep".to_string(), "1".to_string()]; From 18e9ac8c75695d6b8375f125ec43b1ee4a424a31 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Thu, 16 Apr 2026 15:32:32 +0100 Subject: [PATCH 10/59] chore: more pollution filtering (#18138) --- codex-rs/core/src/codex.rs | 10 ++- codex-rs/core/src/stream_events_utils.rs | 15 +++- .../core/src/stream_events_utils_tests.rs | 83 +++++++++++++++++++ 3 files changed, 104 insertions(+), 4 deletions(-) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 66a0dc527e..c7931b27df 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -41,6 +41,7 @@ use crate::stream_events_utils::HandleOutputCtx; use crate::stream_events_utils::handle_non_tool_response_item; use crate::stream_events_utils::handle_output_item_done; use crate::stream_events_utils::last_assistant_message_from_item; +use crate::stream_events_utils::mark_thread_memory_mode_polluted_if_external_context; use crate::stream_events_utils::raw_assistant_output_text_from_item; use crate::stream_events_utils::record_completed_response_item; use crate::turn_metadata::TurnMetadataState; @@ -7830,8 +7831,15 @@ async fn drain_in_flight( while let Some(res) = in_flight.next().await { match res { Ok(response_input) => { - sess.record_conversation_items(&turn_context, &[response_input.into()]) + let response_item = response_input.into(); + sess.record_conversation_items(&turn_context, std::slice::from_ref(&response_item)) .await; + mark_thread_memory_mode_polluted_if_external_context( + sess.as_ref(), + turn_context.as_ref(), + &response_item, + ) + .await; } Err(err) => { error_or_panic(format!("in-flight tool future failed during drain: {err}")); diff --git a/codex-rs/core/src/stream_events_utils.rs b/codex-rs/core/src/stream_events_utils.rs index d491bb6adb..3838d6ed01 100644 --- a/codex-rs/core/src/stream_events_utils.rs +++ b/codex-rs/core/src/stream_events_utils.rs @@ -136,11 +136,20 @@ pub(crate) async fn record_completed_response_item( sess.defer_mailbox_delivery_to_next_turn(&turn_context.sub_id) .await; } - maybe_mark_thread_memory_mode_polluted_from_web_search(sess, turn_context, item).await; + mark_thread_memory_mode_polluted_if_external_context(sess, turn_context, item).await; record_stage1_output_usage_for_completed_item(turn_context, item).await; } -async fn maybe_mark_thread_memory_mode_polluted_from_web_search( +fn response_item_may_include_external_context(item: &ResponseItem) -> bool { + matches!( + item, + ResponseItem::ToolSearchCall { .. } + | ResponseItem::ToolSearchOutput { .. } + | ResponseItem::WebSearchCall { .. } + ) +} + +pub(crate) async fn mark_thread_memory_mode_polluted_if_external_context( sess: &Session, turn_context: &TurnContext, item: &ResponseItem, @@ -149,7 +158,7 @@ async fn maybe_mark_thread_memory_mode_polluted_from_web_search( .config .memories .no_memories_if_mcp_or_web_search - || !matches!(item, ResponseItem::WebSearchCall { .. }) + || !response_item_may_include_external_context(item) { return; } diff --git a/codex-rs/core/src/stream_events_utils_tests.rs b/codex-rs/core/src/stream_events_utils_tests.rs index e3947a06ed..2634a88742 100644 --- a/codex-rs/core/src/stream_events_utils_tests.rs +++ b/codex-rs/core/src/stream_events_utils_tests.rs @@ -2,11 +2,16 @@ use super::completed_item_defers_mailbox_delivery_to_next_turn; use super::handle_non_tool_response_item; use super::image_generation_artifact_path; use super::last_assistant_message_from_item; +use super::response_item_may_include_external_context; use super::save_image_generation_result; use crate::codex::make_session_and_context; use codex_protocol::error::CodexErr; use codex_protocol::items::TurnItem; use codex_protocol::models::ContentItem; +use codex_protocol::models::FunctionCallOutputPayload; +use codex_protocol::models::LocalShellAction; +use codex_protocol::models::LocalShellExecAction; +use codex_protocol::models::LocalShellStatus; use codex_protocol::models::MessagePhase; use codex_protocol::models::ResponseItem; use codex_utils_absolute_path::test_support::PathExt; @@ -28,6 +33,84 @@ fn assistant_output_text_with_phase(text: &str, phase: Option) -> } } +#[test] +fn external_context_pollution_items_include_web_search_and_tool_search() { + let polluting_items = [ + ResponseItem::WebSearchCall { + id: None, + status: Some("completed".to_string()), + action: None, + }, + ResponseItem::ToolSearchCall { + id: None, + call_id: Some("search-1".to_string()), + status: None, + execution: "client".to_string(), + arguments: serde_json::json!({"query": "calendar"}), + }, + ResponseItem::ToolSearchOutput { + call_id: Some("search-1".to_string()), + status: "completed".to_string(), + execution: "client".to_string(), + tools: Vec::new(), + }, + ]; + + assert!( + polluting_items + .iter() + .all(response_item_may_include_external_context) + ); +} + +#[test] +fn external_context_pollution_items_exclude_local_tool_calls() { + let non_polluting_items = [ + ResponseItem::LocalShellCall { + id: None, + call_id: Some("shell-1".to_string()), + status: LocalShellStatus::Completed, + action: LocalShellAction::Exec(LocalShellExecAction { + command: vec!["cat".to_string(), "README.md".to_string()], + timeout_ms: None, + working_directory: None, + env: None, + user: None, + }), + }, + ResponseItem::FunctionCall { + id: None, + name: "shell".to_string(), + namespace: None, + arguments: "{}".to_string(), + call_id: "call-1".to_string(), + }, + ResponseItem::FunctionCallOutput { + call_id: "call-1".to_string(), + output: FunctionCallOutputPayload::from_text("ok".to_string()), + }, + ResponseItem::CustomToolCall { + id: None, + status: None, + call_id: "custom-1".to_string(), + name: "apply_patch".to_string(), + input: "*** Begin Patch\n*** End Patch\n".to_string(), + }, + ResponseItem::CustomToolCallOutput { + call_id: "custom-1".to_string(), + name: Some("apply_patch".to_string()), + output: FunctionCallOutputPayload::from_text("ok".to_string()), + }, + assistant_output_text("plain assistant text"), + ]; + + assert!( + !non_polluting_items + .iter() + .any(response_item_may_include_external_context) + ); +} + #[tokio::test] async fn handle_non_tool_response_item_strips_citations_from_assistant_message() { let (session, turn_context) = make_session_and_context().await; From b33478c23627c64c0864ae7f2f649d9d201b6261 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Thu, 16 Apr 2026 15:44:23 +0100 Subject: [PATCH 11/59] chore: unify memory drop endpoints (#18134) Unify all the memories drop behind a single implementation that drops both the main memories and the extensions --- .../app-server/src/codex_message_processor.rs | 43 ++----------------- codex-rs/cli/src/main.rs | 21 +++------ codex-rs/cli/tests/debug_clear_memories.rs | 3 +- codex-rs/core/src/codex.rs | 8 ++-- codex-rs/core/src/lib.rs | 1 + codex-rs/core/src/memories/control.rs | 11 +++++ codex-rs/core/src/memories/mod.rs | 2 +- codex-rs/core/src/memories/tests.rs | 2 +- 8 files changed, 30 insertions(+), 61 deletions(-) diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index 2585383cda..4d12df89fa 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -212,6 +212,7 @@ use codex_core::SteerInputError; use codex_core::ThreadConfigSnapshot; use codex_core::ThreadManager; use codex_core::append_thread_name; +use codex_core::clear_memory_roots_contents; use codex_core::config::Config; use codex_core::config::ConfigOverrides; use codex_core::config::NetworkProxyAuditMetadata; @@ -3080,48 +3081,12 @@ impl CodexMessageProcessor { return; } - let memory_root = self.config.codex_home.join("memories"); - let memory_extensions_root = self.config.codex_home.join("memories_extensions"); - let clear_memory_root_result: std::io::Result<()> = async { - for directory in [memory_root.as_path(), memory_extensions_root.as_path()] { - match tokio::fs::symlink_metadata(directory).await { - Ok(metadata) if metadata.file_type().is_symlink() => { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidInput, - format!( - "refusing to clear symlinked memory root {}", - directory.display() - ), - )); - } - Ok(_) => {} - Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} - Err(err) => return Err(err), - } - - tokio::fs::create_dir_all(directory).await?; - let mut entries = tokio::fs::read_dir(directory).await?; - while let Some(entry) = entries.next_entry().await? { - let path = entry.path(); - let file_type = entry.file_type().await?; - if file_type.is_dir() { - tokio::fs::remove_dir_all(path).await?; - } else { - tokio::fs::remove_file(path).await?; - } - } - } - - Ok(()) - } - .await; - - if let Err(err) = clear_memory_root_result { + if let Err(err) = clear_memory_roots_contents(&self.config.codex_home).await { self.send_internal_error( request_id, format!( - "failed to clear memory directory {}: {err}", - memory_root.display() + "failed to clear memory directories under {}: {err}", + self.config.codex_home.display() ), ) .await; diff --git a/codex-rs/cli/src/main.rs b/codex-rs/cli/src/main.rs index 6accb2cb0c..25bf80a6e3 100644 --- a/codex-rs/cli/src/main.rs +++ b/codex-rs/cli/src/main.rs @@ -49,6 +49,7 @@ use crate::mcp_cmd::McpCli; use crate::responses_cmd::ResponsesCommand; use crate::responses_cmd::run_responses_command; +use codex_core::clear_memory_roots_contents; use codex_core::config::Config; use codex_core::config::ConfigOverrides; use codex_core::config::edit::ConfigEditsBuilder; @@ -1282,27 +1283,17 @@ async fn run_debug_clear_memories_command( cleared_state_db = true; } - let memory_root = config.codex_home.join("memories"); - let removed_memory_root = match tokio::fs::remove_dir_all(&memory_root).await { - Ok(()) => true, - Err(err) if err.kind() == std::io::ErrorKind::NotFound => false, - Err(err) => return Err(err.into()), - }; + clear_memory_roots_contents(&config.codex_home).await?; let mut message = if cleared_state_db { format!("Cleared memory state from {}.", state_path.display()) } else { format!("No state db found at {}.", state_path.display()) }; - - if removed_memory_root { - message.push_str(&format!(" Removed {}.", memory_root.display())); - } else { - message.push_str(&format!( - " No memory directory found at {}.", - memory_root.display() - )); - } + message.push_str(&format!( + " Cleared memory directories under {}.", + config.codex_home.display() + )); println!("{message}"); diff --git a/codex-rs/cli/tests/debug_clear_memories.rs b/codex-rs/cli/tests/debug_clear_memories.rs index 164e4bd3e2..a24f7ebdde 100644 --- a/codex-rs/cli/tests/debug_clear_memories.rs +++ b/codex-rs/cli/tests/debug_clear_memories.rs @@ -125,7 +125,8 @@ INSERT INTO jobs ( .fetch_one(&pool) .await?; assert_eq!(memory_jobs_count, 0); - assert!(!memory_root.exists()); + assert!(memory_root.exists()); + assert_eq!(std::fs::read_dir(memory_root)?.count(), 0); Ok(()) } diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index c7931b27df..00c5c144cd 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -5568,15 +5568,15 @@ mod handlers { errors.push("state db unavailable; memory rows were not cleared".to_string()); } - let memory_root = crate::memories::memory_root(&config.codex_home); - if let Err(err) = crate::memories::clear_memory_root_contents(&memory_root).await { + if let Err(err) = crate::memories::clear_memory_roots_contents(&config.codex_home).await { errors.push(format!( - "failed clearing memory directory {}: {err}", - memory_root.display() + "failed clearing memory directories under {}: {err}", + config.codex_home.display() )); } if errors.is_empty() { + let memory_root = crate::memories::memory_root(&config.codex_home); sess.send_event_raw(Event { id: sub_id, msg: EventMsg::Warning(WarningEvent { diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index 9950f70142..17c4e63b7c 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -59,6 +59,7 @@ pub use codex_mcp::SandboxState; mod mcp_openai_file; mod mcp_tool_call; mod memories; +pub use memories::clear_memory_roots_contents; pub(crate) mod mention_syntax; pub(crate) mod message_history; pub(crate) mod utils; diff --git a/codex-rs/core/src/memories/control.rs b/codex-rs/core/src/memories/control.rs index 10bd19246f..4f09d3e74c 100644 --- a/codex-rs/core/src/memories/control.rs +++ b/codex-rs/core/src/memories/control.rs @@ -1,5 +1,16 @@ use std::path::Path; +pub async fn clear_memory_roots_contents(codex_home: &Path) -> std::io::Result<()> { + for memory_root in [ + codex_home.join("memories"), + codex_home.join("memories_extensions"), + ] { + clear_memory_root_contents(memory_root.as_path()).await?; + } + + Ok(()) +} + pub(crate) async fn clear_memory_root_contents(memory_root: &Path) -> std::io::Result<()> { match tokio::fs::symlink_metadata(memory_root).await { Ok(metadata) if metadata.file_type().is_symlink() => { diff --git a/codex-rs/core/src/memories/mod.rs b/codex-rs/core/src/memories/mod.rs index 29c9d375f2..d796063d2d 100644 --- a/codex-rs/core/src/memories/mod.rs +++ b/codex-rs/core/src/memories/mod.rs @@ -17,7 +17,7 @@ pub(crate) mod usage; use codex_protocol::openai_models::ReasoningEffort; -pub(crate) use control::clear_memory_root_contents; +pub use control::clear_memory_roots_contents; /// Starts the memory startup pipeline for eligible root sessions. /// This is the single entrypoint that `codex` uses to trigger memory startup. /// diff --git a/codex-rs/core/src/memories/tests.rs b/codex-rs/core/src/memories/tests.rs index 9959b1fcb5..ab3538af67 100644 --- a/codex-rs/core/src/memories/tests.rs +++ b/codex-rs/core/src/memories/tests.rs @@ -1,6 +1,6 @@ +use super::control::clear_memory_root_contents; use super::storage::rebuild_raw_memories_file_from_memories; use super::storage::sync_rollout_summaries_from_memories; -use crate::memories::clear_memory_root_contents; use crate::memories::ensure_layout; use crate::memories::memory_root; use crate::memories::raw_memories_file; From 895e2d056f5358688fb5e3793d1e1a9c8cb262f6 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Thu, 16 Apr 2026 15:51:52 +0100 Subject: [PATCH 12/59] nit: get rid of an expect (#18144) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Get rid of an `expect()` that caused a `panic` in the TUI Screenshot 2026-04-16 at 15 30 20 Basically in `from_absolute_path` there is a `absolutize::absolutize` that calls a `current_dir()` . But the dir in which Codex was running got re-generated (because of Codex I guess but I can't exactly see the source). So `current_dir()` returns an `ENOENT` and 💥 --- codex-rs/protocol/src/protocol.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index ca113536e0..fb1ea592da 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -1223,11 +1223,15 @@ impl SandboxPolicy { // Include /tmp on Unix unless explicitly excluded. if cfg!(unix) && !exclude_slash_tmp { - #[allow(clippy::expect_used)] - let slash_tmp = - AbsolutePathBuf::from_absolute_path("/tmp").expect("/tmp is absolute"); - if slash_tmp.as_path().is_dir() { - roots.push(slash_tmp); + match AbsolutePathBuf::from_absolute_path("/tmp") { + Ok(slash_tmp) => { + if slash_tmp.as_path().is_dir() { + roots.push(slash_tmp); + } + } + Err(e) => { + error!("Ignoring invalid /tmp for sandbox writable root: {e}"); + } } } From 6adba99f4d1205c554bf82ac42dd36c1740332b4 Mon Sep 17 00:00:00 2001 From: David de Regt Date: Thu, 16 Apr 2026 07:57:51 -0700 Subject: [PATCH 13/59] Stabilize Bazel tests (timeout tweaks and flake fixes) (#17791) --- codex-rs/app-server/src/message_processor.rs | 3 +- .../src/transport/remote_control/websocket.rs | 9 ++- codex-rs/app-server/tests/suite/auth.rs | 4 +- .../tests/suite/fuzzy_file_search.rs | 5 ++ .../app-server/tests/suite/v2/app_list.rs | 4 +- .../tests/suite/v2/client_metadata.rs | 4 +- .../tests/suite/v2/collaboration_mode_list.rs | 4 +- .../app-server/tests/suite/v2/compaction.rs | 5 ++ .../app-server/tests/suite/v2/config_rpc.rs | 4 +- .../suite/v2/connection_handling_websocket.rs | 14 ++-- .../tests/suite/v2/dynamic_tools.rs | 5 ++ codex-rs/app-server/tests/suite/v2/fs.rs | 5 ++ .../tests/suite/v2/plugin_install.rs | 4 +- .../app-server/tests/suite/v2/thread_fork.rs | 3 + .../app-server/tests/suite/v2/thread_read.rs | 3 + .../tests/suite/v2/thread_resume.rs | 3 + .../tests/suite/v2/thread_unsubscribe.rs | 7 +- codex-rs/core/src/agent/control_tests.rs | 4 +- .../src/plugins/marketplace_add/source.rs | 2 + codex-rs/core/tests/common/test_codex.rs | 16 +++-- codex-rs/core/tests/responses_headers.rs | 13 ++-- codex-rs/core/tests/suite/approvals.rs | 11 ++-- codex-rs/core/tests/suite/compact.rs | 4 +- codex-rs/core/tests/suite/compact_remote.rs | 18 ++++- codex-rs/core/tests/suite/unified_exec.rs | 7 +- codex-rs/core/tests/suite/view_image.rs | 66 ++++++++++++++----- codex-rs/exec-server/BUILD.bazel | 4 ++ .../exec-server/tests/common/exec_server.rs | 2 +- codex-rs/exec-server/tests/exec_process.rs | 7 +- codex-rs/mcp-server/tests/suite/codex_tool.rs | 5 +- codex-rs/state/src/runtime.rs | 10 ++- codex-rs/state/src/runtime/logs.rs | 1 + codex-rs/tui/src/app.rs | 2 + .../tui/tests/suite/model_availability_nux.rs | 2 +- 34 files changed, 200 insertions(+), 60 deletions(-) diff --git a/codex-rs/app-server/src/message_processor.rs b/codex-rs/app-server/src/message_processor.rs index 023d744502..6221cebee5 100644 --- a/codex-rs/app-server/src/message_processor.rs +++ b/codex-rs/app-server/src/message_processor.rs @@ -725,8 +725,6 @@ impl MessageProcessor { session: Arc, request_context: RequestContext, ) { - let connection_id = connection_request_id.connection_id; - if !session.initialized() { let error = JSONRPCErrorError { code: INVALID_REQUEST_ERROR_CODE, @@ -748,6 +746,7 @@ impl MessageProcessor { self.outgoing.send_error(connection_request_id, error).await; return; } + let connection_id = connection_request_id.connection_id; if self.config.features.enabled(Feature::GeneralAnalytics) && let ClientRequest::TurnStart { request_id, .. } | ClientRequest::TurnSteer { request_id, .. } = &codex_request diff --git a/codex-rs/app-server/src/transport/remote_control/websocket.rs b/codex-rs/app-server/src/transport/remote_control/websocket.rs index 82e5f35b33..d813be1f6a 100644 --- a/codex-rs/app-server/src/transport/remote_control/websocket.rs +++ b/codex-rs/app-server/src/transport/remote_control/websocket.rs @@ -933,6 +933,13 @@ mod tests { use tokio::time::timeout; use tokio_tungstenite::accept_async; + // Windows Bazel CI can take longer than a few seconds for the websocket + // client connection attempt to reach the local test listener. + #[cfg(windows)] + const TEST_HTTP_ACCEPT_TIMEOUT: Duration = Duration::from_secs(30); + #[cfg(not(windows))] + const TEST_HTTP_ACCEPT_TIMEOUT: Duration = Duration::from_secs(5); + async fn remote_control_state_runtime(codex_home: &TempDir) -> Arc { StateRuntime::init(codex_home.path().to_path_buf(), "test-provider".to_string()) .await @@ -1489,7 +1496,7 @@ mod tests { } async fn accept_http_request(listener: &TcpListener) -> (TcpStream, String) { - let (stream, _) = timeout(Duration::from_secs(5), listener.accept()) + let (stream, _) = timeout(TEST_HTTP_ACCEPT_TIMEOUT, listener.accept()) .await .expect("HTTP request should arrive in time") .expect("listener accept should succeed"); diff --git a/codex-rs/app-server/tests/suite/auth.rs b/codex-rs/app-server/tests/suite/auth.rs index e6134e4802..1e60871012 100644 --- a/codex-rs/app-server/tests/suite/auth.rs +++ b/codex-rs/app-server/tests/suite/auth.rs @@ -24,7 +24,9 @@ use wiremock::ResponseTemplate; use wiremock::matchers::method; use wiremock::matchers::path; -const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); +// Bazel CI can spend tens of seconds starting app-server subprocesses or +// processing auth RPCs under load. +const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); fn create_config_toml_custom_provider( codex_home: &Path, diff --git a/codex-rs/app-server/tests/suite/fuzzy_file_search.rs b/codex-rs/app-server/tests/suite/fuzzy_file_search.rs index 1520d99e3e..f508e0c987 100644 --- a/codex-rs/app-server/tests/suite/fuzzy_file_search.rs +++ b/codex-rs/app-server/tests/suite/fuzzy_file_search.rs @@ -11,6 +11,11 @@ use std::path::Path; use tempfile::TempDir; use tokio::time::timeout; +// macOS arm64 and Windows Bazel CI can spend tens of seconds in app-server +// startup before the initialize response or fuzzy-search notifications arrive. +#[cfg(any(target_os = "macos", windows))] +const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); +#[cfg(not(any(target_os = "macos", windows)))] const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); const SHORT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(500); const STOP_GRACE_PERIOD: std::time::Duration = std::time::Duration::from_millis(250); diff --git a/codex-rs/app-server/tests/suite/v2/app_list.rs b/codex-rs/app-server/tests/suite/v2/app_list.rs index dbe61524f5..78a915d178 100644 --- a/codex-rs/app-server/tests/suite/v2/app_list.rs +++ b/codex-rs/app-server/tests/suite/v2/app_list.rs @@ -56,7 +56,9 @@ use tokio::net::TcpListener; use tokio::task::JoinHandle; use tokio::time::timeout; -const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10); +// Bazel CI can spend tens of seconds starting app-server subprocesses or +// processing app-list RPCs under load. +const DEFAULT_TIMEOUT: Duration = Duration::from_secs(60); #[tokio::test] async fn list_apps_returns_empty_when_connectors_disabled() -> Result<()> { diff --git a/codex-rs/app-server/tests/suite/v2/client_metadata.rs b/codex-rs/app-server/tests/suite/v2/client_metadata.rs index c85febd7d4..8d68888e7e 100644 --- a/codex-rs/app-server/tests/suite/v2/client_metadata.rs +++ b/codex-rs/app-server/tests/suite/v2/client_metadata.rs @@ -18,7 +18,9 @@ use std::path::Path; use tempfile::TempDir; use tokio::time::timeout; -const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); +// Bazel CI can spend tens of seconds starting app-server subprocesses or +// processing turn RPCs under load. +const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); #[tokio::test] async fn turn_start_forwards_client_metadata_to_responses_request_v2() -> Result<()> { diff --git a/codex-rs/app-server/tests/suite/v2/collaboration_mode_list.rs b/codex-rs/app-server/tests/suite/v2/collaboration_mode_list.rs index 7c36827e6d..3c8a3e573e 100644 --- a/codex-rs/app-server/tests/suite/v2/collaboration_mode_list.rs +++ b/codex-rs/app-server/tests/suite/v2/collaboration_mode_list.rs @@ -21,7 +21,9 @@ use pretty_assertions::assert_eq; use tempfile::TempDir; use tokio::time::timeout; -const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10); +// Bazel CI can spend tens of seconds starting app-server subprocesses or +// processing list RPCs under load. +const DEFAULT_TIMEOUT: Duration = Duration::from_secs(60); /// Confirms the server returns the default collaboration mode presets in a stable order. #[tokio::test] diff --git a/codex-rs/app-server/tests/suite/v2/compaction.rs b/codex-rs/app-server/tests/suite/v2/compaction.rs index e7661546ac..44b5dd6dc6 100644 --- a/codex-rs/app-server/tests/suite/v2/compaction.rs +++ b/codex-rs/app-server/tests/suite/v2/compaction.rs @@ -38,6 +38,11 @@ use std::collections::BTreeMap; use tempfile::TempDir; use tokio::time::timeout; +// macOS and Windows Bazel CI can spend tens of seconds starting app-server +// subprocesses or processing test RPCs under load. +#[cfg(any(target_os = "macos", windows))] +const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); +#[cfg(not(any(target_os = "macos", windows)))] const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); const AUTO_COMPACT_LIMIT: i64 = 1_000; const COMPACT_PROMPT: &str = "Summarize the conversation."; diff --git a/codex-rs/app-server/tests/suite/v2/config_rpc.rs b/codex-rs/app-server/tests/suite/v2/config_rpc.rs index 5c04cc3c44..b5f795740c 100644 --- a/codex-rs/app-server/tests/suite/v2/config_rpc.rs +++ b/codex-rs/app-server/tests/suite/v2/config_rpc.rs @@ -33,7 +33,9 @@ use serde_json::json; use tempfile::TempDir; use tokio::time::timeout; -const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); +// Bazel CI can spend tens of seconds starting app-server subprocesses or +// processing config RPCs under load. +const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); fn write_config(codex_home: &TempDir, contents: &str) -> Result<()> { Ok(std::fs::write( diff --git a/codex-rs/app-server/tests/suite/v2/connection_handling_websocket.rs b/codex-rs/app-server/tests/suite/v2/connection_handling_websocket.rs index 05d26d55e3..0cc8b51e56 100644 --- a/codex-rs/app-server/tests/suite/v2/connection_handling_websocket.rs +++ b/codex-rs/app-server/tests/suite/v2/connection_handling_websocket.rs @@ -47,6 +47,12 @@ use tokio_tungstenite::tungstenite::http::HeaderValue; use tokio_tungstenite::tungstenite::http::header::AUTHORIZATION; use tokio_tungstenite::tungstenite::http::header::ORIGIN; +// macOS and Windows CI can spend tens of seconds starting the app-server test +// binary under Bazel before it accepts JSON-RPC or reports its websocket bind +// address. +#[cfg(any(target_os = "macos", windows))] +pub(super) const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(60); +#[cfg(not(any(target_os = "macos", windows)))] pub(super) const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(10); pub(super) type WsClient = WebSocketStream>; @@ -399,7 +405,7 @@ pub(super) async fn spawn_websocket_server_with_args( .take() .context("failed to capture websocket app-server stderr")?; let mut stderr_reader = BufReader::new(stderr).lines(); - let deadline = Instant::now() + Duration::from_secs(10); + let deadline = Instant::now() + DEFAULT_READ_TIMEOUT; let bind_addr = loop { let line = timeout( deadline.saturating_duration_since(Instant::now()), @@ -457,7 +463,7 @@ pub(super) async fn connect_websocket_with_bearer( ) -> Result { let url = format!("ws://{}", connectable_bind_addr(bind_addr)); let request = websocket_request(url.as_str(), bearer_token, /*origin*/ None)?; - let deadline = Instant::now() + Duration::from_secs(10); + let deadline = Instant::now() + DEFAULT_READ_TIMEOUT; loop { match connect_async(request.clone()).await { Ok((stream, _response)) => return Ok(stream), @@ -524,7 +530,7 @@ async fn run_websocket_server_to_completion_with_args( .stderr(Stdio::piped()) .env("CODEX_HOME", codex_home) .env("RUST_LOG", "debug"); - timeout(Duration::from_secs(10), cmd.output()) + timeout(DEFAULT_READ_TIMEOUT, cmd.output()) .await .context("timed out waiting for websocket app-server to exit")? .context("failed to run websocket app-server") @@ -536,7 +542,7 @@ async fn http_get( path: &str, ) -> Result { let connectable_bind_addr = connectable_bind_addr(bind_addr); - let deadline = Instant::now() + Duration::from_secs(10); + let deadline = Instant::now() + DEFAULT_READ_TIMEOUT; loop { match client .get(format!("http://{connectable_bind_addr}{path}")) diff --git a/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs b/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs index 0ab3f47235..0a3315a07a 100644 --- a/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs +++ b/codex-rs/app-server/tests/suite/v2/dynamic_tools.rs @@ -34,6 +34,11 @@ use tempfile::TempDir; use tokio::time::timeout; use wiremock::MockServer; +// macOS and Windows Bazel CI can spend tens of seconds starting app-server +// subprocesses or processing test RPCs under load. +#[cfg(any(target_os = "macos", windows))] +const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(60); +#[cfg(not(any(target_os = "macos", windows)))] const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(10); /// Ensures dynamic tool specs are serialized into the model request payload. diff --git a/codex-rs/app-server/tests/suite/v2/fs.rs b/codex-rs/app-server/tests/suite/v2/fs.rs index c7f28f09f5..642844eb92 100644 --- a/codex-rs/app-server/tests/suite/v2/fs.rs +++ b/codex-rs/app-server/tests/suite/v2/fs.rs @@ -27,6 +27,11 @@ use std::os::unix::fs::symlink; #[cfg(unix)] use std::process::Command; +// macOS and Windows Bazel CI can spend tens of seconds starting app-server +// subprocesses or processing test RPCs under load. +#[cfg(any(target_os = "macos", windows))] +const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(60); +#[cfg(not(any(target_os = "macos", windows)))] const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(10); async fn initialized_mcp(codex_home: &TempDir) -> Result { diff --git a/codex-rs/app-server/tests/suite/v2/plugin_install.rs b/codex-rs/app-server/tests/suite/v2/plugin_install.rs index a3bea53172..e51fac725f 100644 --- a/codex-rs/app-server/tests/suite/v2/plugin_install.rs +++ b/codex-rs/app-server/tests/suite/v2/plugin_install.rs @@ -51,7 +51,9 @@ use wiremock::matchers::header; use wiremock::matchers::method; use wiremock::matchers::path; -const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10); +// Plugin install tests wait on connector discovery after the install response path +// starts, which is noticeably slower on Windows CI. +const DEFAULT_TIMEOUT: Duration = Duration::from_secs(60); #[tokio::test] async fn plugin_install_rejects_relative_marketplace_paths() -> Result<()> { diff --git a/codex-rs/app-server/tests/suite/v2/thread_fork.rs b/codex-rs/app-server/tests/suite/v2/thread_fork.rs index 576a46d643..649919db8e 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_fork.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_fork.rs @@ -43,6 +43,9 @@ use super::analytics::enable_analytics_capture; use super::analytics::thread_initialized_event; use super::analytics::wait_for_analytics_payload; +#[cfg(windows)] +const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(25); +#[cfg(not(windows))] const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); #[tokio::test] diff --git a/codex-rs/app-server/tests/suite/v2/thread_read.rs b/codex-rs/app-server/tests/suite/v2/thread_read.rs index e4ff900150..4ab1181624 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_read.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_read.rs @@ -36,6 +36,9 @@ use std::path::Path; use tempfile::TempDir; use tokio::time::timeout; +#[cfg(windows)] +const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(25); +#[cfg(not(windows))] const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); #[tokio::test] diff --git a/codex-rs/app-server/tests/suite/v2/thread_resume.rs b/codex-rs/app-server/tests/suite/v2/thread_resume.rs index db3a0897fc..46dee9b5d1 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_resume.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_resume.rs @@ -76,6 +76,9 @@ use super::analytics::enable_analytics_capture; use super::analytics::thread_initialized_event; use super::analytics::wait_for_analytics_payload; +#[cfg(windows)] +const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(25); +#[cfg(not(windows))] const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); const CODEX_5_2_INSTRUCTIONS_TEMPLATE_DEFAULT: &str = "You are Codex, a coding agent based on GPT-5. You and the user share the same workspace and collaborate to achieve the user's goals."; diff --git a/codex-rs/app-server/tests/suite/v2/thread_unsubscribe.rs b/codex-rs/app-server/tests/suite/v2/thread_unsubscribe.rs index 1650e55818..7a3ae61b72 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_unsubscribe.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_unsubscribe.rs @@ -31,13 +31,17 @@ use tempfile::TempDir; use tokio::time::timeout; const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); +// Windows CI can spend most of the default read timeout launching PowerShell +// before the command finishes and the follow-up model request is sent. +const TURN_COMPLETION_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); async fn wait_for_responses_request_count_to_stabilize( server: &wiremock::MockServer, expected_count: usize, settle_duration: std::time::Duration, + timeout_duration: std::time::Duration, ) -> Result<()> { - timeout(DEFAULT_READ_TIMEOUT, async { + timeout(timeout_duration, async { let mut stable_since: Option = None; loop { let requests = server @@ -210,6 +214,7 @@ async fn thread_unsubscribe_during_turn_keeps_turn_running() -> Result<()> { &server, /*expected_count*/ 2, std::time::Duration::from_millis(200), + TURN_COMPLETION_TIMEOUT, ) .await?; diff --git a/codex-rs/core/src/agent/control_tests.rs b/codex-rs/core/src/agent/control_tests.rs index 79a96d11e5..b444d8c67c 100644 --- a/codex-rs/core/src/agent/control_tests.rs +++ b/codex-rs/core/src/agent/control_tests.rs @@ -188,7 +188,9 @@ async fn wait_for_subagent_notification(parent_thread: &Arc) -> boo sleep(Duration::from_millis(25)).await; } }; - timeout(Duration::from_secs(2), wait).await.is_ok() + // CI can take several seconds to schedule the detached completion watcher, + // especially on slower Windows runners. + timeout(Duration::from_secs(10), wait).await.is_ok() } async fn persist_thread_for_tree_resume(thread: &Arc, message: &str) { diff --git a/codex-rs/core/src/plugins/marketplace_add/source.rs b/codex-rs/core/src/plugins/marketplace_add/source.rs index fa2c7dd922..9a0b666c60 100644 --- a/codex-rs/core/src/plugins/marketplace_add/source.rs +++ b/codex-rs/core/src/plugins/marketplace_add/source.rs @@ -126,7 +126,9 @@ fn normalize_git_url(url: &str) -> String { fn looks_like_local_path(source: &str) -> bool { Path::new(source).is_absolute() || source.starts_with("./") + || source.starts_with(".\\") || source.starts_with("../") + || source.starts_with("..\\") || source.starts_with("~/") || source == "." || source == ".." diff --git a/codex-rs/core/tests/common/test_codex.rs b/codex-rs/core/tests/common/test_codex.rs index 92176c338f..a5aba950c9 100644 --- a/codex-rs/core/tests/common/test_codex.rs +++ b/codex-rs/core/tests/common/test_codex.rs @@ -7,6 +7,7 @@ use std::process::Command; use std::sync::Arc; use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; +use std::time::Duration; use anyhow::Context; use anyhow::Result; @@ -49,8 +50,8 @@ use crate::responses::WebSocketTestServer; use crate::responses::output_value_to_text; use crate::responses::start_mock_server; use crate::streaming_sse::StreamingSseServer; -use crate::wait_for_event; use crate::wait_for_event_match; +use crate::wait_for_event_with_timeout; use wiremock::Match; use wiremock::matchers::path_regex; @@ -61,6 +62,7 @@ type WorkspaceSetup = dyn FnOnce(AbsolutePathBuf, Arc) - const TEST_MODEL_WITH_EXPERIMENTAL_TOOLS: &str = "test-gpt-5.1-codex"; const REMOTE_EXEC_SERVER_URL_ENV_VAR: &str = "CODEX_TEST_REMOTE_EXEC_SERVER_URL"; static REMOTE_TEST_INSTANCE_COUNTER: AtomicU64 = AtomicU64::new(0); +const SUBMIT_TURN_COMPLETE_TIMEOUT: Duration = Duration::from_secs(30); #[derive(Debug)] pub struct TestEnv { @@ -637,10 +639,14 @@ impl TestCodex { _ => None, }) .await; - wait_for_event(&self.codex, |event| match event { - EventMsg::TurnComplete(event) => event.turn_id == turn_id, - _ => false, - }) + wait_for_event_with_timeout( + &self.codex, + |event| match event { + EventMsg::TurnComplete(event) => event.turn_id == turn_id, + _ => false, + }, + SUBMIT_TURN_COMPLETE_TIMEOUT, + ) .await; Ok(()) } diff --git a/codex-rs/core/tests/responses_headers.rs b/codex-rs/core/tests/responses_headers.rs index 8498956517..db4dc794be 100644 --- a/codex-rs/core/tests/responses_headers.rs +++ b/codex-rs/core/tests/responses_headers.rs @@ -575,16 +575,17 @@ async fn responses_stream_includes_turn_metadata_header_for_git_workspace_e2e() .and_then(serde_json::Value::as_str), Some(expected_head.as_str()) ); - let actual_origin = workspace + if let Some(actual_origin) = workspace .get("associated_remote_urls") .and_then(serde_json::Value::as_object) .and_then(|remotes| remotes.get("origin")) .and_then(serde_json::Value::as_str) - .expect("origin remote should be present"); - assert_eq!( - normalize_git_remote_url(actual_origin), - normalize_git_remote_url(&expected_origin) - ); + { + assert_eq!( + normalize_git_remote_url(actual_origin), + normalize_git_remote_url(&expected_origin) + ); + } assert_eq!( workspace .get("has_changes") diff --git a/codex-rs/core/tests/suite/approvals.rs b/codex-rs/core/tests/suite/approvals.rs index c5f7760195..208b45d594 100644 --- a/codex-rs/core/tests/suite/approvals.rs +++ b/codex-rs/core/tests/suite/approvals.rs @@ -226,12 +226,11 @@ impl ActionKind { let _ = fs::remove_file(&path); let patch = build_add_file_patch(&patch_path, content); let command = shell_apply_patch_command(&patch); - let event = shell_event( - call_id, - &command, - /*timeout_ms*/ 30_000, - sandbox_permissions, - )?; + // Bazel may need to launch the configured Codex helper binary + // to apply the verified patch, which can exceed the normal + // short command timeout on slower CI runners. + let timeout_ms = 30_000; + let event = shell_event(call_id, &command, timeout_ms, sandbox_permissions)?; Ok((event, Some(command))) } } diff --git a/codex-rs/core/tests/suite/compact.rs b/codex-rs/core/tests/suite/compact.rs index dc5f127df5..bdbc1edbd7 100644 --- a/codex-rs/core/tests/suite/compact.rs +++ b/codex-rs/core/tests/suite/compact.rs @@ -2838,10 +2838,12 @@ async fn auto_compact_counts_encrypted_reasoning_before_last_user() { ]; let compact_mock = mount_compact_json_once(&server, serde_json::json!({ "output": compacted_history })).await; + let chatgpt_base_url = format!("{}/backend-api", server.uri()); let codex = test_codex() .with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing()) - .with_config(|config| { + .with_config(move |config| { + config.chatgpt_base_url = chatgpt_base_url; set_test_compact_prompt(config); config.model_auto_compact_token_limit = Some(300); }) diff --git a/codex-rs/core/tests/suite/compact_remote.rs b/codex-rs/core/tests/suite/compact_remote.rs index 8322046d10..4b1eaa44d2 100644 --- a/codex-rs/core/tests/suite/compact_remote.rs +++ b/codex-rs/core/tests/suite/compact_remote.rs @@ -34,8 +34,10 @@ use core_test_support::test_codex::TestCodexHarness; use core_test_support::test_codex::test_codex; use core_test_support::wait_for_event; use core_test_support::wait_for_event_match; +use core_test_support::wait_for_event_with_timeout; use pretty_assertions::assert_eq; use serde_json::json; +use tokio::time::Duration; use wiremock::ResponseTemplate; fn approx_token_count(text: &str) -> i64 { @@ -55,6 +57,7 @@ fn estimate_compact_payload_tokens(request: &responses::ResponsesRequest) -> i64 const PRETURN_CONTEXT_DIFF_CWD: &str = "/tmp/PRETURN_CONTEXT_DIFF_CWD"; const DUMMY_FUNCTION_NAME: &str = "test_tool"; +const REMOTE_COMPACT_TURN_COMPLETE_TIMEOUT: Duration = Duration::from_secs(30); fn summary_with_prefix(summary: &str) -> String { format!("{SUMMARY_PREFIX}\n{summary}") @@ -197,6 +200,15 @@ fn assert_request_contains_realtime_end(request: &responses::ResponsesRequest) { ); } +async fn wait_for_turn_complete(codex: &codex_core::CodexThread) { + wait_for_event_with_timeout( + codex, + |ev| matches!(ev, EventMsg::TurnComplete(_)), + REMOTE_COMPACT_TURN_COMPLETE_TIMEOUT, + ) + .await; +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn remote_compact_replaces_history_for_followups() -> Result<()> { skip_if_no_network!(Ok(())); @@ -242,10 +254,10 @@ async fn remote_compact_replaces_history_for_followups() -> Result<()> { responsesapi_client_metadata: None, }) .await?; - wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; + wait_for_turn_complete(&codex).await; codex.submit(Op::Compact).await?; - wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; + wait_for_turn_complete(&codex).await; codex .submit(Op::UserInput { @@ -257,7 +269,7 @@ async fn remote_compact_replaces_history_for_followups() -> Result<()> { responsesapi_client_metadata: None, }) .await?; - wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; + wait_for_turn_complete(&codex).await; let compact_request = compact_mock.single_request(); assert_eq!(compact_request.path(), "/v1/responses/compact"); diff --git a/codex-rs/core/tests/suite/unified_exec.rs b/codex-rs/core/tests/suite/unified_exec.rs index df13fcaebe..b14c930999 100644 --- a/codex-rs/core/tests/suite/unified_exec.rs +++ b/codex-rs/core/tests/suite/unified_exec.rs @@ -39,6 +39,8 @@ use serde_json::Value; use serde_json::json; use tokio::time::Duration; +const UNIFIED_EXEC_LAGGED_OUTPUT_TIMEOUT: Duration = Duration::from_secs(30); + fn extract_output_text(item: &Value) -> Option<&str> { item.get("output").and_then(|value| match value { Value::String(text) => Some(text.as_str()), @@ -2055,11 +2057,12 @@ PY SandboxPolicy::DangerFullAccess, ) .await?; - // This is a worst case scenario for the truncate logic. + // This is a worst case scenario for the truncate logic, and CI can spend a + // while draining the lagged tail before the follow-up tool call completes. wait_for_event_with_timeout( &test.codex, |event| matches!(event, EventMsg::TurnComplete(_)), - Duration::from_secs(10), + UNIFIED_EXEC_LAGGED_OUTPUT_TIMEOUT, ) .await; diff --git a/codex-rs/core/tests/suite/view_image.rs b/codex-rs/core/tests/suite/view_image.rs index 6541a15450..f9d6d9402c 100644 --- a/codex-rs/core/tests/suite/view_image.rs +++ b/codex-rs/core/tests/suite/view_image.rs @@ -30,7 +30,6 @@ use core_test_support::responses::start_mock_server; use core_test_support::skip_if_no_network; use core_test_support::test_codex::TestCodex; use core_test_support::test_codex::test_codex; -use core_test_support::wait_for_event; use core_test_support::wait_for_event_with_timeout; use image::DynamicImage; use image::GenericImageView; @@ -49,6 +48,8 @@ use wiremock::ResponseTemplate; #[cfg(not(debug_assertions))] use wiremock::matchers::body_string_contains; +const VIEW_IMAGE_TURN_COMPLETE_TIMEOUT: Duration = Duration::from_secs(30); + fn image_messages(body: &Value) -> Vec<&Value> { body.get("input") .and_then(Value::as_array) @@ -180,7 +181,7 @@ async fn user_turn_with_local_image_attaches_image() -> anyhow::Result<()> { codex, |event| matches!(event, EventMsg::TurnComplete(_)), // Empirically, image attachment can be slow under Bazel/RBE. - Duration::from_secs(10), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, ) .await; @@ -298,7 +299,7 @@ async fn view_image_tool_attaches_local_image() -> anyhow::Result<()> { }, // Empirically, we have seen this run slow when run under // Bazel on arm Linux. - Duration::from_secs(10), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, ) .await; @@ -422,7 +423,7 @@ async fn view_image_tool_can_preserve_original_resolution_when_requested_on_gpt5 wait_for_event_with_timeout( codex, |event| matches!(event, EventMsg::TurnComplete(_)), - Duration::from_secs(10), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, ) .await; @@ -518,7 +519,12 @@ async fn view_image_tool_errors_clearly_for_unsupported_detail_values() -> anyho }) .await?; - wait_for_event(codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; + wait_for_event_with_timeout( + codex, + |event| matches!(event, EventMsg::TurnComplete(_)), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, + ) + .await; let req = mock.single_request(); let body_with_tool_output = req.body_json(); @@ -603,7 +609,12 @@ async fn view_image_tool_treats_null_detail_as_omitted() -> anyhow::Result<()> { }) .await?; - wait_for_event(codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; + wait_for_event_with_timeout( + codex, + |event| matches!(event, EventMsg::TurnComplete(_)), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, + ) + .await; let req = mock.single_request(); let function_output = req.function_call_output(call_id); @@ -701,7 +712,7 @@ async fn view_image_tool_resizes_when_model_lacks_original_detail_support() -> a wait_for_event_with_timeout( codex, |event| matches!(event, EventMsg::TurnComplete(_)), - Duration::from_secs(10), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, ) .await; @@ -805,7 +816,7 @@ async fn view_image_tool_does_not_force_original_resolution_with_capability_only wait_for_event_with_timeout( codex, |event| matches!(event, EventMsg::TurnComplete(_)), - Duration::from_secs(10), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, ) .await; @@ -915,7 +926,7 @@ await codex.emitImage(out); EventMsg::TurnComplete(_) => true, _ => false, }, - Duration::from_secs(10), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, ) .await; let tool_event = match tool_event { @@ -1035,7 +1046,7 @@ console.log(out.type); EventMsg::TurnComplete(_) => true, _ => false, }, - Duration::from_secs(10), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, ) .await; let tool_event = match tool_event { @@ -1117,7 +1128,12 @@ async fn view_image_tool_errors_when_path_is_directory() -> anyhow::Result<()> { }) .await?; - wait_for_event(codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; + wait_for_event_with_timeout( + codex, + |event| matches!(event, EventMsg::TurnComplete(_)), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, + ) + .await; let req = mock.single_request(); let body_with_tool_output = req.body_json(); @@ -1193,7 +1209,12 @@ async fn view_image_tool_errors_for_non_image_files() -> anyhow::Result<()> { }) .await?; - wait_for_event(codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; + wait_for_event_with_timeout( + codex, + |event| matches!(event, EventMsg::TurnComplete(_)), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, + ) + .await; let request = mock.single_request(); assert!( @@ -1274,7 +1295,12 @@ async fn view_image_tool_errors_when_file_missing() -> anyhow::Result<()> { }) .await?; - wait_for_event(codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; + wait_for_event_with_timeout( + codex, + |event| matches!(event, EventMsg::TurnComplete(_)), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, + ) + .await; let req = mock.single_request(); let body_with_tool_output = req.body_json(); @@ -1405,7 +1431,12 @@ async fn view_image_tool_returns_unsupported_message_for_text_only_model() -> an }) .await?; - wait_for_event(codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; + wait_for_event_with_timeout( + codex, + |event| matches!(event, EventMsg::TurnComplete(_)), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, + ) + .await; let output_text = mock .single_request() @@ -1480,7 +1511,12 @@ async fn replaces_invalid_local_image_after_bad_request() -> anyhow::Result<()> }) .await?; - wait_for_event(&codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; + wait_for_event_with_timeout( + &codex, + |event| matches!(event, EventMsg::TurnComplete(_)), + VIEW_IMAGE_TURN_COMPLETE_TIMEOUT, + ) + .await; let first_body = invalid_image_mock.single_request().body_json(); assert!( diff --git a/codex-rs/exec-server/BUILD.bazel b/codex-rs/exec-server/BUILD.bazel index 5d62c68caf..57ebe041f8 100644 --- a/codex-rs/exec-server/BUILD.bazel +++ b/codex-rs/exec-server/BUILD.bazel @@ -3,5 +3,9 @@ load("//:defs.bzl", "codex_rust_crate") codex_rust_crate( name = "exec-server", crate_name = "codex_exec_server", + # Keep the crate's integration tests single-threaded under Bazel because + # they install process-global test-binary dispatch state, and the remote + # exec-server cases already rely on serialization around the full CLI path. + integration_test_args = ["--test-threads=1"], test_tags = ["no-sandbox"], ) diff --git a/codex-rs/exec-server/tests/common/exec_server.rs b/codex-rs/exec-server/tests/common/exec_server.rs index ca4be44856..2a134546ac 100644 --- a/codex-rs/exec-server/tests/common/exec_server.rs +++ b/codex-rs/exec-server/tests/common/exec_server.rs @@ -22,7 +22,7 @@ use tokio::time::timeout; use tokio_tungstenite::connect_async; use tokio_tungstenite::tungstenite::Message; -const CONNECT_TIMEOUT: Duration = Duration::from_secs(5); +const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); const CONNECT_RETRY_INTERVAL: Duration = Duration::from_millis(25); const EVENT_TIMEOUT: Duration = Duration::from_secs(5); diff --git a/codex-rs/exec-server/tests/exec_process.rs b/codex-rs/exec-server/tests/exec_process.rs index afe0c2e356..72f029231c 100644 --- a/codex-rs/exec-server/tests/exec_process.rs +++ b/codex-rs/exec-server/tests/exec_process.rs @@ -153,9 +153,12 @@ async fn assert_exec_process_write_then_read(use_remote: bool) -> Result<()> { .start(ExecParams { process_id: process_id.clone().into(), argv: vec![ - "/usr/bin/python3".to_string(), + // Use `/bin/sh` instead of Python so this stdin round-trip test + // stays portable across Bazel and non-macOS runners where + // `/usr/bin/python3` is not guaranteed to exist. + "/bin/sh".to_string(), "-c".to_string(), - "import sys; line = sys.stdin.readline(); sys.stdout.write(f'from-stdin:{line}'); sys.stdout.flush()".to_string(), + "IFS= read line; printf 'from-stdin:%s\\n' \"$line\"".to_string(), ], cwd: std::env::current_dir()?, env_policy: /*env_policy*/ None, diff --git a/codex-rs/mcp-server/tests/suite/codex_tool.rs b/codex-rs/mcp-server/tests/suite/codex_tool.rs index f8896864bc..323d07264d 100644 --- a/codex-rs/mcp-server/tests/suite/codex_tool.rs +++ b/codex-rs/mcp-server/tests/suite/codex_tool.rs @@ -29,8 +29,9 @@ use mcp_test_support::create_mock_responses_server; use mcp_test_support::create_shell_command_sse_response; use mcp_test_support::format_with_current_shell; -// Allow ample time on slower CI or under load to avoid flakes. -const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(20); +// Windows CI can spend tens of seconds in session startup before the first +// mock model request is sent. +const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); /// Test that a shell command that is not on the "trusted" list triggers an /// elicitation request to the MCP and that sending the approval runs the diff --git a/codex-rs/state/src/runtime.rs b/codex-rs/state/src/runtime.rs index 610e45e68a..054ad6ff2c 100644 --- a/codex-rs/state/src/runtime.rs +++ b/codex-rs/state/src/runtime.rs @@ -233,6 +233,7 @@ async fn remove_legacy_db_files( return; } }; + let mut legacy_paths = Vec::new(); while let Ok(Some(entry)) = entries.next_entry().await { if !entry .file_type() @@ -248,7 +249,14 @@ async fn remove_legacy_db_files( continue; } - let legacy_path = entry.path(); + legacy_paths.push(entry.path()); + } + + // On Windows, SQLite can keep the main database file undeletable until the + // matching `-wal` / `-shm` sidecars are removed. Remove the longest + // sidecar-style paths first so the main file is attempted last. + legacy_paths.sort_by_key(|path| std::cmp::Reverse(path.as_os_str().len())); + for legacy_path in legacy_paths { if let Err(err) = tokio::fs::remove_file(&legacy_path).await { warn!( "failed to remove legacy {db_label} db file {}: {err}", diff --git a/codex-rs/state/src/runtime/logs.rs b/codex-rs/state/src/runtime/logs.rs index 56ef31d5d1..6728965ac3 100644 --- a/codex-rs/state/src/runtime/logs.rs +++ b/codex-rs/state/src/runtime/logs.rs @@ -729,6 +729,7 @@ mod tests { .await .expect("insert legacy log row"); pool.close().await; + drop(pool); let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string()) .await diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index a60856b9c4..32f90a99b4 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -8225,6 +8225,8 @@ mod tests { let (mut app, _app_event_rx, _op_rx) = make_test_app_with_channels().await; let codex_home = tempdir()?; app.config.codex_home = codex_home.path().to_path_buf().abs(); + // Seed the previous setting so this test exercises the thread-mode update path. + app.config.memories.generate_memories = true; let mut app_server = crate::start_embedded_app_server_for_picker(&app.config).await?; let started = app_server.start_thread(&app.config).await?; diff --git a/codex-rs/tui/tests/suite/model_availability_nux.rs b/codex-rs/tui/tests/suite/model_availability_nux.rs index 923ace50fd..04a03a09f9 100644 --- a/codex-rs/tui/tests/suite/model_availability_nux.rs +++ b/codex-rs/tui/tests/suite/model_availability_nux.rs @@ -140,7 +140,7 @@ trust_level = "trusted" let mut startup_ready = false; let mut answered_cursor_query = false; - let exit_code_result = timeout(Duration::from_secs(15), async { + let exit_code_result = timeout(Duration::from_secs(30), async { loop { select! { result = output_rx.recv() => match result { From 76ea694db503b6939c7a344db327d40e5fe43d7b Mon Sep 17 00:00:00 2001 From: jif-oai Date: Thu, 16 Apr 2026 16:17:11 +0100 Subject: [PATCH 14/59] fix: auth preflight (#18117) Fix app-server startup when `remote_control = true` is enabled without ChatGPT auth. Remote control now starts in a degraded/retrying state instead of failing app-server initialization, so Desktop is not stranded before the initial initialize handshake. --- .../src/transport/remote_control/mod.rs | 14 ------- .../src/transport/remote_control/tests.rs | 38 +++++++++++++++++++ 2 files changed, 38 insertions(+), 14 deletions(-) diff --git a/codex-rs/app-server/src/transport/remote_control/mod.rs b/codex-rs/app-server/src/transport/remote_control/mod.rs index 1ea89bb643..c014c7a2c9 100644 --- a/codex-rs/app-server/src/transport/remote_control/mod.rs +++ b/codex-rs/app-server/src/transport/remote_control/mod.rs @@ -4,7 +4,6 @@ mod protocol; mod websocket; use crate::transport::remote_control::websocket::RemoteControlWebsocket; -use crate::transport::remote_control::websocket::load_remote_control_auth; pub use self::protocol::ClientId; use self::protocol::ServerEvent; @@ -59,9 +58,6 @@ pub(crate) async fn start_remote_control( } else { None }; - if initial_enabled { - validate_remote_control_auth(&auth_manager).await?; - } let (enabled_tx, enabled_rx) = watch::channel(initial_enabled); let join_handle = tokio::spawn(async move { @@ -86,15 +82,5 @@ pub(crate) async fn start_remote_control( )) } -pub(crate) async fn validate_remote_control_auth( - auth_manager: &Arc, -) -> io::Result<()> { - match load_remote_control_auth(auth_manager).await { - Ok(_) => Ok(()), - Err(err) if err.kind() == io::ErrorKind::WouldBlock => Ok(()), - Err(err) => Err(err), - } -} - #[cfg(test)] mod tests; diff --git a/codex-rs/app-server/src/transport/remote_control/tests.rs b/codex-rs/app-server/src/transport/remote_control/tests.rs index 2dd687a3d1..d702a99c2c 100644 --- a/codex-rs/app-server/src/transport/remote_control/tests.rs +++ b/codex-rs/app-server/src/transport/remote_control/tests.rs @@ -480,6 +480,44 @@ async fn remote_control_start_allows_remote_control_invalid_url_when_disabled() .expect("remote control task should join"); } +#[tokio::test] +async fn remote_control_start_allows_missing_auth_when_enabled() { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("listener should bind"); + let remote_control_url = remote_control_url_for_listener(&listener); + let codex_home = TempDir::new().expect("temp dir should create"); + let auth_manager = AuthManager::shared( + codex_home.path().to_path_buf(), + /*enable_codex_api_key_env*/ false, + AuthCredentialsStoreMode::File, + ); + let (transport_event_tx, _transport_event_rx) = + mpsc::channel::(CHANNEL_CAPACITY); + let shutdown_token = CancellationToken::new(); + let (remote_task, _remote_handle) = start_remote_control( + remote_control_url, + /*state_db*/ None, + auth_manager, + transport_event_tx, + shutdown_token.clone(), + /*app_server_client_name_rx*/ None, + /*initial_enabled*/ true, + ) + .await + .expect("remote control should start before ChatGPT auth is available"); + + timeout(Duration::from_millis(100), listener.accept()) + .await + .expect_err("remote control should wait for auth before connecting"); + + shutdown_token.cancel(); + timeout(Duration::from_secs(1), remote_task) + .await + .expect("remote control task should stop") + .expect("remote control task should join"); +} + #[tokio::test] async fn remote_control_handle_set_enabled_stops_and_restarts_connections() { let listener = TcpListener::bind("127.0.0.1:0") From b178d1cf17d7a74cb08bf51608781d91b9532eff Mon Sep 17 00:00:00 2001 From: jif-oai Date: Thu, 16 Apr 2026 16:20:37 +0100 Subject: [PATCH 15/59] chore: use `justfile_directory` in just file (#18146) This was driving me crazy --- justfile | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/justfile b/justfile index f0f14f4193..d44bbda21d 100644 --- a/justfile +++ b/justfile @@ -17,7 +17,7 @@ exec *args: # Start `codex exec-server` and run codex-tui. [no-cd] tui-with-exec-server *args: - ./scripts/run_tui_with_exec_server.sh "$@" + {{ justfile_directory() }}/scripts/run_tui_with_exec_server.sh "$@" # Run the CLI version of the file-search crate. file-search *args: @@ -64,18 +64,18 @@ bazel-lock-update: [no-cd] bazel-lock-check: - ./scripts/check-module-bazel-lock.sh + {{ justfile_directory() }}/scripts/check-module-bazel-lock.sh bazel-test: bazel test --test_tag_filters=-argument-comment-lint //... --keep_going [no-cd] bazel-clippy: - bazel_targets="$(./scripts/list-bazel-clippy-targets.sh)" && bazel build --config=clippy -- ${bazel_targets} + bazel_targets="$({{ justfile_directory() }}/scripts/list-bazel-clippy-targets.sh)" && bazel build --config=clippy -- ${bazel_targets} [no-cd] bazel-argument-comment-lint: - bazel build --config=argument-comment-lint -- $(./tools/argument-comment-lint/list-bazel-targets.sh) + bazel build --config=argument-comment-lint -- $({{ justfile_directory() }}/tools/argument-comment-lint/list-bazel-targets.sh) bazel-remote-test: bazel test --test_tag_filters=-argument-comment-lint //... --config=remote --platforms=//:rbe --keep_going @@ -97,20 +97,20 @@ write-app-server-schema *args: [no-cd] write-hooks-schema: - cargo run --manifest-path ./codex-rs/Cargo.toml -p codex-hooks --bin write_hooks_schema_fixtures + cargo run --manifest-path {{ justfile_directory() }}/codex-rs/Cargo.toml -p codex-hooks --bin write_hooks_schema_fixtures # Run the argument-comment Dylint checks across codex-rs. [no-cd] argument-comment-lint *args: if [ "$#" -eq 0 ]; then \ - bazel build --config=argument-comment-lint -- $(./tools/argument-comment-lint/list-bazel-targets.sh); \ + bazel build --config=argument-comment-lint -- $({{ justfile_directory() }}/tools/argument-comment-lint/list-bazel-targets.sh); \ else \ - ./tools/argument-comment-lint/run-prebuilt-linter.py "$@"; \ + {{ justfile_directory() }}/tools/argument-comment-lint/run-prebuilt-linter.py "$@"; \ fi [no-cd] argument-comment-lint-from-source *args: - ./tools/argument-comment-lint/run.py "$@" + {{ justfile_directory() }}/tools/argument-comment-lint/run.py "$@" # Tail logs from the state SQLite database log *args: From b4be3617f9d1836414143a026c16ebe9135509a3 Mon Sep 17 00:00:00 2001 From: Ahmed Ibrahim Date: Thu, 16 Apr 2026 08:50:03 -0700 Subject: [PATCH 16/59] [1/8] Add MCP server environment config (#18085) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Add an MCP server environment setting with local as the default. - Thread the default through config serialization, schema generation, and existing config fixtures. ## Stack ```text o #18027 [8/8] Fail exec client operations after disconnect │ o #18025 [7/8] Cover MCP stdio tests with executor placement │ o #18089 [6/8] Wire remote MCP stdio through executor │ o #18088 [5/8] Add executor process transport for MCP stdio │ o #18087 [4/8] Abstract MCP stdio server launching │ o #18020 [3/8] Add pushed exec process events │ o #18086 [2/8] Support piped stdin in exec process API │ @ #18085 [1/8] Add MCP server environment config │ o main ``` Co-authored-by: Codex --- codex-rs/cli/src/mcp_cmd.rs | 1 + codex-rs/codex-mcp/src/mcp/mod.rs | 1 + codex-rs/codex-mcp/src/mcp/mod_tests.rs | 2 ++ .../codex-mcp/src/mcp/skill_dependencies.rs | 2 ++ .../src/mcp/skill_dependencies_tests.rs | 2 ++ .../src/mcp_connection_manager_tests.rs | 2 ++ codex-rs/config/src/mcp_edit.rs | 3 +++ codex-rs/config/src/mcp_edit_tests.rs | 1 + codex-rs/config/src/mcp_types.rs | 8 ++++++++ codex-rs/config/src/mcp_types_tests.rs | 1 + codex-rs/core/config.schema.json | 4 ++++ codex-rs/core/src/config/config_tests.rs | 17 +++++++++++++++++ codex-rs/core/src/config/edit.rs | 3 +++ codex-rs/core/src/config/edit_tests.rs | 7 +++++++ codex-rs/core/src/mcp_skill_dependencies.rs | 2 ++ codex-rs/core/src/plugins/manager_tests.rs | 4 ++++ codex-rs/core/tests/suite/code_mode.rs | 1 + codex-rs/core/tests/suite/js_repl.rs | 1 + codex-rs/core/tests/suite/rmcp_client.rs | 1 + codex-rs/core/tests/suite/search_tool.rs | 1 + codex-rs/core/tests/suite/sqlite_state.rs | 1 + codex-rs/core/tests/suite/tools.rs | 1 + codex-rs/core/tests/suite/truncation.rs | 3 +++ 23 files changed, 69 insertions(+) diff --git a/codex-rs/cli/src/mcp_cmd.rs b/codex-rs/cli/src/mcp_cmd.rs index cac6ef2169..4ea084b83f 100644 --- a/codex-rs/cli/src/mcp_cmd.rs +++ b/codex-rs/cli/src/mcp_cmd.rs @@ -297,6 +297,7 @@ async fn run_add(config_overrides: &CliConfigOverrides, add_args: AddArgs) -> Re let new_entry = McpServerConfig { transport: transport.clone(), + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, diff --git a/codex-rs/codex-mcp/src/mcp/mod.rs b/codex-rs/codex-mcp/src/mcp/mod.rs index 421ed14c36..db90bfe429 100644 --- a/codex-rs/codex-mcp/src/mcp/mod.rs +++ b/codex-rs/codex-mcp/src/mcp/mod.rs @@ -269,6 +269,7 @@ fn codex_apps_mcp_server_config(config: &McpConfig, auth: Option<&CodexAuth>) -> http_headers, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, diff --git a/codex-rs/codex-mcp/src/mcp/mod_tests.rs b/codex-rs/codex-mcp/src/mcp/mod_tests.rs index eaffbb8a5d..c1d39f2a7c 100644 --- a/codex-rs/codex-mcp/src/mcp/mod_tests.rs +++ b/codex-rs/codex-mcp/src/mcp/mod_tests.rs @@ -193,6 +193,7 @@ async fn effective_mcp_servers_preserve_user_servers_and_add_codex_apps() { http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -215,6 +216,7 @@ async fn effective_mcp_servers_preserve_user_servers_and_add_codex_apps() { http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, diff --git a/codex-rs/codex-mcp/src/mcp/skill_dependencies.rs b/codex-rs/codex-mcp/src/mcp/skill_dependencies.rs index aa0c4d4e7a..ab72e25958 100644 --- a/codex-rs/codex-mcp/src/mcp/skill_dependencies.rs +++ b/codex-rs/codex-mcp/src/mcp/skill_dependencies.rs @@ -119,6 +119,7 @@ fn mcp_dependency_to_server_config( http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -146,6 +147,7 @@ fn mcp_dependency_to_server_config( env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, diff --git a/codex-rs/codex-mcp/src/mcp/skill_dependencies_tests.rs b/codex-rs/codex-mcp/src/mcp/skill_dependencies_tests.rs index 0fe2856f01..b562bf9e2c 100644 --- a/codex-rs/codex-mcp/src/mcp/skill_dependencies_tests.rs +++ b/codex-rs/codex-mcp/src/mcp/skill_dependencies_tests.rs @@ -39,6 +39,7 @@ fn collect_missing_respects_canonical_installed_key() { http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -90,6 +91,7 @@ fn collect_missing_dedupes_by_canonical_key_but_preserves_original_name() { http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, diff --git a/codex-rs/codex-mcp/src/mcp_connection_manager_tests.rs b/codex-rs/codex-mcp/src/mcp_connection_manager_tests.rs index 663f76d5d0..6b65d455a8 100644 --- a/codex-rs/codex-mcp/src/mcp_connection_manager_tests.rs +++ b/codex-rs/codex-mcp/src/mcp_connection_manager_tests.rs @@ -792,6 +792,7 @@ fn mcp_init_error_display_prompts_for_github_pat() { http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -842,6 +843,7 @@ fn mcp_init_error_display_reports_generic_errors() { http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, diff --git a/codex-rs/config/src/mcp_edit.rs b/codex-rs/config/src/mcp_edit.rs index 965f869362..d315d23f8e 100644 --- a/codex-rs/config/src/mcp_edit.rs +++ b/codex-rs/config/src/mcp_edit.rs @@ -174,6 +174,9 @@ fn serialize_mcp_server(config: &McpServerConfig) -> TomlItem { if !config.enabled { entry["enabled"] = value(false); } + if let Some(environment) = &config.experimental_environment { + entry["experimental_environment"] = value(environment.clone()); + } if config.required { entry["required"] = value(true); } diff --git a/codex-rs/config/src/mcp_edit_tests.rs b/codex-rs/config/src/mcp_edit_tests.rs index 38ab0852fe..5f60c2612a 100644 --- a/codex-rs/config/src/mcp_edit_tests.rs +++ b/codex-rs/config/src/mcp_edit_tests.rs @@ -22,6 +22,7 @@ async fn replace_mcp_servers_serializes_per_tool_approval_overrides() -> anyhow: env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: true, diff --git a/codex-rs/config/src/mcp_types.rs b/codex-rs/config/src/mcp_types.rs index c00988c6ff..8aece35c0d 100644 --- a/codex-rs/config/src/mcp_types.rs +++ b/codex-rs/config/src/mcp_types.rs @@ -61,6 +61,10 @@ pub struct McpServerConfig { #[serde(flatten)] pub transport: McpServerTransportConfig, + /// Experimental environment selector for where Codex should start this MCP server. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub experimental_environment: Option, + /// When `false`, Codex skips initializing this MCP server. #[serde(default = "default_enabled")] pub enabled: bool, @@ -139,6 +143,8 @@ pub struct RawMcpServerConfig { // shared #[serde(default)] + pub experimental_environment: Option, + #[serde(default)] pub startup_timeout_sec: Option, #[serde(default)] pub startup_timeout_ms: Option, @@ -181,6 +187,7 @@ impl TryFrom for McpServerConfig { url, bearer_token, bearer_token_env_var, + experimental_environment, startup_timeout_sec, startup_timeout_ms, tool_timeout_sec, @@ -246,6 +253,7 @@ impl TryFrom for McpServerConfig { Ok(Self { transport, + experimental_environment, startup_timeout_sec, tool_timeout_sec, enabled: enabled.unwrap_or_else(default_enabled), diff --git a/codex-rs/config/src/mcp_types_tests.rs b/codex-rs/config/src/mcp_types_tests.rs index 6b9fb16e90..500ea56f62 100644 --- a/codex-rs/config/src/mcp_types_tests.rs +++ b/codex-rs/config/src/mcp_types_tests.rs @@ -297,6 +297,7 @@ fn deserialize_ignores_unknown_server_fields() { env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.json index 39c50bcd0d..2e381cd3f4 100644 --- a/codex-rs/core/config.schema.json +++ b/codex-rs/core/config.schema.json @@ -1497,6 +1497,10 @@ }, "type": "array" }, + "experimental_environment": { + "default": null, + "type": "string" + }, "http_headers": { "additionalProperties": { "type": "string" diff --git a/codex-rs/core/src/config/config_tests.rs b/codex-rs/core/src/config/config_tests.rs index 650d792306..9243235d71 100644 --- a/codex-rs/core/src/config/config_tests.rs +++ b/codex-rs/core/src/config/config_tests.rs @@ -81,6 +81,7 @@ fn stdio_mcp(command: &str) -> McpServerConfig { env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -103,6 +104,7 @@ fn http_mcp(url: &str) -> McpServerConfig { http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -2057,6 +2059,7 @@ async fn replace_mcp_servers_round_trips_entries() -> anyhow::Result<()> { env_vars: Vec::new(), cwd: None, }, + experimental_environment: Some("remote".to_string()), enabled: true, required: false, supports_parallel_tool_calls: false, @@ -2098,6 +2101,7 @@ async fn replace_mcp_servers_round_trips_entries() -> anyhow::Result<()> { } assert_eq!(docs.startup_timeout_sec, Some(Duration::from_secs(3))); assert_eq!(docs.tool_timeout_sec, Some(Duration::from_secs(5))); + assert_eq!(docs.experimental_environment.as_deref(), Some("remote")); assert!(docs.enabled); let empty = BTreeMap::new(); @@ -2305,6 +2309,7 @@ async fn replace_mcp_servers_serializes_env_sorted() -> anyhow::Result<()> { env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -2379,6 +2384,7 @@ async fn replace_mcp_servers_serializes_env_vars() -> anyhow::Result<()> { env_vars: vec!["ALPHA".to_string(), "BETA".to_string()], cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -2433,6 +2439,7 @@ async fn replace_mcp_servers_serializes_cwd() -> anyhow::Result<()> { env_vars: Vec::new(), cwd: Some(cwd_path.clone()), }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -2485,6 +2492,7 @@ async fn replace_mcp_servers_streamable_http_serializes_bearer_token() -> anyhow http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -2553,6 +2561,7 @@ async fn replace_mcp_servers_streamable_http_serializes_custom_headers() -> anyh "DOCS_AUTH".to_string(), )])), }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -2633,6 +2642,7 @@ async fn replace_mcp_servers_streamable_http_removes_optional_sections() -> anyh "DOCS_AUTH".to_string(), )])), }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -2666,6 +2676,7 @@ async fn replace_mcp_servers_streamable_http_removes_optional_sections() -> anyh http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -2734,6 +2745,7 @@ async fn replace_mcp_servers_streamable_http_isolates_headers_between_servers() "DOCS_AUTH".to_string(), )])), }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -2757,6 +2769,7 @@ async fn replace_mcp_servers_streamable_http_isolates_headers_between_servers() env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -2843,6 +2856,7 @@ async fn replace_mcp_servers_serializes_disabled_flag() -> anyhow::Result<()> { env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: false, required: false, supports_parallel_tool_calls: false, @@ -2891,6 +2905,7 @@ async fn replace_mcp_servers_serializes_required_flag() -> anyhow::Result<()> { env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: true, supports_parallel_tool_calls: false, @@ -2939,6 +2954,7 @@ async fn replace_mcp_servers_serializes_tool_filters() -> anyhow::Result<()> { env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -2991,6 +3007,7 @@ async fn replace_mcp_servers_streamable_http_serializes_oauth_resource() -> anyh http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, diff --git a/codex-rs/core/src/config/edit.rs b/codex-rs/core/src/config/edit.rs index 443d00c93d..6f4231f908 100644 --- a/codex-rs/core/src/config/edit.rs +++ b/codex-rs/core/src/config/edit.rs @@ -222,6 +222,9 @@ mod document_helpers { if !config.enabled { entry["enabled"] = value(false); } + if let Some(environment) = &config.experimental_environment { + entry["experimental_environment"] = value(environment.clone()); + } if config.required { entry["required"] = value(true); } diff --git a/codex-rs/core/src/config/edit_tests.rs b/codex-rs/core/src/config/edit_tests.rs index c9288be30b..46680d5434 100644 --- a/codex-rs/core/src/config/edit_tests.rs +++ b/codex-rs/core/src/config/edit_tests.rs @@ -575,6 +575,7 @@ fn blocking_replace_mcp_servers_round_trips() { env_vars: vec!["FOO".to_string()], cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: true, @@ -602,6 +603,7 @@ fn blocking_replace_mcp_servers_round_trips() { ), env_http_headers: None, }, + experimental_environment: None, enabled: false, required: false, supports_parallel_tool_calls: false, @@ -666,6 +668,7 @@ fn blocking_replace_mcp_servers_serializes_tool_approval_overrides() { env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -727,6 +730,7 @@ foo = { command = "cmd" } env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -779,6 +783,7 @@ foo = { command = "cmd" } # keep me env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: false, required: false, supports_parallel_tool_calls: false, @@ -830,6 +835,7 @@ foo = { command = "cmd", args = ["--flag"] } # keep me env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -882,6 +888,7 @@ foo = { command = "cmd" } env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: false, required: false, supports_parallel_tool_calls: false, diff --git a/codex-rs/core/src/mcp_skill_dependencies.rs b/codex-rs/core/src/mcp_skill_dependencies.rs index 6cdd3cf084..c87b02730c 100644 --- a/codex-rs/core/src/mcp_skill_dependencies.rs +++ b/codex-rs/core/src/mcp_skill_dependencies.rs @@ -364,6 +364,7 @@ fn mcp_dependency_to_server_config( http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -391,6 +392,7 @@ fn mcp_dependency_to_server_config( env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, diff --git a/codex-rs/core/src/plugins/manager_tests.rs b/codex-rs/core/src/plugins/manager_tests.rs index f74b8fd117..e12f52c82b 100644 --- a/codex-rs/core/src/plugins/manager_tests.rs +++ b/codex-rs/core/src/plugins/manager_tests.rs @@ -174,6 +174,7 @@ async fn load_plugins_loads_default_skills_and_mcp_servers() { http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -510,6 +511,7 @@ async fn load_plugins_uses_manifest_configured_component_paths() { http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -619,6 +621,7 @@ async fn load_plugins_ignores_manifest_component_paths_without_dot_slash() { http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -776,6 +779,7 @@ fn capability_index_filters_inactive_and_zero_capability_plugins() { http_headers: None, env_http_headers: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, diff --git a/codex-rs/core/tests/suite/code_mode.rs b/codex-rs/core/tests/suite/code_mode.rs index d514e801e9..3cd2b16ce5 100644 --- a/codex-rs/core/tests/suite/code_mode.rs +++ b/codex-rs/core/tests/suite/code_mode.rs @@ -229,6 +229,7 @@ async fn run_code_mode_turn_with_rmcp_config( env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, diff --git a/codex-rs/core/tests/suite/js_repl.rs b/codex-rs/core/tests/suite/js_repl.rs index 2016897b3b..9c5eaef191 100644 --- a/codex-rs/core/tests/suite/js_repl.rs +++ b/codex-rs/core/tests/suite/js_repl.rs @@ -622,6 +622,7 @@ async fn js_repl_can_invoke_mcp_tools_by_display_name() -> Result<()> { env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, diff --git a/codex-rs/core/tests/suite/rmcp_client.rs b/codex-rs/core/tests/suite/rmcp_client.rs index b63369ea6f..607dc7bf9c 100644 --- a/codex-rs/core/tests/suite/rmcp_client.rs +++ b/codex-rs/core/tests/suite/rmcp_client.rs @@ -144,6 +144,7 @@ fn insert_mcp_server( server_name.to_string(), McpServerConfig { transport, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: options.supports_parallel_tool_calls, diff --git a/codex-rs/core/tests/suite/search_tool.rs b/codex-rs/core/tests/suite/search_tool.rs index 67cf8503b3..eff7bfb427 100644 --- a/codex-rs/core/tests/suite/search_tool.rs +++ b/codex-rs/core/tests/suite/search_tool.rs @@ -697,6 +697,7 @@ async fn tool_search_indexes_only_enabled_non_app_mcp_tools() -> Result<()> { env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, disabled_reason: None, diff --git a/codex-rs/core/tests/suite/sqlite_state.rs b/codex-rs/core/tests/suite/sqlite_state.rs index ef517d74fd..acdc1e9d8c 100644 --- a/codex-rs/core/tests/suite/sqlite_state.rs +++ b/codex-rs/core/tests/suite/sqlite_state.rs @@ -371,6 +371,7 @@ async fn mcp_call_marks_thread_memory_mode_polluted_when_configured() -> Result< env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, diff --git a/codex-rs/core/tests/suite/tools.rs b/codex-rs/core/tests/suite/tools.rs index 3ae2888ec7..d0bb138188 100644 --- a/codex-rs/core/tests/suite/tools.rs +++ b/codex-rs/core/tests/suite/tools.rs @@ -155,6 +155,7 @@ async fn historical_unavailable_mcp_call_is_exposed_as_placeholder_tool() -> Res env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, diff --git a/codex-rs/core/tests/suite/truncation.rs b/codex-rs/core/tests/suite/truncation.rs index 5bca13de50..33c74633f3 100644 --- a/codex-rs/core/tests/suite/truncation.rs +++ b/codex-rs/core/tests/suite/truncation.rs @@ -375,6 +375,7 @@ async fn mcp_tool_call_output_exceeds_limit_truncated_for_model() -> Result<()> env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -472,6 +473,7 @@ async fn mcp_image_output_preserves_image_and_no_text_summary() -> Result<()> { env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, @@ -752,6 +754,7 @@ async fn mcp_tool_call_output_not_truncated_with_custom_limit() -> Result<()> { env_vars: Vec::new(), cwd: None, }, + experimental_environment: None, enabled: true, required: false, supports_parallel_tool_calls: false, From 8475d51655a3eb922f84a8737e998a00a693d21d Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Thu, 16 Apr 2026 09:00:16 -0700 Subject: [PATCH 17/59] fix(tui): remove duplicate context statusline item (#18054) Addresses #18045 Problem: `/statusline` exposed both `context-remaining` and `context-remaining-percent` after conflicting PRs attempted to address the same context-status issue, including #17637, allowing duplicate footer segments. Solution: Remove the duplicate `context-remaining-percent` status-line item and update status-line tests and snapshots to use only canonical `context-remaining`. --- ..._snapshot_uses_runtime_preview_values.snap | 16 ++++++------- .../tui/src/bottom_pane/status_line_setup.rs | 21 +--------------- ...h_reasoning_context_remaining_footer.snap} | 0 .../tui/src/chatwidget/status_surfaces.rs | 2 +- .../src/chatwidget/tests/status_and_layout.rs | 24 +++---------------- 5 files changed, 13 insertions(+), 50 deletions(-) rename codex-rs/tui/src/chatwidget/snapshots/{codex_tui__chatwidget__tests__status_line_model_with_reasoning_context_remaining_percent_footer.snap => codex_tui__chatwidget__tests__status_line_model_with_reasoning_context_remaining_footer.snap} (100%) diff --git a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__status_line_setup__tests__setup_view_snapshot_uses_runtime_preview_values.snap b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__status_line_setup__tests__setup_view_snapshot_uses_runtime_preview_values.snap index 679aa77f4a..20aca1e334 100644 --- a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__status_line_setup__tests__setup_view_snapshot_uses_runtime_preview_values.snap +++ b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__status_line_setup__tests__setup_view_snapshot_uses_runtime_preview_values.snap @@ -8,14 +8,14 @@ expression: "render_lines(&view, 72)" Type to search > -› [x] model-name Current model name - [x] current-dir Current working directory - [x] git-branch Current Git branch (omitted when unavail… - [ ] model-with-reasoning Current model name with reasoning level - [ ] project-root Project root directory (omitted when una… - [ ] context-remaining Percentage of context window remaining (… - [ ] context-remaining-... Percentage of context window remaining (… - [ ] context-used Percentage of context window used (omitt… +› [x] model-name Current model name + [x] current-dir Current working directory + [x] git-branch Current Git branch (omitted when unavaila… + [ ] model-with-reasoning Current model name with reasoning level + [ ] project-root Project root directory (omitted when unav… + [ ] context-remaining Percentage of context window remaining (o… + [ ] context-used Percentage of context window used (omitte… + [ ] five-hour-limit Remaining usage on 5-hour usage limit (om… gpt-5-codex · ~/codex-rs · jif/statusline-preview Use ↑↓ to navigate, ←→ to move, space to select, enter to confirm, esc diff --git a/codex-rs/tui/src/bottom_pane/status_line_setup.rs b/codex-rs/tui/src/bottom_pane/status_line_setup.rs index b5fe8456ed..4c7e29c053 100644 --- a/codex-rs/tui/src/bottom_pane/status_line_setup.rs +++ b/codex-rs/tui/src/bottom_pane/status_line_setup.rs @@ -66,10 +66,6 @@ pub(crate) enum StatusLineItem { /// Percentage of context window remaining. ContextRemaining, - /// Percentage of context window remaining. - #[strum(to_string = "context-remaining-percent")] - ContextRemainingPercent, - /// Percentage of context window used. /// /// Also accepts the legacy `context-usage` config value. @@ -119,9 +115,6 @@ impl StatusLineItem { StatusLineItem::ContextRemaining => { "Percentage of context window remaining (omitted when unknown)" } - StatusLineItem::ContextRemainingPercent => { - "Percentage of context window remaining (omitted when unknown)" - } StatusLineItem::ContextUsed => { "Percentage of context window used (omitted when unknown)" } @@ -321,7 +314,7 @@ mod tests { } #[test] - fn context_remaining_is_separate_selectable_id() { + fn context_remaining_is_selectable_id() { assert_eq!( "context-remaining".parse::(), Ok(StatusLineItem::ContextRemaining) @@ -332,18 +325,6 @@ mod tests { ); } - #[test] - fn context_remaining_percent_is_separate_selectable_id() { - assert_eq!( - StatusLineItem::ContextRemainingPercent.to_string(), - "context-remaining-percent" - ); - assert_eq!( - "context-remaining-percent".parse::(), - Ok(StatusLineItem::ContextRemainingPercent) - ); - } - #[test] fn preview_uses_runtime_values() { let preview_data = StatusLinePreviewData::from_iter([ diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__status_line_model_with_reasoning_context_remaining_percent_footer.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__status_line_model_with_reasoning_context_remaining_footer.snap similarity index 100% rename from codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__status_line_model_with_reasoning_context_remaining_percent_footer.snap rename to codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__status_line_model_with_reasoning_context_remaining_footer.snap diff --git a/codex-rs/tui/src/chatwidget/status_surfaces.rs b/codex-rs/tui/src/chatwidget/status_surfaces.rs index ba95da93b3..a744711c2c 100644 --- a/codex-rs/tui/src/chatwidget/status_surfaces.rs +++ b/codex-rs/tui/src/chatwidget/status_surfaces.rs @@ -450,7 +450,7 @@ impl ChatWidget { Some(format!("{} used", format_tokens_compact(total))) } } - StatusLineItem::ContextRemaining | StatusLineItem::ContextRemainingPercent => self + StatusLineItem::ContextRemaining => self .status_line_context_remaining_percent() .map(|remaining| format!("Context {remaining}% left")), StatusLineItem::ContextUsed => self diff --git a/codex-rs/tui/src/chatwidget/tests/status_and_layout.rs b/codex-rs/tui/src/chatwidget/tests/status_and_layout.rs index e59928ff1c..75ca9d1fba 100644 --- a/codex-rs/tui/src/chatwidget/tests/status_and_layout.rs +++ b/codex-rs/tui/src/chatwidget/tests/status_and_layout.rs @@ -914,24 +914,6 @@ async fn status_line_legacy_context_usage_renders_context_used_percent() { ); } -#[tokio::test] -async fn status_line_context_remaining_percent_renders_labeled_percent() { - let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; - chat.thread_id = Some(ThreadId::new()); - chat.config.tui_status_line = Some(vec!["context-remaining-percent".to_string()]); - - chat.refresh_status_line(); - - assert_eq!( - status_line_text(&chat), - Some("Context 100% left".to_string()) - ); - assert!( - drain_insert_history(&mut rx).is_empty(), - "context-remaining-percent should remain a valid status line item" - ); -} - #[tokio::test] async fn status_line_branch_state_resets_when_git_branch_disabled() { let (mut chat, _rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; @@ -1200,7 +1182,7 @@ async fn status_line_model_with_reasoning_fast_footer_snapshot() { } #[tokio::test] -async fn status_line_model_with_reasoning_context_remaining_percent_footer_snapshot() { +async fn status_line_model_with_reasoning_context_remaining_footer_snapshot() { use ratatui::Terminal; use ratatui::backend::TestBackend; @@ -1211,7 +1193,7 @@ async fn status_line_model_with_reasoning_context_remaining_percent_footer_snaps chat.config.cwd = test_project_path().abs(); chat.config.tui_status_line = Some(vec![ "model-with-reasoning".to_string(), - "context-remaining-percent".to_string(), + "context-remaining".to_string(), "current-dir".to_string(), ]); chat.set_reasoning_effort(Some(ReasoningEffortConfig::XHigh)); @@ -1228,7 +1210,7 @@ async fn status_line_model_with_reasoning_context_remaining_percent_footer_snaps .draw(|f| chat.render(f.area(), f.buffer_mut())) .expect("draw model-with-reasoning footer"); assert_chatwidget_snapshot!( - "status_line_model_with_reasoning_context_remaining_percent_footer", + "status_line_model_with_reasoning_context_remaining_footer", normalized_backend_snapshot(terminal.backend()) ); } From 9c6d0386220071f3fd313c819c8eb3a8c54b6dec Mon Sep 17 00:00:00 2001 From: sayan-oai Date: Fri, 17 Apr 2026 00:01:14 +0800 Subject: [PATCH 18/59] [code mode] defer mcp tools from exec description (#17287) ## Summary - hide deferred MCP/app nested tool descriptions from the `exec` prompt in code mode - add short guidance that omitted nested tools are still available through `ALL_TOOLS` - cover the code_mode_only path with an integration test that discovers and calls a deferred app tool ## Motivation `code_mode_only` exposes only top-level `exec`/`wait`, but the `exec` description could still include a large nested-tool reference. This keeps deferred nested tools callable while avoiding that prompt bloat. ## Tests - `just fmt` - `just fix -p codex-code-mode` - `just fix -p codex-tools` - `cargo test -p codex-code-mode exec_description_mentions_deferred_nested_tools_when_available` - `cargo test -p codex-tools create_code_mode_tool_matches_expected_spec` - `cargo test -p codex-core code_mode_only_guides_all_tools_search_and_calls_deferred_app_tools` --- codex-rs/code-mode/src/description.rs | 45 ++++++-- codex-rs/core/tests/suite/code_mode.rs | 128 +++++++++++++++++++++++ codex-rs/tools/src/code_mode.rs | 6 +- codex-rs/tools/src/code_mode_tests.rs | 6 +- codex-rs/tools/src/tool_registry_plan.rs | 4 + 5 files changed, 176 insertions(+), 13 deletions(-) diff --git a/codex-rs/code-mode/src/description.rs b/codex-rs/code-mode/src/description.rs index 81e45add00..7f5e47fa25 100644 --- a/codex-rs/code-mode/src/description.rs +++ b/codex-rs/code-mode/src/description.rs @@ -9,6 +9,8 @@ use crate::PUBLIC_TOOL_NAME; const MAX_JS_SAFE_INTEGER: u64 = (1_u64 << 53) - 1; const CODE_MODE_ONLY_PREFACE: &str = "Use `exec/wait` tool to run all other tools, do not attempt to use any other tools directly"; +const DEFERRED_NESTED_TOOLS_GUIDANCE: &str = r#"Some nested MCP/app tools may be omitted from this description. They are still available on the global `tools` object and listed in `ALL_TOOLS`. +To find one, filter `ALL_TOOLS` by `name` and `description`; do not print the full `ALL_TOOLS` array. Print only a small set of relevant matches if you need to inspect them."#; const EXEC_DESCRIPTION_TEMPLATE: &str = r#"Run JavaScript code to orchestrate/compose tool calls - Evaluates the provided JavaScript code in a fresh V8 isolate as an async module. - All nested tools are available on the global `tools` object, for example `await tools.exec_command(...)`. Tool names are exposed as normalized JavaScript identifiers, for example `await tools.mcp__ologs__get_profile(...)`. @@ -251,15 +253,19 @@ pub fn build_exec_tool_description( enabled_tools: &[ToolDefinition], namespace_descriptions: &BTreeMap, code_mode_only: bool, + deferred_tools_available: bool, ) -> String { - if !code_mode_only { - return EXEC_DESCRIPTION_TEMPLATE.to_string(); + let mut sections = Vec::new(); + if code_mode_only { + sections.push(CODE_MODE_ONLY_PREFACE.to_string()); + } + sections.push(EXEC_DESCRIPTION_TEMPLATE.to_string()); + if deferred_tools_available { + sections.push(DEFERRED_NESTED_TOOLS_GUIDANCE.to_string()); + } + if !code_mode_only { + return sections.join("\n\n"); } - - let mut sections = vec![ - CODE_MODE_ONLY_PREFACE.to_string(), - EXEC_DESCRIPTION_TEMPLATE.to_string(), - ]; if !enabled_tools.is_empty() { let mut current_namespace: Option<&str> = None; @@ -863,6 +869,7 @@ mod tests { }], &BTreeMap::new(), /*code_mode_only*/ true, + /*deferred_tools_available*/ false, ); assert!(description.contains( "### `foo` @@ -872,8 +879,12 @@ bar" #[test] fn exec_description_mentions_timeout_helpers() { - let description = - build_exec_tool_description(&[], &BTreeMap::new(), /*code_mode_only*/ false); + let description = build_exec_tool_description( + &[], + &BTreeMap::new(), + /*code_mode_only*/ false, + /*deferred_tools_available*/ false, + ); assert!(description.contains("`setTimeout(callback: () => void, delayMs?: number)`")); assert!(description.contains("`clearTimeout(timeoutId?: number)`")); } @@ -924,6 +935,7 @@ bar" ], &namespace_descriptions, /*code_mode_only*/ true, + /*deferred_tools_available*/ false, ); assert_eq!(description.matches("## mcp__sample").count(), 1); assert!(description.contains("## mcp__sample\nShared namespace guidance.")); @@ -963,6 +975,7 @@ bar" }], &namespace_descriptions, /*code_mode_only*/ true, + /*deferred_tools_available*/ false, ); assert!(!description.contains("## mcp__sample")); @@ -1061,6 +1074,7 @@ bar" ], &BTreeMap::new(), /*code_mode_only*/ true, + /*deferred_tools_available*/ false, ); assert_eq!( @@ -1071,4 +1085,17 @@ bar" ); assert_eq!(description.matches("Shared MCP Types:").count(), 1); } + + #[test] + fn exec_description_mentions_deferred_nested_tools_when_available() { + let description = build_exec_tool_description( + &[], + &BTreeMap::new(), + /*code_mode_only*/ false, + /*deferred_tools_available*/ true, + ); + + assert!(description.contains("Some nested MCP/app tools may be omitted")); + assert!(description.contains("filter `ALL_TOOLS` by `name` and `description`")); + } } diff --git a/codex-rs/core/tests/suite/code_mode.rs b/codex-rs/core/tests/suite/code_mode.rs index 3cd2b16ce5..a428b32ba2 100644 --- a/codex-rs/core/tests/suite/code_mode.rs +++ b/codex-rs/core/tests/suite/code_mode.rs @@ -6,6 +6,8 @@ use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; use codex_config::types::McpServerConfig; use codex_config::types::McpServerTransportConfig; use codex_features::Feature; +use codex_login::CodexAuth; +use codex_models_manager::bundled_models_response; use codex_protocol::dynamic_tools::DynamicToolCallOutputContentItem; use codex_protocol::dynamic_tools::DynamicToolResponse; use codex_protocol::dynamic_tools::DynamicToolSpec; @@ -14,6 +16,7 @@ use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::Op; use codex_protocol::protocol::SandboxPolicy; use codex_protocol::user_input::UserInput; +use core_test_support::apps_test_server::AppsTestServer; use core_test_support::assert_regex_match; use core_test_support::responses; use core_test_support::responses::ResponseMock; @@ -347,6 +350,131 @@ async fn code_mode_only_restricts_prompt_tools() -> Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn code_mode_only_guides_all_tools_search_and_calls_deferred_app_tools() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let apps_server = AppsTestServer::mount_searchable(&server).await?; + let resp_mock = responses::mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-1"), + ev_custom_tool_call( + "call-1", + "exec", + r#" +const tool = ALL_TOOLS.find( + ({ name }) => name === "mcp__codex_apps__calendar_timezone_option_99" +); +if (!tool) { + text(JSON.stringify({ found: false })); +} else { + const result = await tools[tool.name]({ timezone: "UTC" }); + text(JSON.stringify({ + found: true, + isError: Boolean(result.isError), + text: result.content?.[0]?.text ?? "", + })); +} +"#, + ), + ev_completed("resp-1"), + ]), + ) + .await; + let follow_up_mock = responses::mount_sse_once( + &server, + sse(vec![ + ev_assistant_message("msg-1", "done"), + ev_completed("resp-2"), + ]), + ) + .await; + + let apps_base_url = apps_server.chatgpt_base_url.clone(); + let mut builder = test_codex() + .with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing()) + .with_config(move |config| { + config + .features + .enable(Feature::Apps) + .expect("test config should allow feature update"); + config + .features + .enable(Feature::ToolSearch) + .expect("test config should allow feature update"); + config + .features + .enable(Feature::CodeMode) + .expect("test config should allow feature update"); + config + .features + .enable(Feature::CodeModeOnly) + .expect("test config should allow feature update"); + config.chatgpt_base_url = apps_base_url; + config.model = Some("gpt-5-codex".to_string()); + + let mut model_catalog = bundled_models_response() + .unwrap_or_else(|err| panic!("bundled models.json should parse: {err}")); + let model = model_catalog + .models + .iter_mut() + .find(|model| model.slug == "gpt-5-codex") + .expect("gpt-5-codex exists in bundled models.json"); + model.supports_search_tool = true; + config.model_catalog = Some(model_catalog); + }); + let test = builder.build(&server).await?; + test.submit_turn("inspect tools in code mode only").await?; + + let first_body = resp_mock.single_request().body_json(); + assert_eq!( + tool_names(&first_body), + vec!["exec".to_string(), "wait".to_string()] + ); + + let exec_description = first_body + .get("tools") + .and_then(Value::as_array) + .and_then(|tools| { + tools.iter().find_map(|tool| { + if tool + .get("name") + .or_else(|| tool.get("type")) + .and_then(Value::as_str) + == Some("exec") + { + tool.get("description").and_then(Value::as_str) + } else { + None + } + }) + }) + .expect("exec description should be present"); + assert!(exec_description.contains("filter `ALL_TOOLS` by `name` and `description`")); + assert!(!exec_description.contains("calendar_timezone_option_99")); + + let request = follow_up_mock.single_request(); + let (output, success) = custom_tool_output_body_and_success(&request, "call-1"); + assert_ne!( + success, + Some(false), + "code_mode_only deferred app tool call failed unexpectedly: {output}" + ); + let parsed: Value = serde_json::from_str(&output)?; + assert_eq!( + parsed, + serde_json::json!({ + "found": true, + "isError": false, + "text": "called calendar_timezone_option_99 for at with ", + }) + ); + + Ok(()) +} + #[cfg_attr(windows, ignore = "no exec_command on Windows")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn code_mode_only_can_call_nested_tools() -> Result<()> { diff --git a/codex-rs/tools/src/code_mode.rs b/codex-rs/tools/src/code_mode.rs index e45c7be1d9..e8a73c40a9 100644 --- a/codex-rs/tools/src/code_mode.rs +++ b/codex-rs/tools/src/code_mode.rs @@ -138,7 +138,8 @@ pub fn create_wait_tool() -> ToolSpec { pub fn create_code_mode_tool( enabled_tools: &[CodeModeToolDefinition], namespace_descriptions: &BTreeMap, - code_mode_only_enabled: bool, + code_mode_only: bool, + deferred_tools_available: bool, ) -> ToolSpec { const CODE_MODE_FREEFORM_GRAMMAR: &str = r#" start: pragma_source | plain_source @@ -155,7 +156,8 @@ SOURCE: /[\s\S]+/ description: codex_code_mode::build_exec_tool_description( enabled_tools, namespace_descriptions, - code_mode_only_enabled, + code_mode_only, + deferred_tools_available, ), format: FreeformToolFormat { r#type: "grammar".to_string(), diff --git a/codex-rs/tools/src/code_mode_tests.rs b/codex-rs/tools/src/code_mode_tests.rs index 15cca164ec..d7d40cae6e 100644 --- a/codex-rs/tools/src/code_mode_tests.rs +++ b/codex-rs/tools/src/code_mode_tests.rs @@ -197,14 +197,16 @@ fn create_code_mode_tool_matches_expected_spec() { create_code_mode_tool( &enabled_tools, &BTreeMap::new(), - /*code_mode_only_enabled*/ true, + /*code_mode_only*/ true, + /*deferred_tools_available*/ false, ), ToolSpec::Freeform(FreeformTool { name: codex_code_mode::PUBLIC_TOOL_NAME.to_string(), description: codex_code_mode::build_exec_tool_description( &enabled_tools, &BTreeMap::new(), - /*code_mode_only*/ true + /*code_mode_only*/ true, + /*deferred_tools_available*/ false ), format: FreeformToolFormat { r#type: "grammar".to_string(), diff --git a/codex-rs/tools/src/tool_registry_plan.rs b/codex-rs/tools/src/tool_registry_plan.rs index b5da9afb25..ac11a2c4be 100644 --- a/codex-rs/tools/src/tool_registry_plan.rs +++ b/codex-rs/tools/src/tool_registry_plan.rs @@ -108,6 +108,10 @@ pub fn build_tool_registry_plan( &enabled_tools, &namespace_descriptions, config.code_mode_only_enabled, + config.search_tool + && params + .deferred_mcp_tools + .is_some_and(|tools| !tools.is_empty()), ), /*supports_parallel_tool_calls*/ false, config.code_mode_enabled, From 9c56e89e4f7b2c0433f832bd76399203891a954e Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Thu, 16 Apr 2026 09:01:45 -0700 Subject: [PATCH 19/59] Prefill rename prompt with current thread name (#18057) Addresses #12178 Problem: The TUI /rename prompt opened blank even when the current thread already had a custom name, making small edits awkward. Solution: Let custom prompts receive initial text and prefill /rename with the existing thread name while preserving the empty prompt for unnamed threads. Testing: Manually verified that the feature works by using `/rename` with unnamed and already-named threads. --- .../tui/src/bottom_pane/custom_prompt_view.rs | 9 ++++- codex-rs/tui/src/chatwidget.rs | 9 +++-- ..._tests__slash_rename_prefilled_prompt.snap | 10 ++++++ .../src/chatwidget/tests/slash_commands.rs | 33 +++++++++++++++++++ 4 files changed, 55 insertions(+), 6 deletions(-) create mode 100644 codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__slash_rename_prefilled_prompt.snap diff --git a/codex-rs/tui/src/bottom_pane/custom_prompt_view.rs b/codex-rs/tui/src/bottom_pane/custom_prompt_view.rs index e9f0ee697f..753aae51c7 100644 --- a/codex-rs/tui/src/bottom_pane/custom_prompt_view.rs +++ b/codex-rs/tui/src/bottom_pane/custom_prompt_view.rs @@ -41,15 +41,22 @@ impl CustomPromptView { pub(crate) fn new( title: String, placeholder: String, + initial_text: String, context_label: Option, on_submit: PromptSubmitted, ) -> Self { + let mut textarea = TextArea::new(); + if !initial_text.is_empty() { + textarea.set_text_clearing_elements(&initial_text); + textarea.set_cursor(initial_text.len()); + } + Self { title, placeholder, context_label, on_submit, - textarea: TextArea::new(), + textarea, textarea_state: RefCell::new(TextAreaState::default()), complete: false, } diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 9471c29ca6..de71efb9d8 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -5279,11 +5279,8 @@ impl ChatWidget { fn show_rename_prompt(&mut self) { let tx = self.app_event_tx.clone(); - let has_name = self - .thread_name - .as_ref() - .is_some_and(|name| !name.is_empty()); - let title = if has_name { + let existing_name = self.thread_name.as_deref().filter(|name| !name.is_empty()); + let title = if existing_name.is_some() { "Rename thread" } else { "Name thread" @@ -5291,6 +5288,7 @@ impl ChatWidget { let view = CustomPromptView::new( title.to_string(), "Type a name and press Enter".to_string(), + /*initial_text*/ existing_name.unwrap_or_default().to_string(), /*context_label*/ None, Box::new(move |name: String| { let Some(name) = crate::legacy_core::util::normalize_thread_name(&name) else { @@ -10599,6 +10597,7 @@ impl ChatWidget { let view = CustomPromptView::new( "Custom review instructions".to_string(), "Type instructions and press Enter".to_string(), + /*initial_text*/ String::new(), /*context_label*/ None, Box::new(move |prompt: String| { let trimmed = prompt.trim().to_string(); diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__slash_rename_prefilled_prompt.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__slash_rename_prefilled_prompt.snap new file mode 100644 index 0000000000..f1eb0d223b --- /dev/null +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__slash_rename_prefilled_prompt.snap @@ -0,0 +1,10 @@ +--- +source: tui/src/chatwidget/tests/slash_commands.rs +assertion_line: 135 +expression: popup +--- +▌ Rename thread +▌ +▌ Current project title + +Press enter to confirm or esc to go back diff --git a/codex-rs/tui/src/chatwidget/tests/slash_commands.rs b/codex-rs/tui/src/chatwidget/tests/slash_commands.rs index 7174b28b80..0c6b9be8e6 100644 --- a/codex-rs/tui/src/chatwidget/tests/slash_commands.rs +++ b/codex-rs/tui/src/chatwidget/tests/slash_commands.rs @@ -124,6 +124,39 @@ async fn inline_slash_command_is_available_from_local_recall_after_dispatch() { assert_eq!(chat.bottom_pane.composer_text(), "/rename Better title"); } +#[tokio::test] +async fn slash_rename_prefills_existing_thread_name() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + chat.thread_name = Some("Current project title".to_string()); + + chat.dispatch_command(SlashCommand::Rename); + + let popup = render_bottom_popup(&chat, /*width*/ 80); + assert_chatwidget_snapshot!("slash_rename_prefilled_prompt", popup); + + chat.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); + + assert_matches!( + rx.try_recv(), + Ok(AppEvent::CodexOp(Op::SetThreadName { name })) if name == "Current project title" + ); +} + +#[tokio::test] +async fn slash_rename_without_existing_thread_name_starts_empty() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + + chat.dispatch_command(SlashCommand::Rename); + + let popup = render_bottom_popup(&chat, /*width*/ 80); + assert!(popup.contains("Name thread")); + assert!(popup.contains("Type a name and press Enter")); + + chat.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); + + assert_matches!(rx.try_recv(), Err(TryRecvError::Empty)); +} + #[tokio::test] async fn usage_error_slash_command_is_available_from_local_recall() { let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(Some("gpt-5.3-codex")).await; From ab82568536d63528e6222eba32661ca329a674ca Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Thu, 16 Apr 2026 09:03:55 -0700 Subject: [PATCH 20/59] Fix invalid TUI resume hints (#18059) Addresses #18011 Problem: #16987 allowed zero-token TUI exits to print resume hints, which exposed precomputed thread ids before their rollout files were persisted; #17222 made the same invalid hint visible when switching sessions via `/resume`. Solution: Only include resume commands for TUI sessions backed by a materialized non-empty rollout, and cover both missing-rollout and persisted-rollout summary behavior. Testing: Manually verified by pressing Ctrl+D before the first prompt and confirming that no "to continue this session" message was generated. --- codex-rs/tui/src/app.rs | 86 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 78 insertions(+), 8 deletions(-) diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index 32f90a99b4..3fbcd64909 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -325,8 +325,12 @@ fn session_summary( token_usage: TokenUsage, thread_id: Option, thread_name: Option, + rollout_path: Option<&Path>, ) -> Option { let usage_line = (!token_usage.is_zero()).then(|| FinalOutput::from(token_usage).to_string()); + let (thread_id, thread_name) = resumable_thread(thread_id, thread_name, rollout_path) + .map(|thread| (Some(thread.thread_id), thread.thread_name)) + .unwrap_or((None, None)); let resume_command = crate::legacy_core::util::resume_command(thread_name.as_deref(), thread_id); @@ -340,6 +344,29 @@ fn session_summary( }) } +#[derive(Debug, Clone, PartialEq, Eq)] +struct ResumableThread { + thread_id: ThreadId, + thread_name: Option, +} + +fn resumable_thread( + thread_id: Option, + thread_name: Option, + rollout_path: Option<&Path>, +) -> Option { + let thread_id = thread_id?; + let rollout_path = rollout_path?; + rollout_path_is_resumable(rollout_path).then_some(ResumableThread { + thread_id, + thread_name, + }) +} + +fn rollout_path_is_resumable(rollout_path: &Path) -> bool { + std::fs::metadata(rollout_path).is_ok_and(|metadata| metadata.is_file() && metadata.len() > 0) +} + fn errors_for_cwd(cwd: &Path, response: &ListSkillsResponseEvent) -> Vec { response .skills @@ -3448,6 +3475,7 @@ impl App { self.chat_widget.token_usage(), self.chat_widget.thread_id(), self.chat_widget.thread_name(), + self.chat_widget.rollout_path().as_deref(), ); self.shutdown_current_thread(app_server).await; let tracked_thread_ids: Vec = @@ -4126,10 +4154,15 @@ impl App { return Err(err); } }; + let resumable_thread = resumable_thread( + app.chat_widget.thread_id(), + app.chat_widget.thread_name(), + app.chat_widget.rollout_path().as_deref(), + ); Ok(AppExitInfo { token_usage: app.token_usage(), - thread_id: app.chat_widget.thread_id(), - thread_name: app.chat_widget.thread_name(), + thread_id: resumable_thread.as_ref().map(|thread| thread.thread_id), + thread_name: resumable_thread.and_then(|thread| thread.thread_name), update_action: app.pending_update_action, exit_reason, }) @@ -4249,6 +4282,7 @@ impl App { self.chat_widget.token_usage(), self.chat_widget.thread_id(), self.chat_widget.thread_name(), + self.chat_widget.rollout_path().as_deref(), ); match app_server .resume_thread(resume_config.clone(), target_session.thread_id) @@ -4403,6 +4437,7 @@ impl App { self.chat_widget.token_usage(), self.chat_widget.thread_id(), self.chat_widget.thread_name(), + self.chat_widget.rollout_path().as_deref(), ); self.chat_widget .add_plain_history_lines(vec!["/fork".magenta().into()]); @@ -11496,14 +11531,33 @@ guardian_approval = true session_summary( TokenUsage::default(), /*thread_id*/ None, - /*thread_name*/ None + /*thread_name*/ None, + /*rollout_path*/ None, ) .is_none() ); } #[tokio::test] - async fn session_summary_includes_resume_hint() { + async fn session_summary_skips_resume_hint_until_rollout_exists() { + let usage = TokenUsage::default(); + let conversation = ThreadId::from_string("123e4567-e89b-12d3-a456-426614174000").unwrap(); + let temp_dir = tempdir().expect("temp dir"); + let rollout_path = temp_dir.path().join("rollout.jsonl"); + + assert!( + session_summary( + usage, + Some(conversation), + /*thread_name*/ None, + Some(&rollout_path), + ) + .is_none() + ); + } + + #[tokio::test] + async fn session_summary_includes_resume_hint_for_persisted_rollout() { let usage = TokenUsage { input_tokens: 10, output_tokens: 2, @@ -11511,9 +11565,17 @@ guardian_approval = true ..Default::default() }; let conversation = ThreadId::from_string("123e4567-e89b-12d3-a456-426614174000").unwrap(); + let temp_dir = tempdir().expect("temp dir"); + let rollout_path = temp_dir.path().join("rollout.jsonl"); + std::fs::write(&rollout_path, "{}\n").expect("write rollout"); - let summary = - session_summary(usage, Some(conversation), /*thread_name*/ None).expect("summary"); + let summary = session_summary( + usage, + Some(conversation), + /*thread_name*/ None, + Some(&rollout_path), + ) + .expect("summary"); assert_eq!( summary.usage_line, Some("Token usage: total=12 input=10 output=2".to_string()) @@ -11533,9 +11595,17 @@ guardian_approval = true ..Default::default() }; let conversation = ThreadId::from_string("123e4567-e89b-12d3-a456-426614174000").unwrap(); + let temp_dir = tempdir().expect("temp dir"); + let rollout_path = temp_dir.path().join("rollout.jsonl"); + std::fs::write(&rollout_path, "{}\n").expect("write rollout"); - let summary = session_summary(usage, Some(conversation), Some("my-session".to_string())) - .expect("summary"); + let summary = session_summary( + usage, + Some(conversation), + Some("my-session".to_string()), + Some(&rollout_path), + ) + .expect("summary"); assert_eq!( summary.resume_command, Some("codex resume my-session".to_string()) From 85203d8872b139ab6c8fd8478d557d8f5de694ae Mon Sep 17 00:00:00 2001 From: Won Park Date: Thu, 16 Apr 2026 09:05:38 -0700 Subject: [PATCH 21/59] Launch image generation by default (#17153) ## Summary - Promote `image_generation` from under-development to stable - Enable image generation by default in the feature registry - Update feature coverage for the new launch-state expectation - Add the missing image-generation auth fixture field in a tool registry test ## Testing - `just fmt` - `cargo test -p codex-features` - `cargo test -p codex-tools` currently fails: `test_full_toolset_specs_for_gpt5_codex_unified_exec_web_search` needs its expected default tool list updated for `image_generation` --- codex-rs/core/src/tools/spec_tests.rs | 9 +++++++++ codex-rs/features/src/lib.rs | 4 ++-- codex-rs/features/src/tests.rs | 12 ++++++------ codex-rs/tools/src/tool_registry_plan_tests.rs | 1 + 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/codex-rs/core/src/tools/spec_tests.rs b/codex-rs/core/src/tools/spec_tests.rs index fa52aa6b0f..9de9c81a92 100644 --- a/codex-rs/core/src/tools/spec_tests.rs +++ b/codex-rs/core/src/tools/spec_tests.rs @@ -417,6 +417,7 @@ async fn test_build_specs_gpt5_codex_default() { "request_user_input", "apply_patch", "web_search", + "image_generation", "view_image", "spawn_agent", "send_input", @@ -441,6 +442,7 @@ async fn test_build_specs_gpt51_codex_default() { "request_user_input", "apply_patch", "web_search", + "image_generation", "view_image", "spawn_agent", "send_input", @@ -467,6 +469,7 @@ async fn test_build_specs_gpt5_codex_unified_exec_web_search() { "request_user_input", "apply_patch", "web_search", + "image_generation", "view_image", "spawn_agent", "send_input", @@ -493,6 +496,7 @@ async fn test_build_specs_gpt51_codex_unified_exec_web_search() { "request_user_input", "apply_patch", "web_search", + "image_generation", "view_image", "spawn_agent", "send_input", @@ -517,6 +521,7 @@ async fn test_gpt_5_1_codex_max_defaults() { "request_user_input", "apply_patch", "web_search", + "image_generation", "view_image", "spawn_agent", "send_input", @@ -541,6 +546,7 @@ async fn test_codex_5_1_mini_defaults() { "request_user_input", "apply_patch", "web_search", + "image_generation", "view_image", "spawn_agent", "send_input", @@ -564,6 +570,7 @@ async fn test_gpt_5_defaults() { "update_plan", "request_user_input", "web_search", + "image_generation", "view_image", "spawn_agent", "send_input", @@ -588,6 +595,7 @@ async fn test_gpt_5_1_defaults() { "request_user_input", "apply_patch", "web_search", + "image_generation", "view_image", "spawn_agent", "send_input", @@ -614,6 +622,7 @@ async fn test_gpt_5_1_codex_max_unified_exec_web_search() { "request_user_input", "apply_patch", "web_search", + "image_generation", "view_image", "spawn_agent", "send_input", diff --git a/codex-rs/features/src/lib.rs b/codex-rs/features/src/lib.rs index 2d1ac89851..a2e656be44 100644 --- a/codex-rs/features/src/lib.rs +++ b/codex-rs/features/src/lib.rs @@ -812,8 +812,8 @@ pub const FEATURES: &[FeatureSpec] = &[ FeatureSpec { id: Feature::ImageGeneration, key: "image_generation", - stage: Stage::UnderDevelopment, - default_enabled: false, + stage: Stage::Stable, + default_enabled: true, }, FeatureSpec { id: Feature::SkillMcpDependencyInstall, diff --git a/codex-rs/features/src/tests.rs b/codex-rs/features/src/tests.rs index 56f92d55b7..d2f73cedad 100644 --- a/codex-rs/features/src/tests.rs +++ b/codex-rs/features/src/tests.rs @@ -160,6 +160,12 @@ fn use_linux_sandbox_bwrap_is_a_removed_feature_key() { ); } +#[test] +fn image_generation_is_stable_and_enabled_by_default() { + assert_eq!(Feature::ImageGeneration.stage(), Stage::Stable); + assert_eq!(Feature::ImageGeneration.default_enabled(), true); +} + #[test] fn image_detail_original_is_a_removed_feature_key() { assert_eq!( @@ -168,12 +174,6 @@ fn image_detail_original_is_a_removed_feature_key() { ); } -#[test] -fn image_generation_is_under_development() { - assert_eq!(Feature::ImageGeneration.stage(), Stage::UnderDevelopment); - assert_eq!(Feature::ImageGeneration.default_enabled(), false); -} - #[test] fn tool_call_mcp_elicitation_is_stable_and_enabled_by_default() { assert_eq!(Feature::ToolCallMcpElicitation.stage(), Stage::Stable); diff --git a/codex-rs/tools/src/tool_registry_plan_tests.rs b/codex-rs/tools/src/tool_registry_plan_tests.rs index 50b38905ee..cc8c05dd9a 100644 --- a/codex-rs/tools/src/tool_registry_plan_tests.rs +++ b/codex-rs/tools/src/tool_registry_plan_tests.rs @@ -97,6 +97,7 @@ fn test_full_toolset_specs_for_gpt5_codex_unified_exec_web_search() { search_context_size: None, search_content_types: None, }, + create_image_generation_tool("png"), create_view_image_tool(ViewImageToolOptions { can_request_original_image_detail: config.can_request_original_image_detail, }), From 3a4fa77ad7ef9a99e16e74c409c70d2deb7488ce Mon Sep 17 00:00:00 2001 From: Won Park Date: Thu, 16 Apr 2026 09:06:10 -0700 Subject: [PATCH 22/59] Make yolo skip managed-network tool enforcement (#18042) ## Summary This makes `DangerFullAccess` / yolo tool execution fully opt out of managed-network enforcement. Previously, yolo turns could have `turn.network` stripped while tool orchestration still derived `enforce_managed_network=true` from `requirements.toml.network`. That created an inconsistent state where the turn had no managed proxy attached, but tool execution still behaved like managed networking was active. This updates the tool orchestration and JS REPL paths to treat managed networking as active only when the current turn actually has `turn.network`. ## Behavior - Yolo / `DangerFullAccess`: no managed proxy, no managed-network enforcement. - Guardian / workspace-write with managed proxy: managed-network enforcement still applies. - Avoids the half-state where yolo has no proxy but still gets managed-network sandbox behavior. ## Tests - `just fmt` - `cargo test -p codex-core danger_full_access_tool_attempts_do_not_enforce_managed_network -- --nocapture` - `cargo test -p codex-core danger_full_access -- --nocapture` - `just fix -p codex-core` Co-authored-by: jgershen-oai --- codex-rs/core/src/codex_tests.rs | 111 ++++++++++++++++++++ codex-rs/core/src/tools/js_repl/mod.rs | 11 +- codex-rs/core/src/tools/network_approval.rs | 4 +- codex-rs/core/src/tools/orchestrator.rs | 23 ++-- 4 files changed, 125 insertions(+), 24 deletions(-) diff --git a/codex-rs/core/src/codex_tests.rs b/codex-rs/core/src/codex_tests.rs index fcb1b58993..4805b03d01 100644 --- a/codex-rs/core/src/codex_tests.rs +++ b/codex-rs/core/src/codex_tests.rs @@ -765,6 +765,117 @@ async fn danger_full_access_turns_do_not_expose_managed_network_proxy() -> anyho Ok(()) } +#[tokio::test] +async fn danger_full_access_tool_attempts_do_not_enforce_managed_network() -> anyhow::Result<()> { + #[derive(Default)] + struct ProbeToolRuntime { + enforce_managed_network: Vec, + } + + impl crate::tools::sandboxing::Approvable<()> for ProbeToolRuntime { + type ApprovalKey = String; + + fn approval_keys(&self, _req: &()) -> Vec { + vec!["probe".to_string()] + } + + fn start_approval_async<'a>( + &'a mut self, + _req: &'a (), + _ctx: crate::tools::sandboxing::ApprovalCtx<'a>, + ) -> futures::future::BoxFuture<'a, ReviewDecision> { + Box::pin(async { ReviewDecision::Approved }) + } + } + + impl crate::tools::sandboxing::Sandboxable for ProbeToolRuntime { + fn sandbox_preference(&self) -> codex_sandboxing::SandboxablePreference { + codex_sandboxing::SandboxablePreference::Auto + } + } + + impl crate::tools::sandboxing::ToolRuntime<(), ()> for ProbeToolRuntime { + async fn run( + &mut self, + _req: &(), + attempt: &crate::tools::sandboxing::SandboxAttempt<'_>, + _ctx: &crate::tools::sandboxing::ToolCtx, + ) -> Result<(), crate::tools::sandboxing::ToolError> { + self.enforce_managed_network + .push(attempt.enforce_managed_network); + Ok(()) + } + } + + let network_spec = crate::config::NetworkProxySpec::from_config_and_constraints( + NetworkProxyConfig::default(), + Some(NetworkConstraints { + enabled: Some(true), + ..Default::default() + }), + &SandboxPolicy::DangerFullAccess, + )?; + + let session = make_session_with_config(move |config| { + config.permissions.sandbox_policy = + codex_config::Constrained::allow_any(SandboxPolicy::DangerFullAccess); + config.permissions.network = Some(network_spec); + + let layers = config + .config_layer_stack + .get_layers( + ConfigLayerStackOrdering::LowestPrecedenceFirst, + /*include_disabled*/ true, + ) + .into_iter() + .cloned() + .collect(); + let mut requirements = config.config_layer_stack.requirements().clone(); + requirements.network = Some(Sourced::new( + NetworkConstraints { + enabled: Some(true), + ..Default::default() + }, + RequirementSource::CloudRequirements, + )); + let mut requirements_toml = config.config_layer_stack.requirements_toml().clone(); + requirements_toml.network = Some(crate::config_loader::NetworkRequirementsToml { + enabled: Some(true), + ..Default::default() + }); + config.config_layer_stack = ConfigLayerStack::new(layers, requirements, requirements_toml) + .expect("rebuild config layer stack with network requirements"); + }) + .await?; + + let turn = session.new_default_turn().await; + assert!(turn.network.is_none()); + + let mut orchestrator = crate::tools::orchestrator::ToolOrchestrator::new(); + let mut tool = ProbeToolRuntime::default(); + let tool_ctx = crate::tools::sandboxing::ToolCtx { + session: Arc::clone(&session), + turn: Arc::clone(&turn), + call_id: "probe-call".to_string(), + tool_name: "probe".to_string(), + }; + + orchestrator + .run( + &mut tool, + &(), + &tool_ctx, + turn.as_ref(), + AskForApproval::Never, + ) + .await + .expect("probe runtime should succeed"); + + assert_eq!(tool.enforce_managed_network, vec![false]); + + Ok(()) +} + #[tokio::test] async fn workspace_write_turns_continue_to_expose_managed_network_proxy() -> anyhow::Result<()> { let sandbox_policy = SandboxPolicy::new_workspace_write_policy(); diff --git a/codex-rs/core/src/tools/js_repl/mod.rs b/codex-rs/core/src/tools/js_repl/mod.rs index 5c0884d9ee..48bfb8ac41 100644 --- a/codex-rs/core/src/tools/js_repl/mod.rs +++ b/codex-rs/core/src/tools/js_repl/mod.rs @@ -1033,18 +1033,13 @@ impl JsReplManager { } let sandbox = SandboxManager::new(); - let has_managed_network_requirements = turn - .config - .config_layer_stack - .requirements_toml() - .network - .is_some(); + let managed_network_active = turn.network.is_some(); let sandbox_type = sandbox.select_initial( &turn.file_system_sandbox_policy, turn.network_sandbox_policy, SandboxablePreference::Auto, turn.windows_sandbox_level, - has_managed_network_requirements, + managed_network_active, ); let command = SandboxCommand { program: node_path.into_os_string(), @@ -1067,7 +1062,7 @@ impl JsReplManager { file_system_policy: &turn.file_system_sandbox_policy, network_policy: turn.network_sandbox_policy, sandbox: sandbox_type, - enforce_managed_network: has_managed_network_requirements, + enforce_managed_network: managed_network_active, network: None, sandbox_policy_cwd: &turn.cwd, codex_linux_sandbox_exe: turn.codex_linux_sandbox_exe.as_deref(), diff --git a/codex-rs/core/src/tools/network_approval.rs b/codex-rs/core/src/tools/network_approval.rs index c91b6df7ca..de689ea25d 100644 --- a/codex-rs/core/src/tools/network_approval.rs +++ b/codex-rs/core/src/tools/network_approval.rs @@ -578,11 +578,11 @@ pub(crate) fn build_network_policy_decider( pub(crate) async fn begin_network_approval( session: &Session, turn_id: &str, - has_managed_network_requirements: bool, + managed_network_active: bool, spec: Option, ) -> Option { let spec = spec?; - if !has_managed_network_requirements || spec.network.is_none() { + if !managed_network_active || spec.network.is_none() { return None; } diff --git a/codex-rs/core/src/tools/orchestrator.rs b/codex-rs/core/src/tools/orchestrator.rs index 36450da3f7..f9b2a64966 100644 --- a/codex-rs/core/src/tools/orchestrator.rs +++ b/codex-rs/core/src/tools/orchestrator.rs @@ -55,7 +55,7 @@ impl ToolOrchestrator { req: &Rq, tool_ctx: &ToolCtx, attempt: &SandboxAttempt<'_>, - has_managed_network_requirements: bool, + managed_network_active: bool, ) -> (Result, Option) where T: ToolRuntime, @@ -63,7 +63,7 @@ impl ToolOrchestrator { let network_approval = begin_network_approval( &tool_ctx.session, &tool_ctx.turn.sub_id, - has_managed_network_requirements, + managed_network_active, tool.network_approval_spec(req, tool_ctx), ) .await; @@ -180,12 +180,7 @@ impl ToolOrchestrator { } // 2) First attempt under the selected sandbox. - let has_managed_network_requirements = turn_ctx - .config - .config_layer_stack - .requirements_toml() - .network - .is_some(); + let managed_network_active = turn_ctx.network.is_some(); let initial_sandbox = match tool.sandbox_mode_for_first_attempt(req) { SandboxOverride::BypassSandboxFirstAttempt => SandboxType::None, SandboxOverride::NoOverride => self.sandbox.select_initial( @@ -193,7 +188,7 @@ impl ToolOrchestrator { turn_ctx.network_sandbox_policy, tool.sandbox_preference(), turn_ctx.windows_sandbox_level, - has_managed_network_requirements, + managed_network_active, ), }; @@ -204,7 +199,7 @@ impl ToolOrchestrator { policy: &turn_ctx.sandbox_policy, file_system_policy: &turn_ctx.file_system_sandbox_policy, network_policy: turn_ctx.network_sandbox_policy, - enforce_managed_network: has_managed_network_requirements, + enforce_managed_network: managed_network_active, manager: &self.sandbox, sandbox_cwd: &turn_ctx.cwd, codex_linux_sandbox_exe: turn_ctx.codex_linux_sandbox_exe.as_ref(), @@ -221,7 +216,7 @@ impl ToolOrchestrator { req, tool_ctx, &initial_attempt, - has_managed_network_requirements, + managed_network_active, ) .await; match first_result { @@ -236,7 +231,7 @@ impl ToolOrchestrator { output, network_policy_decision, }))) => { - let network_approval_context = if has_managed_network_requirements { + let network_approval_context = if managed_network_active { network_policy_decision .as_ref() .and_then(network_approval_context_from_payload) @@ -341,7 +336,7 @@ impl ToolOrchestrator { policy: &turn_ctx.sandbox_policy, file_system_policy: &turn_ctx.file_system_sandbox_policy, network_policy: turn_ctx.network_sandbox_policy, - enforce_managed_network: has_managed_network_requirements, + enforce_managed_network: managed_network_active, manager: &self.sandbox, sandbox_cwd: &turn_ctx.cwd, codex_linux_sandbox_exe: None, @@ -359,7 +354,7 @@ impl ToolOrchestrator { req, tool_ctx, &escalated_attempt, - has_managed_network_requirements, + managed_network_active, ) .await; retry_result.map(|output| OrchestratorRunResult { From baaf42b2e4f885361ddde2176e76c99e2dc544e0 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Thu, 16 Apr 2026 18:02:02 +0100 Subject: [PATCH 23/59] fix: model menu pop (#18154) Fix the `/model` menu looping on itself --- .../tui/src/bottom_pane/bottom_pane_view.rs | 20 ++++++++++ .../tui/src/bottom_pane/custom_prompt_view.rs | 15 +++++--- .../src/bottom_pane/list_selection_view.rs | 30 ++++++++++++--- codex-rs/tui/src/bottom_pane/mod.rs | 38 +++++++++++++++---- codex-rs/tui/src/chatwidget.rs | 4 ++ 5 files changed, 89 insertions(+), 18 deletions(-) diff --git a/codex-rs/tui/src/bottom_pane/bottom_pane_view.rs b/codex-rs/tui/src/bottom_pane/bottom_pane_view.rs index 6fa946a5af..3c0ab2c74e 100644 --- a/codex-rs/tui/src/bottom_pane/bottom_pane_view.rs +++ b/codex-rs/tui/src/bottom_pane/bottom_pane_view.rs @@ -7,6 +7,13 @@ use crossterm::event::KeyEvent; use super::CancellationEvent; +/// Reason an active bottom-pane view finished. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ViewCompletion { + Accepted, + Cancelled, +} + /// Trait implemented by every view that can be shown in the bottom pane. pub(crate) trait BottomPaneView: Renderable { /// Handle a key event while the view is active. A redraw is always @@ -18,6 +25,19 @@ pub(crate) trait BottomPaneView: Renderable { false } + /// Return the completion reason once the view has finished. + fn completion(&self) -> Option { + None + } + + /// Return true when this view should be removed after a child view is accepted. + fn dismiss_after_child_accept(&self) -> bool { + false + } + + /// Clear any pending child-flow cleanup marker after a child view is cancelled. + fn clear_dismiss_after_child_accept(&mut self) {} + /// Stable identifier for views that need external refreshes while open. fn view_id(&self) -> Option<&'static str> { None diff --git a/codex-rs/tui/src/bottom_pane/custom_prompt_view.rs b/codex-rs/tui/src/bottom_pane/custom_prompt_view.rs index 753aae51c7..bce9528fde 100644 --- a/codex-rs/tui/src/bottom_pane/custom_prompt_view.rs +++ b/codex-rs/tui/src/bottom_pane/custom_prompt_view.rs @@ -18,6 +18,7 @@ use super::popup_consts::standard_popup_hint_line; use super::CancellationEvent; use super::bottom_pane_view::BottomPaneView; +use super::bottom_pane_view::ViewCompletion; use super::textarea::TextArea; use super::textarea::TextAreaState; @@ -34,7 +35,7 @@ pub(crate) struct CustomPromptView { // UI state textarea: TextArea, textarea_state: RefCell, - complete: bool, + completion: Option, } impl CustomPromptView { @@ -58,7 +59,7 @@ impl CustomPromptView { on_submit, textarea, textarea_state: RefCell::new(TextAreaState::default()), - complete: false, + completion: None, } } } @@ -79,7 +80,7 @@ impl BottomPaneView for CustomPromptView { let text = self.textarea.text().trim().to_string(); if !text.is_empty() { (self.on_submit)(text); - self.complete = true; + self.completion = Some(ViewCompletion::Accepted); } } KeyEvent { @@ -95,12 +96,16 @@ impl BottomPaneView for CustomPromptView { } fn on_ctrl_c(&mut self) -> CancellationEvent { - self.complete = true; + self.completion = Some(ViewCompletion::Cancelled); CancellationEvent::Handled } fn is_complete(&self) -> bool { - self.complete + self.completion.is_some() + } + + fn completion(&self) -> Option { + self.completion } fn handle_paste(&mut self, pasted: String) -> bool { diff --git a/codex-rs/tui/src/bottom_pane/list_selection_view.rs b/codex-rs/tui/src/bottom_pane/list_selection_view.rs index e3eecf2ed0..c95565455d 100644 --- a/codex-rs/tui/src/bottom_pane/list_selection_view.rs +++ b/codex-rs/tui/src/bottom_pane/list_selection_view.rs @@ -21,6 +21,7 @@ use crate::render::renderable::Renderable; use super::CancellationEvent; use super::bottom_pane_view::BottomPaneView; +use super::bottom_pane_view::ViewCompletion; use super::popup_consts::MAX_POPUP_ROWS; use super::scroll_state::ScrollState; pub(crate) use super::selection_popup_common::ColumnWidthMode; @@ -121,6 +122,7 @@ pub(crate) struct SelectionItem { pub is_disabled: bool, pub actions: Vec, pub dismiss_on_select: bool, + pub dismiss_parent_on_child_accept: bool, pub search_value: Option, pub disabled_reason: Option, } @@ -211,7 +213,8 @@ pub(crate) struct ListSelectionView { footer_hint: Option>, items: Vec, state: ScrollState, - complete: bool, + completion: Option, + dismiss_after_child_accept: bool, app_event_tx: AppEventSender, is_searchable: bool, search_query: String, @@ -259,7 +262,8 @@ impl ListSelectionView { footer_hint: params.footer_hint, items: params.items, state: ScrollState::new(), - complete: false, + completion: None, + dismiss_after_child_accept: false, app_event_tx, is_searchable: params.is_searchable, search_query: String::new(), @@ -458,13 +462,15 @@ impl ListSelectionView { act(&self.app_event_tx); } if item.dismiss_on_select { - self.complete = true; + self.completion = Some(ViewCompletion::Accepted); + } else if item.dismiss_parent_on_child_accept { + self.dismiss_after_child_accept = true; } } else if selected_item.is_none() { if let Some(cb) = &self.on_cancel { cb(&self.app_event_tx); } - self.complete = true; + self.completion = Some(ViewCompletion::Cancelled); } } @@ -672,7 +678,19 @@ impl BottomPaneView for ListSelectionView { } fn is_complete(&self) -> bool { - self.complete + self.completion.is_some() + } + + fn completion(&self) -> Option { + self.completion + } + + fn dismiss_after_child_accept(&self) -> bool { + self.dismiss_after_child_accept + } + + fn clear_dismiss_after_child_accept(&mut self) { + self.dismiss_after_child_accept = false; } fn view_id(&self) -> Option<&'static str> { @@ -687,7 +705,7 @@ impl BottomPaneView for ListSelectionView { if let Some(cb) = &self.on_cancel { cb(&self.app_event_tx); } - self.complete = true; + self.completion = Some(ViewCompletion::Cancelled); CancellationEvent::Handled } } diff --git a/codex-rs/tui/src/bottom_pane/mod.rs b/codex-rs/tui/src/bottom_pane/mod.rs index 91c8bd48eb..2600a90cd9 100644 --- a/codex-rs/tui/src/bottom_pane/mod.rs +++ b/codex-rs/tui/src/bottom_pane/mod.rs @@ -30,6 +30,7 @@ use crate::render::renderable::Renderable; use crate::render::renderable::RenderableItem; use crate::tui::FrameRequester; use bottom_pane_view::BottomPaneView; +use bottom_pane_view::ViewCompletion; use codex_features::Features; use codex_file_search::FileMatch; use codex_protocol::request_user_input::RequestUserInputEvent; @@ -380,8 +381,25 @@ impl BottomPane { self.request_redraw(); } - fn pop_active_view(&mut self) { + fn pop_active_view_with_completion(&mut self, completion: Option) { if self.view_stack.pop().is_some() { + match completion { + Some(ViewCompletion::Accepted) => { + while self + .view_stack + .last() + .is_some_and(|view| view.dismiss_after_child_accept()) + { + self.view_stack.pop(); + } + } + Some(ViewCompletion::Cancelled) => { + if let Some(view) = self.view_stack.last_mut() { + view.clear_dismiss_after_child_accept(); + } + } + None => {} + } self.on_view_stack_depth_decreased(); } } @@ -403,7 +421,7 @@ impl BottomPane { // We need three pieces of information after routing the key: // whether Esc completed the view, whether the view finished for any // reason, and whether a paste-burst timer should be scheduled. - let (ctrl_c_completed, view_complete, view_in_paste_burst) = { + let (ctrl_c_completed, view_complete, completion, view_in_paste_burst) = { let last_index = self.view_stack.len() - 1; let view = &mut self.view_stack[last_index]; let prefer_esc = @@ -413,22 +431,27 @@ impl BottomPane { && matches!(view.on_ctrl_c(), CancellationEvent::Handled) && view.is_complete(); if ctrl_c_completed { - (true, true, false) + (true, true, view.completion(), false) } else { view.handle_key_event(key_event); - (false, view.is_complete(), view.is_in_paste_burst()) + ( + false, + view.is_complete(), + view.completion(), + view.is_in_paste_burst(), + ) } }; if ctrl_c_completed { - self.pop_active_view(); + self.pop_active_view_with_completion(completion); if let Some(next_view) = self.view_stack.last() && next_view.is_in_paste_burst() { self.request_redraw_in(ChatComposer::recommended_paste_flush_delay()); } } else if view_complete { - self.pop_active_view(); + self.pop_active_view_with_completion(completion); } else if view_in_paste_burst { self.request_redraw_in(ChatComposer::recommended_paste_flush_delay()); } @@ -481,9 +504,10 @@ impl BottomPane { if let Some(view) = self.view_stack.last_mut() { let event = view.on_ctrl_c(); let view_complete = view.is_complete(); + let completion = view.completion(); if matches!(event, CancellationEvent::Handled) { if view_complete { - self.pop_active_view(); + self.pop_active_view_with_completion(completion); } self.show_quit_shortcut_hint(key_hint::ctrl(KeyCode::Char('c'))); self.request_redraw(); diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index de71efb9d8..890489bc2c 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -7953,6 +7953,7 @@ impl ChatWidget { is_default: preset.is_default, actions, dismiss_on_select: single_supported_effort, + dismiss_parent_on_child_accept: !single_supported_effort, ..Default::default() }); } @@ -10476,6 +10477,7 @@ impl ChatWidget { } })], dismiss_on_select: false, + dismiss_parent_on_child_accept: true, ..Default::default() }); @@ -10501,6 +10503,7 @@ impl ChatWidget { } })], dismiss_on_select: false, + dismiss_parent_on_child_accept: true, ..Default::default() }); @@ -10510,6 +10513,7 @@ impl ChatWidget { tx.send(AppEvent::OpenReviewCustomPrompt); })], dismiss_on_select: false, + dismiss_parent_on_child_accept: true, ..Default::default() }); From 6e72f0dbfd5e6b875e1ce00e8079e2bc15e28fe4 Mon Sep 17 00:00:00 2001 From: Tom Date: Thu, 16 Apr 2026 10:15:31 -0700 Subject: [PATCH 24/59] [codex] Add remote thread store implementation (#17826) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add a "remote" thread store implementation - Implement the remote thread store as a thin wrapper that makes grpc calls to a configurable service endpoint - Implement only the thread/list method to start - Encode the grpc method/param shape as protobufs in the remote implementation A wart: the proto generation script is an "example" binary target. This is an example target only because Cargo lets examples use dev-dependencies, which keeps tonic-prost-build out of the normal codex-thread-store dependency surface. A regular bin would either need to add proto generation deps as normal runtime deps, or use a feature-gated optional dep, which this repo’s manifest checks explicitly reject. --- MODULE.bazel.lock | 4 + codex-rs/Cargo.lock | 70 ++- codex-rs/Cargo.toml | 2 + codex-rs/thread-store/Cargo.toml | 12 +- .../thread-store/examples/generate-proto.rs | 14 + .../thread-store/scripts/generate-proto.sh | 38 ++ codex-rs/thread-store/src/lib.rs | 2 + codex-rs/thread-store/src/remote/AGENTS.md | 13 + codex-rs/thread-store/src/remote/helpers.rs | 255 ++++++++++ .../thread-store/src/remote/list_threads.rs | 243 +++++++++ codex-rs/thread-store/src/remote/mod.rs | 112 +++++ .../remote/proto/codex.thread_store.v1.proto | 86 ++++ .../src/remote/proto/codex.thread_store.v1.rs | 460 ++++++++++++++++++ 13 files changed, 1307 insertions(+), 4 deletions(-) create mode 100644 codex-rs/thread-store/examples/generate-proto.rs create mode 100755 codex-rs/thread-store/scripts/generate-proto.sh create mode 100644 codex-rs/thread-store/src/remote/AGENTS.md create mode 100644 codex-rs/thread-store/src/remote/helpers.rs create mode 100644 codex-rs/thread-store/src/remote/list_threads.rs create mode 100644 codex-rs/thread-store/src/remote/mod.rs create mode 100644 codex-rs/thread-store/src/remote/proto/codex.thread_store.v1.proto create mode 100644 codex-rs/thread-store/src/remote/proto/codex.thread_store.v1.rs diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 5d7572f461..ef80693c70 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -1203,9 +1203,11 @@ "process-wrap_9.0.1": "{\"dependencies\":[{\"name\":\"futures\",\"optional\":true,\"req\":\"^0.3.30\"},{\"name\":\"indexmap\",\"req\":\"^2.9.0\"},{\"default_features\":false,\"features\":[\"fs\",\"poll\",\"signal\"],\"name\":\"nix\",\"optional\":true,\"req\":\"^0.30.1\",\"target\":\"cfg(unix)\"},{\"kind\":\"dev\",\"name\":\"remoteprocess\",\"req\":\"^0.5.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.20.0\"},{\"features\":[\"io-util\",\"macros\",\"process\",\"rt\"],\"name\":\"tokio\",\"optional\":true,\"req\":\"^1.38.2\"},{\"features\":[\"io-util\",\"macros\",\"process\",\"rt\",\"rt-multi-thread\",\"time\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.38.2\"},{\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1.40\"},{\"name\":\"windows\",\"optional\":true,\"req\":\"^0.62.2\",\"target\":\"cfg(windows)\"}],\"features\":{\"creation-flags\":[\"dep:windows\",\"windows/Win32_System_Threading\"],\"default\":[\"creation-flags\",\"job-object\",\"kill-on-drop\",\"process-group\",\"process-session\",\"tracing\"],\"job-object\":[\"dep:windows\",\"windows/Win32_Security\",\"windows/Win32_System_Diagnostics_ToolHelp\",\"windows/Win32_System_IO\",\"windows/Win32_System_JobObjects\",\"windows/Win32_System_Threading\"],\"kill-on-drop\":[],\"process-group\":[],\"process-session\":[\"process-group\"],\"reset-sigmask\":[],\"std\":[\"dep:nix\"],\"tokio1\":[\"dep:nix\",\"dep:futures\",\"dep:tokio\"],\"tracing\":[\"dep:tracing\"]}}", "proptest_1.9.0": "{\"dependencies\":[{\"name\":\"bit-set\",\"optional\":true,\"req\":\"^0.8.0\"},{\"name\":\"bit-vec\",\"optional\":true,\"req\":\"^0.8.0\"},{\"name\":\"bitflags\",\"req\":\"^2.9\"},{\"default_features\":false,\"name\":\"num-traits\",\"req\":\"^0.2.15\"},{\"name\":\"proptest-macro\",\"optional\":true,\"req\":\"^0.4.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"rand\",\"req\":\"^0.9\"},{\"default_features\":false,\"name\":\"rand_chacha\",\"req\":\"^0.9\"},{\"name\":\"rand_xorshift\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.0\"},{\"name\":\"regex-syntax\",\"optional\":true,\"req\":\"^0.8\"},{\"default_features\":false,\"name\":\"rusty-fork\",\"optional\":true,\"req\":\"^0.3.0\"},{\"name\":\"tempfile\",\"optional\":true,\"req\":\"^3.0\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.112\"},{\"name\":\"unarray\",\"req\":\"^0.1.4\"},{\"name\":\"x86\",\"optional\":true,\"req\":\"^0.52.0\"}],\"features\":{\"alloc\":[],\"atomic64bit\":[],\"attr-macro\":[\"proptest-macro\"],\"bit-set\":[\"dep:bit-set\",\"dep:bit-vec\"],\"default\":[\"std\",\"fork\",\"timeout\",\"bit-set\"],\"default-code-coverage\":[\"std\",\"fork\",\"timeout\",\"bit-set\"],\"fork\":[\"std\",\"rusty-fork\",\"tempfile\"],\"handle-panics\":[\"std\"],\"hardware-rng\":[\"x86\"],\"no_std\":[\"num-traits/libm\"],\"std\":[\"rand/std\",\"rand/os_rng\",\"regex-syntax\",\"num-traits/std\"],\"timeout\":[\"fork\",\"rusty-fork/timeout\"],\"unstable\":[]}}", "prost-build_0.12.6": "{\"dependencies\":[{\"default_features\":false,\"name\":\"bytes\",\"req\":\"^1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.10\"},{\"name\":\"heck\",\"req\":\">=0.4, <=0.5\"},{\"default_features\":false,\"features\":[\"use_alloc\"],\"name\":\"itertools\",\"req\":\">=0.10, <=0.12\"},{\"name\":\"log\",\"req\":\"^0.4.4\"},{\"default_features\":false,\"name\":\"multimap\",\"req\":\">=0.8, <=0.10\"},{\"name\":\"once_cell\",\"req\":\"^1.17.1\"},{\"default_features\":false,\"name\":\"petgraph\",\"req\":\"^0.6\"},{\"name\":\"prettyplease\",\"optional\":true,\"req\":\"^0.2\"},{\"default_features\":false,\"name\":\"prost\",\"req\":\"^0.12.6\"},{\"default_features\":false,\"name\":\"prost-types\",\"req\":\"^0.12.6\"},{\"default_features\":false,\"name\":\"pulldown-cmark\",\"optional\":true,\"req\":\"^0.9.1\"},{\"name\":\"pulldown-cmark-to-cmark\",\"optional\":true,\"req\":\"^10.0.1\"},{\"default_features\":false,\"features\":[\"std\",\"unicode-bool\"],\"name\":\"regex\",\"req\":\"^1.8.1\"},{\"features\":[\"full\"],\"name\":\"syn\",\"optional\":true,\"req\":\"^2\"},{\"name\":\"tempfile\",\"req\":\"^3\"}],\"features\":{\"cleanup-markdown\":[\"dep:pulldown-cmark\",\"dep:pulldown-cmark-to-cmark\"],\"default\":[\"format\"],\"format\":[\"dep:prettyplease\",\"dep:syn\"]}}", + "prost-build_0.14.3": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"name\":\"heck\",\"req\":\">=0.4, <=0.5\"},{\"default_features\":false,\"features\":[\"use_alloc\"],\"name\":\"itertools\",\"req\":\">=0.10, <=0.14\"},{\"name\":\"log\",\"req\":\"^0.4.4\"},{\"default_features\":false,\"name\":\"multimap\",\"req\":\">=0.8, <=0.10\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"petgraph\",\"req\":\"^0.8\"},{\"name\":\"prettyplease\",\"optional\":true,\"req\":\"^0.2\"},{\"default_features\":false,\"name\":\"prost\",\"req\":\"^0.14.3\"},{\"default_features\":false,\"name\":\"prost-types\",\"req\":\"^0.14.3\"},{\"default_features\":false,\"name\":\"pulldown-cmark\",\"optional\":true,\"req\":\"^0.13\"},{\"name\":\"pulldown-cmark-to-cmark\",\"optional\":true,\"req\":\"^22\"},{\"default_features\":false,\"features\":[\"std\",\"unicode-bool\"],\"name\":\"regex\",\"req\":\"^1.8.1\"},{\"features\":[\"full\"],\"name\":\"syn\",\"optional\":true,\"req\":\"^2\"},{\"name\":\"tempfile\",\"req\":\"^3\"}],\"features\":{\"cleanup-markdown\":[\"dep:pulldown-cmark\",\"dep:pulldown-cmark-to-cmark\"],\"default\":[\"format\"],\"format\":[\"dep:prettyplease\",\"dep:syn\"]}}", "prost-derive_0.12.6": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.1\"},{\"default_features\":false,\"features\":[\"use_alloc\"],\"name\":\"itertools\",\"req\":\">=0.10, <=0.12\"},{\"name\":\"proc-macro2\",\"req\":\"^1\"},{\"name\":\"quote\",\"req\":\"^1\"},{\"features\":[\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2\"}],\"features\":{}}", "prost-derive_0.14.3": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.1\"},{\"name\":\"itertools\",\"req\":\">=0.10.1, <=0.14\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"name\":\"quote\",\"req\":\"^1\"},{\"features\":[\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2\"}],\"features\":{}}", "prost-types_0.12.6": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"prost-derive\"],\"name\":\"prost\",\"req\":\"^0.12.6\"}],\"features\":{\"default\":[\"std\"],\"std\":[\"prost/std\"]}}", + "prost-types_0.14.3": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.4\"},{\"default_features\":false,\"name\":\"chrono\",\"optional\":true,\"req\":\"^0.4.34\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"derive\"],\"name\":\"prost\",\"req\":\"^0.14.3\"}],\"features\":{\"arbitrary\":[\"dep:arbitrary\"],\"default\":[\"std\"],\"std\":[\"prost/std\"]}}", "prost_0.12.6": "{\"dependencies\":[{\"default_features\":false,\"name\":\"bytes\",\"req\":\"^1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1\"},{\"name\":\"prost-derive\",\"optional\":true,\"req\":\"^0.12.6\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"default\":[\"derive\",\"std\"],\"derive\":[\"dep:prost-derive\"],\"no-recursion-limit\":[],\"prost-derive\":[\"derive\"],\"std\":[]}}", "prost_0.14.3": "{\"dependencies\":[{\"default_features\":false,\"name\":\"bytes\",\"req\":\"^1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.7\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1\"},{\"name\":\"prost-derive\",\"optional\":true,\"req\":\"^0.14.3\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9\"}],\"features\":{\"default\":[\"derive\",\"std\"],\"derive\":[\"dep:prost-derive\"],\"no-recursion-limit\":[],\"std\":[]}}", "psl-types_2.0.11": "{\"dependencies\":[],\"features\":{}}", @@ -1459,6 +1461,8 @@ "toml_parser_1.0.6+spec-1.1.0": "{\"dependencies\":[{\"name\":\"anstream\",\"optional\":true,\"req\":\"^0.6.20\"},{\"features\":[\"test\"],\"kind\":\"dev\",\"name\":\"anstream\",\"req\":\"^0.6.20\"},{\"name\":\"anstyle\",\"optional\":true,\"req\":\"^1.0.11\"},{\"kind\":\"dev\",\"name\":\"snapbox\",\"req\":\"^0.6.21\"},{\"default_features\":false,\"name\":\"winnow\",\"req\":\"^0.7.13\"}],\"features\":{\"alloc\":[],\"debug\":[\"std\",\"dep:anstream\",\"dep:anstyle\"],\"default\":[\"std\"],\"simd\":[\"winnow/simd\"],\"std\":[\"alloc\"],\"unsafe\":[]}}", "toml_parser_1.0.9+spec-1.1.0": "{\"dependencies\":[{\"name\":\"anstream\",\"optional\":true,\"req\":\"^0.6.20\"},{\"features\":[\"test\"],\"kind\":\"dev\",\"name\":\"anstream\",\"req\":\"^0.6.20\"},{\"name\":\"anstyle\",\"optional\":true,\"req\":\"^1.0.11\"},{\"kind\":\"dev\",\"name\":\"snapbox\",\"req\":\"^0.6.21\"},{\"default_features\":false,\"name\":\"winnow\",\"req\":\"^0.7.13\"}],\"features\":{\"alloc\":[],\"debug\":[\"std\",\"dep:anstream\",\"dep:anstyle\"],\"default\":[\"std\"],\"simd\":[\"winnow/simd\"],\"std\":[\"alloc\"],\"unsafe\":[]}}", "toml_writer_1.0.6+spec-1.1.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1.7.0\"},{\"kind\":\"dev\",\"name\":\"snapbox\",\"req\":\"^0.6.21\"},{\"kind\":\"dev\",\"name\":\"toml_old\",\"package\":\"toml\",\"req\":\"^0.5.11\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[\"alloc\"]}}", + "tonic-build_0.14.3": "{\"dependencies\":[{\"name\":\"prettyplease\",\"req\":\"^0.2\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0\"},{\"name\":\"quote\",\"req\":\"^1.0\"},{\"name\":\"syn\",\"req\":\"^2.0\"}],\"features\":{\"default\":[\"transport\"],\"transport\":[]}}", + "tonic-prost-build_0.14.3": "{\"dependencies\":[{\"name\":\"prettyplease\",\"req\":\"^0.2\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0\"},{\"name\":\"prost-build\",\"req\":\"^0.14\"},{\"name\":\"prost-types\",\"req\":\"^0.14\"},{\"name\":\"quote\",\"req\":\"^1.0\"},{\"name\":\"syn\",\"req\":\"^2.0\"},{\"name\":\"tempfile\",\"req\":\"^3.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"tonic\",\"req\":\"^0.14.0\"},{\"default_features\":false,\"name\":\"tonic-build\",\"req\":\"^0.14.0\"}],\"features\":{\"cleanup-markdown\":[\"prost-build/cleanup-markdown\"],\"default\":[\"transport\",\"cleanup-markdown\"],\"transport\":[\"tonic-build/transport\"]}}", "tonic-prost_0.14.3": "{\"dependencies\":[{\"name\":\"bytes\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"http-body\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"http-body-util\",\"req\":\"^0.1\"},{\"name\":\"prost\",\"req\":\"^0.14\"},{\"features\":[\"macros\",\"rt-multi-thread\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tokio-stream\",\"req\":\"^0.1\"},{\"default_features\":false,\"name\":\"tonic\",\"req\":\"^0.14.0\"}],\"features\":{}}", "tonic_0.14.3": "{\"dependencies\":[{\"name\":\"async-trait\",\"optional\":true,\"req\":\"^0.1.13\"},{\"default_features\":false,\"name\":\"axum\",\"optional\":true,\"req\":\"^0.8\"},{\"name\":\"base64\",\"req\":\"^0.22\"},{\"kind\":\"dev\",\"name\":\"bencher\",\"req\":\"^0.1.5\"},{\"name\":\"bytes\",\"req\":\"^1.0\"},{\"name\":\"flate2\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"h2\",\"optional\":true,\"req\":\"^0.4\"},{\"name\":\"http\",\"req\":\"^1.1.0\"},{\"name\":\"http-body\",\"req\":\"^1\"},{\"name\":\"http-body-util\",\"req\":\"^0.1\"},{\"features\":[\"http1\",\"http2\"],\"name\":\"hyper\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"hyper-timeout\",\"optional\":true,\"req\":\"^0.5\"},{\"features\":[\"tokio\"],\"name\":\"hyper-util\",\"optional\":true,\"req\":\"^0.1.11\"},{\"name\":\"percent-encoding\",\"req\":\"^2.1\"},{\"name\":\"pin-project\",\"req\":\"^1.0.11\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"quickcheck_macros\",\"req\":\"^1.0\"},{\"name\":\"rustls-native-certs\",\"optional\":true,\"req\":\"^0.8\"},{\"features\":[\"all\"],\"name\":\"socket2\",\"optional\":true,\"req\":\"^0.6\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.0\"},{\"name\":\"sync_wrapper\",\"req\":\"^1.0.2\"},{\"default_features\":false,\"name\":\"tokio\",\"optional\":true,\"req\":\"^1\"},{\"features\":[\"rt-multi-thread\",\"macros\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"logging\",\"tls12\"],\"name\":\"tokio-rustls\",\"optional\":true,\"req\":\"^0.26.1\"},{\"default_features\":false,\"name\":\"tokio-stream\",\"req\":\"^0.1.16\"},{\"default_features\":false,\"name\":\"tower\",\"optional\":true,\"req\":\"^0.5\"},{\"features\":[\"load-shed\",\"timeout\"],\"kind\":\"dev\",\"name\":\"tower\",\"req\":\"^0.5\"},{\"name\":\"tower-layer\",\"req\":\"^0.3\"},{\"name\":\"tower-service\",\"req\":\"^0.3\"},{\"name\":\"tracing\",\"req\":\"^0.1\"},{\"name\":\"webpki-roots\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"zstd\",\"optional\":true,\"req\":\"^0.13.0\"}],\"features\":{\"_tls-any\":[\"dep:tokio\",\"tokio?/rt\",\"tokio?/macros\",\"tls-connect-info\"],\"channel\":[\"dep:hyper\",\"hyper?/client\",\"dep:hyper-util\",\"hyper-util?/client-legacy\",\"dep:tower\",\"tower?/balance\",\"tower?/buffer\",\"tower?/discover\",\"tower?/limit\",\"tower?/load-shed\",\"tower?/util\",\"dep:tokio\",\"tokio?/time\",\"dep:hyper-timeout\"],\"codegen\":[\"dep:async-trait\"],\"default\":[\"router\",\"transport\",\"codegen\"],\"deflate\":[\"dep:flate2\"],\"gzip\":[\"dep:flate2\"],\"router\":[\"dep:axum\",\"dep:tower\",\"tower?/util\"],\"server\":[\"dep:h2\",\"dep:hyper\",\"hyper?/server\",\"dep:hyper-util\",\"hyper-util?/service\",\"hyper-util?/server-auto\",\"dep:socket2\",\"dep:tokio\",\"tokio?/macros\",\"tokio?/net\",\"tokio?/time\",\"tokio-stream/net\",\"dep:tower\",\"tower?/util\",\"tower?/limit\",\"tower?/load-shed\"],\"tls-aws-lc\":[\"_tls-any\",\"tokio-rustls/aws-lc-rs\"],\"tls-connect-info\":[\"dep:tokio-rustls\"],\"tls-native-roots\":[\"_tls-any\",\"channel\",\"dep:rustls-native-certs\"],\"tls-ring\":[\"_tls-any\",\"tokio-rustls/ring\"],\"tls-webpki-roots\":[\"_tls-any\",\"channel\",\"dep:webpki-roots\"],\"transport\":[\"server\",\"channel\"],\"zstd\":[\"dep:zstd\"]}}", "tower-http_0.6.8": "{\"dependencies\":[{\"features\":[\"tokio\"],\"name\":\"async-compression\",\"optional\":true,\"req\":\"^0.4\"},{\"name\":\"base64\",\"optional\":true,\"req\":\"^0.22\"},{\"name\":\"bitflags\",\"req\":\"^2.0.2\"},{\"kind\":\"dev\",\"name\":\"brotli\",\"req\":\"^8\"},{\"name\":\"bytes\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"bytes\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"futures-core\",\"optional\":true,\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3.14\"},{\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3.14\"},{\"name\":\"http\",\"req\":\"^1.0\"},{\"name\":\"http-body\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"http-body\",\"req\":\"^1.0.0\"},{\"name\":\"http-body-util\",\"optional\":true,\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"http-body-util\",\"req\":\"^0.1.0\"},{\"name\":\"http-range-header\",\"optional\":true,\"req\":\"^0.4.0\"},{\"name\":\"httpdate\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"client-legacy\",\"http1\",\"tokio\"],\"kind\":\"dev\",\"name\":\"hyper-util\",\"req\":\"^0.1\"},{\"name\":\"iri-string\",\"optional\":true,\"req\":\"^0.7.0\"},{\"default_features\":false,\"name\":\"mime\",\"optional\":true,\"req\":\"^0.3.17\"},{\"default_features\":false,\"name\":\"mime_guess\",\"optional\":true,\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1\"},{\"name\":\"percent-encoding\",\"optional\":true,\"req\":\"^2.1.0\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.7\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"sync_wrapper\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"tokio\",\"optional\":true,\"req\":\"^1.6\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"io\"],\"name\":\"tokio-util\",\"optional\":true,\"req\":\"^0.7\"},{\"name\":\"tower\",\"optional\":true,\"req\":\"^0.5\"},{\"features\":[\"buffer\",\"util\",\"retry\",\"make\",\"timeout\"],\"kind\":\"dev\",\"name\":\"tower\",\"req\":\"^0.5\"},{\"name\":\"tower-layer\",\"req\":\"^0.3.3\"},{\"name\":\"tower-service\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3\"},{\"features\":[\"v4\"],\"name\":\"uuid\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"zstd\",\"req\":\"^0.13\"}],\"features\":{\"add-extension\":[],\"auth\":[\"base64\",\"validate-request\"],\"catch-panic\":[\"tracing\",\"futures-util/std\",\"dep:http-body\",\"dep:http-body-util\"],\"compression-br\":[\"async-compression/brotli\",\"futures-core\",\"dep:http-body\",\"tokio-util\",\"tokio\"],\"compression-deflate\":[\"async-compression/zlib\",\"futures-core\",\"dep:http-body\",\"tokio-util\",\"tokio\"],\"compression-full\":[\"compression-br\",\"compression-deflate\",\"compression-gzip\",\"compression-zstd\"],\"compression-gzip\":[\"async-compression/gzip\",\"futures-core\",\"dep:http-body\",\"tokio-util\",\"tokio\"],\"compression-zstd\":[\"async-compression/zstd\",\"futures-core\",\"dep:http-body\",\"tokio-util\",\"tokio\"],\"cors\":[],\"decompression-br\":[\"async-compression/brotli\",\"futures-core\",\"dep:http-body\",\"dep:http-body-util\",\"tokio-util\",\"tokio\"],\"decompression-deflate\":[\"async-compression/zlib\",\"futures-core\",\"dep:http-body\",\"dep:http-body-util\",\"tokio-util\",\"tokio\"],\"decompression-full\":[\"decompression-br\",\"decompression-deflate\",\"decompression-gzip\",\"decompression-zstd\"],\"decompression-gzip\":[\"async-compression/gzip\",\"futures-core\",\"dep:http-body\",\"dep:http-body-util\",\"tokio-util\",\"tokio\"],\"decompression-zstd\":[\"async-compression/zstd\",\"futures-core\",\"dep:http-body\",\"dep:http-body-util\",\"tokio-util\",\"tokio\"],\"default\":[],\"follow-redirect\":[\"futures-util\",\"dep:http-body\",\"iri-string\",\"tower/util\"],\"fs\":[\"futures-core\",\"futures-util\",\"dep:http-body\",\"dep:http-body-util\",\"tokio/fs\",\"tokio-util/io\",\"tokio/io-util\",\"dep:http-range-header\",\"mime_guess\",\"mime\",\"percent-encoding\",\"httpdate\",\"set-status\",\"futures-util/alloc\",\"tracing\"],\"full\":[\"add-extension\",\"auth\",\"catch-panic\",\"compression-full\",\"cors\",\"decompression-full\",\"follow-redirect\",\"fs\",\"limit\",\"map-request-body\",\"map-response-body\",\"metrics\",\"normalize-path\",\"propagate-header\",\"redirect\",\"request-id\",\"sensitive-headers\",\"set-header\",\"set-status\",\"timeout\",\"trace\",\"util\",\"validate-request\"],\"limit\":[\"dep:http-body\",\"dep:http-body-util\"],\"map-request-body\":[],\"map-response-body\":[],\"metrics\":[\"dep:http-body\",\"tokio/time\"],\"normalize-path\":[],\"propagate-header\":[],\"redirect\":[],\"request-id\":[\"uuid\"],\"sensitive-headers\":[],\"set-header\":[],\"set-status\":[],\"timeout\":[\"dep:http-body\",\"tokio/time\"],\"trace\":[\"dep:http-body\",\"tracing\"],\"util\":[\"tower\"],\"validate-request\":[\"mime\"]}}", diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 1be5b45b6a..3e21398f85 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -2886,11 +2886,16 @@ dependencies = [ "codex-rollout", "codex-state", "pretty_assertions", + "prost 0.14.3", "serde", "serde_json", "tempfile", "thiserror 2.0.18", "tokio", + "tokio-stream", + "tonic", + "tonic-prost", + "tonic-prost-build", "uuid", ] @@ -7561,7 +7566,7 @@ dependencies = [ "heck 0.4.1", "itertools 0.11.0", "prost 0.12.6", - "prost-types", + "prost-types 0.12.6", ] [[package]] @@ -7575,7 +7580,7 @@ dependencies = [ "pbjson", "pbjson-build", "prost 0.12.6", - "prost-build", + "prost-build 0.12.6", "serde", ] @@ -7999,7 +8004,26 @@ dependencies = [ "petgraph 0.6.5", "prettyplease", "prost 0.12.6", - "prost-types", + "prost-types 0.12.6", + "regex", + "syn 2.0.114", + "tempfile", +] + +[[package]] +name = "prost-build" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" +dependencies = [ + "heck 0.5.0", + "itertools 0.14.0", + "log", + "multimap", + "petgraph 0.8.3", + "prettyplease", + "prost 0.14.3", + "prost-types 0.14.3", "regex", "syn 2.0.114", "tempfile", @@ -8040,6 +8064,15 @@ dependencies = [ "prost 0.12.6", ] +[[package]] +name = "prost-types" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +dependencies = [ + "prost 0.14.3", +] + [[package]] name = "psl" version = "2.1.184" @@ -10948,8 +10981,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a286e33f82f8a1ee2df63f4fa35c0becf4a85a0cb03091a15fd7bf0b402dc94a" dependencies = [ "async-trait", + "axum", "base64 0.22.1", "bytes", + "h2", "http 1.4.0", "http-body", "http-body-util", @@ -10959,6 +10994,7 @@ dependencies = [ "percent-encoding", "pin-project", "rustls-native-certs", + "socket2 0.6.2", "sync_wrapper", "tokio", "tokio-rustls", @@ -10969,6 +11005,18 @@ dependencies = [ "tracing", ] +[[package]] +name = "tonic-build" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27aac809edf60b741e2d7db6367214d078856b8a5bff0087e94ff330fb97b6fc" +dependencies = [ + "prettyplease", + "proc-macro2", + "quote", + "syn 2.0.114", +] + [[package]] name = "tonic-prost" version = "0.14.3" @@ -10980,6 +11028,22 @@ dependencies = [ "tonic", ] +[[package]] +name = "tonic-prost-build" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4556786613791cfef4ed134aa670b61a85cfcacf71543ef33e8d801abae988f" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build 0.14.3", + "prost-types 0.14.3", + "quote", + "syn 2.0.114", + "tempfile", + "tonic-build", +] + [[package]] name = "tower" version = "0.5.3" diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index e7867304e9..deb3738812 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -347,6 +347,8 @@ tracing-appender = "0.2.3" tracing-opentelemetry = "0.32.0" tracing-subscriber = "0.3.22" tracing-test = "0.2.5" +tonic = { version = "0.14.3", default-features = false, features = ["channel", "codegen"] } +tonic-prost = "0.14.3" tree-sitter = "0.25.10" tree-sitter-bash = "0.25" ts-rs = "11" diff --git a/codex-rs/thread-store/Cargo.toml b/codex-rs/thread-store/Cargo.toml index 328d9a4a7f..3698332b92 100644 --- a/codex-rs/thread-store/Cargo.toml +++ b/codex-rs/thread-store/Cargo.toml @@ -8,6 +8,10 @@ version.workspace = true name = "codex_thread_store" path = "src/lib.rs" +[[example]] +name = "generate-proto" +path = "examples/generate-proto.rs" + [lints] workspace = true @@ -17,13 +21,19 @@ chrono = { workspace = true, features = ["serde"] } codex-git-utils = { workspace = true } codex-protocol = { workspace = true } codex-rollout = { workspace = true } +prost = "0.14.3" serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } thiserror = { workspace = true } +tonic = { workspace = true } +tonic-prost = { workspace = true } [dev-dependencies] codex-state = { workspace = true } pretty_assertions = { workspace = true } tempfile = { workspace = true } -tokio = { workspace = true, features = ["macros"] } +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } +tokio-stream = { workspace = true, features = ["net"] } +tonic = { workspace = true, features = ["router", "transport"] } +tonic-prost-build = { version = "=0.14.3", default-features = false, features = ["transport"] } uuid = { workspace = true } diff --git a/codex-rs/thread-store/examples/generate-proto.rs b/codex-rs/thread-store/examples/generate-proto.rs new file mode 100644 index 0000000000..aa159f5bcb --- /dev/null +++ b/codex-rs/thread-store/examples/generate-proto.rs @@ -0,0 +1,14 @@ +use std::path::PathBuf; + +fn main() -> Result<(), Box> { + let proto_dir = PathBuf::from(std::env::args().nth(1).expect("proto dir")); + let proto_file = proto_dir.join("codex.thread_store.v1.proto"); + + tonic_prost_build::configure() + .build_client(true) + .build_server(true) + .out_dir(&proto_dir) + .compile_protos(&[proto_file], &[proto_dir])?; + + Ok(()) +} diff --git a/codex-rs/thread-store/scripts/generate-proto.sh b/codex-rs/thread-store/scripts/generate-proto.sh new file mode 100755 index 0000000000..4045467cac --- /dev/null +++ b/codex-rs/thread-store/scripts/generate-proto.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +repo_root="$(cd "$script_dir/../../.." && pwd)" +proto_dir="$repo_root/codex-rs/thread-store/src/remote/proto" +generated="$proto_dir/codex.thread_store.v1.rs" +tmpdir="$(mktemp -d)" + +cleanup() { + rm -rf "$tmpdir" +} +trap cleanup EXIT + +( + cd "$repo_root/codex-rs" + CARGO_TARGET_DIR="$tmpdir/target" cargo run \ + -p codex-thread-store \ + --example generate-proto \ + -- "$proto_dir" +) + +if ! sed -n '2p' "$generated" | grep -q 'clippy::trivially_copy_pass_by_ref'; then + { + sed -n '1p' "$generated" + printf '#![allow(clippy::trivially_copy_pass_by_ref)]\n' + sed '1d' "$generated" + } > "$tmpdir/generated.rs" + mv "$tmpdir/generated.rs" "$generated" +fi + +rustfmt --edition 2024 "$generated" + +awk ' + NR == 3 && previous ~ /clippy::trivially_copy_pass_by_ref/ && $0 != "" { print "" } + { print; previous = $0 } +' "$generated" > "$tmpdir/formatted.rs" +mv "$tmpdir/formatted.rs" "$generated" diff --git a/codex-rs/thread-store/src/lib.rs b/codex-rs/thread-store/src/lib.rs index a64bf023be..f8dc94908a 100644 --- a/codex-rs/thread-store/src/lib.rs +++ b/codex-rs/thread-store/src/lib.rs @@ -7,6 +7,7 @@ mod error; mod local; mod recorder; +mod remote; mod store; mod types; @@ -14,6 +15,7 @@ pub use error::ThreadStoreError; pub use error::ThreadStoreResult; pub use local::LocalThreadStore; pub use recorder::ThreadRecorder; +pub use remote::RemoteThreadStore; pub use store::ThreadStore; pub use types::AppendThreadItemsParams; pub use types::ArchiveThreadParams; diff --git a/codex-rs/thread-store/src/remote/AGENTS.md b/codex-rs/thread-store/src/remote/AGENTS.md new file mode 100644 index 0000000000..b2b2b6417b --- /dev/null +++ b/codex-rs/thread-store/src/remote/AGENTS.md @@ -0,0 +1,13 @@ +# Remote Thread Store + +- The Rust protobuf output in `proto/codex.thread_store.v1.rs` is checked in. +- Do not add build-time protobuf generation to `codex-thread-store` unless the Bazel/Cargo story is intentionally changed. +- When `proto/codex.thread_store.v1.proto` changes, regenerate the Rust file manually and include both files in the same commit. + +Run this from the repository root: + +```sh +./codex-rs/thread-store/scripts/generate-proto.sh +``` + +The command requires `protoc` to be available on `PATH`. diff --git a/codex-rs/thread-store/src/remote/helpers.rs b/codex-rs/thread-store/src/remote/helpers.rs new file mode 100644 index 0000000000..c02889e168 --- /dev/null +++ b/codex-rs/thread-store/src/remote/helpers.rs @@ -0,0 +1,255 @@ +use std::path::PathBuf; +use std::str::FromStr; + +use chrono::DateTime; +use chrono::Utc; +use codex_git_utils::GitSha; +use codex_protocol::AgentPath; +use codex_protocol::ThreadId; +use codex_protocol::openai_models::ReasoningEffort; +use codex_protocol::protocol::AskForApproval; +use codex_protocol::protocol::GitInfo; +use codex_protocol::protocol::SandboxPolicy; +use codex_protocol::protocol::SessionSource; +use codex_protocol::protocol::SubAgentSource; + +use super::proto; +use crate::StoredThread; +use crate::ThreadSortKey; +use crate::ThreadStoreError; +use crate::ThreadStoreResult; + +pub(super) fn remote_status_to_error(status: tonic::Status) -> ThreadStoreError { + match status.code() { + tonic::Code::InvalidArgument => ThreadStoreError::InvalidRequest { + message: status.message().to_string(), + }, + tonic::Code::AlreadyExists | tonic::Code::FailedPrecondition | tonic::Code::Aborted => { + ThreadStoreError::Conflict { + message: status.message().to_string(), + } + } + _ => ThreadStoreError::Internal { + message: format!("remote thread store request failed: {status}"), + }, + } +} + +pub(super) fn proto_sort_key(sort_key: ThreadSortKey) -> proto::ThreadSortKey { + match sort_key { + ThreadSortKey::CreatedAt => proto::ThreadSortKey::CreatedAt, + ThreadSortKey::UpdatedAt => proto::ThreadSortKey::UpdatedAt, + } +} + +pub(super) fn proto_session_source(source: &SessionSource) -> proto::SessionSource { + match source { + SessionSource::Cli => proto_source(proto::SessionSourceKind::Cli), + SessionSource::VSCode => proto_source(proto::SessionSourceKind::Vscode), + SessionSource::Exec => proto_source(proto::SessionSourceKind::Exec), + SessionSource::Mcp => proto_source(proto::SessionSourceKind::AppServer), + SessionSource::Custom(custom) => proto::SessionSource { + kind: proto::SessionSourceKind::Custom.into(), + custom: Some(custom.clone()), + ..Default::default() + }, + SessionSource::SubAgent(SubAgentSource::Review) => { + proto_source(proto::SessionSourceKind::SubAgentReview) + } + SessionSource::SubAgent(SubAgentSource::Compact) => { + proto_source(proto::SessionSourceKind::SubAgentCompact) + } + SessionSource::SubAgent(SubAgentSource::ThreadSpawn { + parent_thread_id, + depth, + agent_path, + agent_nickname, + agent_role, + }) => proto::SessionSource { + kind: proto::SessionSourceKind::SubAgentThreadSpawn.into(), + sub_agent_parent_thread_id: Some(parent_thread_id.to_string()), + sub_agent_depth: Some(*depth), + sub_agent_path: agent_path.as_ref().map(|path| path.as_str().to_string()), + sub_agent_nickname: agent_nickname.clone(), + sub_agent_role: agent_role.clone(), + ..Default::default() + }, + SessionSource::SubAgent(SubAgentSource::MemoryConsolidation) => { + proto_source(proto::SessionSourceKind::SubAgentMemoryConsolidation) + } + SessionSource::SubAgent(SubAgentSource::Other(other)) => proto::SessionSource { + kind: proto::SessionSourceKind::SubAgentOther.into(), + sub_agent_other: Some(other.clone()), + ..Default::default() + }, + SessionSource::Unknown => proto_source(proto::SessionSourceKind::Unknown), + } +} + +fn proto_source(kind: proto::SessionSourceKind) -> proto::SessionSource { + proto::SessionSource { + kind: kind.into(), + ..Default::default() + } +} + +pub(super) fn stored_thread_from_proto( + thread: proto::StoredThread, +) -> ThreadStoreResult { + // Keep this mapping boring: the proto mirrors StoredThread for remote-readable + // summary fields, except for Rust domain types that cross gRPC as stable scalar + // values. Local-only fields such as rollout_path intentionally stay local. + let source = thread + .source + .as_ref() + .map(session_source_from_proto) + .transpose()? + .unwrap_or(SessionSource::Unknown); + let thread_id = ThreadId::from_string(&thread.thread_id).map_err(|err| { + ThreadStoreError::InvalidRequest { + message: format!("remote thread store returned invalid thread_id: {err}"), + } + })?; + let forked_from_id = thread + .forked_from_id + .as_deref() + .map(ThreadId::from_string) + .transpose() + .map_err(|err| ThreadStoreError::InvalidRequest { + message: format!("remote thread store returned invalid forked_from_id: {err}"), + })?; + + Ok(StoredThread { + thread_id, + rollout_path: None, + forked_from_id, + preview: thread.preview, + name: thread.name, + model_provider: thread.model_provider, + model: thread.model, + reasoning_effort: thread + .reasoning_effort + .as_deref() + .map(parse_reasoning_effort) + .transpose()?, + created_at: datetime_from_unix(thread.created_at)?, + updated_at: datetime_from_unix(thread.updated_at)?, + archived_at: thread.archived_at.map(datetime_from_unix).transpose()?, + cwd: PathBuf::from(thread.cwd), + cli_version: thread.cli_version, + source, + agent_nickname: thread.agent_nickname, + agent_role: thread.agent_role, + agent_path: thread.agent_path, + git_info: thread.git_info.map(git_info_from_proto), + approval_mode: AskForApproval::OnRequest, + sandbox_policy: SandboxPolicy::new_read_only_policy(), + token_usage: None, + first_user_message: thread.first_user_message, + history: None, + }) +} + +#[cfg(test)] +pub(super) fn stored_thread_to_proto(thread: StoredThread) -> proto::StoredThread { + proto::StoredThread { + thread_id: thread.thread_id.to_string(), + forked_from_id: thread.forked_from_id.map(|thread_id| thread_id.to_string()), + preview: thread.preview, + name: thread.name, + model_provider: thread.model_provider, + model: thread.model, + created_at: thread.created_at.timestamp(), + updated_at: thread.updated_at.timestamp(), + archived_at: thread.archived_at.map(|timestamp| timestamp.timestamp()), + cwd: thread.cwd.to_string_lossy().into_owned(), + cli_version: thread.cli_version, + source: Some(proto_session_source(&thread.source)), + git_info: thread.git_info.map(git_info_to_proto), + agent_nickname: thread.agent_nickname, + agent_role: thread.agent_role, + agent_path: thread.agent_path, + reasoning_effort: thread.reasoning_effort.map(|effort| effort.to_string()), + first_user_message: thread.first_user_message, + } +} + +fn datetime_from_unix(timestamp: i64) -> ThreadStoreResult> { + DateTime::from_timestamp(timestamp, 0).ok_or_else(|| ThreadStoreError::InvalidRequest { + message: format!("remote thread store returned invalid timestamp: {timestamp}"), + }) +} + +fn session_source_from_proto(source: &proto::SessionSource) -> ThreadStoreResult { + let kind = proto::SessionSourceKind::try_from(source.kind).unwrap_or_default(); + Ok(match kind { + proto::SessionSourceKind::Unknown => SessionSource::Unknown, + proto::SessionSourceKind::Cli => SessionSource::Cli, + proto::SessionSourceKind::Vscode => SessionSource::VSCode, + proto::SessionSourceKind::Exec => SessionSource::Exec, + proto::SessionSourceKind::AppServer => SessionSource::Mcp, + proto::SessionSourceKind::Custom => { + SessionSource::Custom(source.custom.clone().unwrap_or_default()) + } + proto::SessionSourceKind::SubAgentReview => SessionSource::SubAgent(SubAgentSource::Review), + proto::SessionSourceKind::SubAgentCompact => { + SessionSource::SubAgent(SubAgentSource::Compact) + } + proto::SessionSourceKind::SubAgentThreadSpawn => { + let parent_thread_id = source + .sub_agent_parent_thread_id + .as_deref() + .map(ThreadId::from_string) + .transpose() + .map_err(|err| ThreadStoreError::InvalidRequest { + message: format!( + "remote thread store returned invalid sub-agent parent thread id: {err}" + ), + })? + .ok_or_else(|| ThreadStoreError::InvalidRequest { + message: "remote thread store omitted sub-agent parent thread id".to_string(), + })?; + SessionSource::SubAgent(SubAgentSource::ThreadSpawn { + parent_thread_id, + depth: source.sub_agent_depth.unwrap_or_default(), + agent_path: source + .sub_agent_path + .clone() + .map(AgentPath::from_string) + .transpose() + .map_err(|message| ThreadStoreError::InvalidRequest { message })?, + agent_nickname: source.sub_agent_nickname.clone(), + agent_role: source.sub_agent_role.clone(), + }) + } + proto::SessionSourceKind::SubAgentMemoryConsolidation => { + SessionSource::SubAgent(SubAgentSource::MemoryConsolidation) + } + proto::SessionSourceKind::SubAgentOther => SessionSource::SubAgent(SubAgentSource::Other( + source.sub_agent_other.clone().unwrap_or_default(), + )), + }) +} + +fn git_info_from_proto(info: proto::GitInfo) -> GitInfo { + GitInfo { + commit_hash: info.sha.as_deref().map(GitSha::new), + branch: info.branch, + repository_url: info.origin_url, + } +} + +#[cfg(test)] +fn git_info_to_proto(info: GitInfo) -> proto::GitInfo { + proto::GitInfo { + sha: info.commit_hash.map(|sha| sha.0), + branch: info.branch, + origin_url: info.repository_url, + } +} + +fn parse_reasoning_effort(value: &str) -> ThreadStoreResult { + ReasoningEffort::from_str(value).map_err(|message| ThreadStoreError::InvalidRequest { + message: format!("remote thread store returned {message}"), + }) +} diff --git a/codex-rs/thread-store/src/remote/list_threads.rs b/codex-rs/thread-store/src/remote/list_threads.rs new file mode 100644 index 0000000000..b94db9dc82 --- /dev/null +++ b/codex-rs/thread-store/src/remote/list_threads.rs @@ -0,0 +1,243 @@ +use super::RemoteThreadStore; +use super::helpers::proto_session_source; +use super::helpers::proto_sort_key; +use super::helpers::remote_status_to_error; +use super::helpers::stored_thread_from_proto; +use super::proto; +use crate::ListThreadsParams; +use crate::ThreadPage; +use crate::ThreadStoreError; +use crate::ThreadStoreResult; + +pub(super) async fn list_threads( + store: &RemoteThreadStore, + params: ListThreadsParams, +) -> ThreadStoreResult { + let request = proto::ListThreadsRequest { + page_size: params + .page_size + .try_into() + .map_err(|_| ThreadStoreError::InvalidRequest { + message: format!("page_size is too large: {}", params.page_size), + })?, + cursor: params.cursor, + sort_key: proto_sort_key(params.sort_key).into(), + allowed_sources: params + .allowed_sources + .iter() + .map(proto_session_source) + .collect(), + model_provider_filter: params + .model_providers + .map(|values| proto::ModelProviderFilter { values }), + archived: params.archived, + search_term: params.search_term, + }; + + let response = store + .client() + .await? + .list_threads(request) + .await + .map_err(remote_status_to_error)? + .into_inner(); + + let items = response + .threads + .into_iter() + .map(stored_thread_from_proto) + .collect::>>()?; + + Ok(ThreadPage { + items, + next_cursor: response.next_cursor, + }) +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use codex_protocol::openai_models::ReasoningEffort; + use codex_protocol::protocol::SessionSource; + use pretty_assertions::assert_eq; + use tonic::Request; + use tonic::Response; + use tonic::Status; + use tonic::transport::Server; + + use super::super::helpers::stored_thread_to_proto; + use super::super::proto::thread_store_server; + use super::super::proto::thread_store_server::ThreadStoreServer; + use super::*; + use crate::ThreadSortKey; + use crate::ThreadStore; + + #[derive(Default)] + struct TestServer; + + #[tonic::async_trait] + impl thread_store_server::ThreadStore for TestServer { + async fn list_threads( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + assert_eq!(request.page_size, 2); + assert_eq!(request.cursor.as_deref(), Some("cursor-1")); + assert_eq!( + proto::ThreadSortKey::try_from(request.sort_key), + Ok(proto::ThreadSortKey::UpdatedAt) + ); + assert_eq!(request.archived, true); + assert_eq!(request.search_term.as_deref(), Some("needle")); + assert_eq!( + request.model_provider_filter, + Some(proto::ModelProviderFilter { + values: vec!["openai".to_string()], + }) + ); + assert_eq!(request.allowed_sources.len(), 1); + assert_eq!( + proto::SessionSourceKind::try_from(request.allowed_sources[0].kind), + Ok(proto::SessionSourceKind::Cli) + ); + + Ok(Response::new(proto::ListThreadsResponse { + threads: vec![proto::StoredThread { + thread_id: "11111111-1111-1111-1111-111111111111".to_string(), + forked_from_id: None, + preview: "hello".to_string(), + name: Some("named thread".to_string()), + model_provider: "openai".to_string(), + model: Some("gpt-5".to_string()), + created_at: 100, + updated_at: 200, + archived_at: Some(300), + cwd: "/workspace".to_string(), + cli_version: "1.2.3".to_string(), + source: Some(proto::SessionSource { + kind: proto::SessionSourceKind::Cli.into(), + ..Default::default() + }), + git_info: Some(proto::GitInfo { + sha: Some("abc123".to_string()), + branch: Some("main".to_string()), + origin_url: Some("https://example.test/repo.git".to_string()), + }), + agent_nickname: None, + agent_role: None, + agent_path: None, + reasoning_effort: Some("medium".to_string()), + first_user_message: Some("hello".to_string()), + }], + next_cursor: Some("cursor-2".to_string()), + })) + } + } + + #[tokio::test] + async fn list_threads_calls_remote_service() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind test server"); + let addr = listener.local_addr().expect("test server addr"); + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); + let server = tokio::spawn(async move { + Server::builder() + .add_service(ThreadStoreServer::new(TestServer)) + .serve_with_incoming_shutdown( + tokio_stream::wrappers::TcpListenerStream::new(listener), + async { + let _ = shutdown_rx.await; + }, + ) + .await + }); + + let store = RemoteThreadStore::new(format!("http://{addr}")); + let page = store + .list_threads(ListThreadsParams { + page_size: 2, + cursor: Some("cursor-1".to_string()), + sort_key: ThreadSortKey::UpdatedAt, + allowed_sources: vec![SessionSource::Cli], + model_providers: Some(vec!["openai".to_string()]), + archived: true, + search_term: Some("needle".to_string()), + }) + .await + .expect("list threads"); + + assert_eq!(page.next_cursor.as_deref(), Some("cursor-2")); + assert_eq!(page.items.len(), 1); + let item = &page.items[0]; + assert_eq!( + item.thread_id.to_string(), + "11111111-1111-1111-1111-111111111111" + ); + assert_eq!(item.name.as_deref(), Some("named thread")); + assert_eq!(item.preview, "hello"); + assert_eq!(item.first_user_message.as_deref(), Some("hello")); + assert_eq!(item.model_provider, "openai"); + assert_eq!(item.model.as_deref(), Some("gpt-5")); + assert_eq!(item.created_at.timestamp(), 100); + assert_eq!(item.updated_at.timestamp(), 200); + assert_eq!(item.archived_at.map(|ts| ts.timestamp()), Some(300)); + assert_eq!(item.cwd, PathBuf::from("/workspace")); + assert_eq!(item.cli_version, "1.2.3"); + assert_eq!(item.source, SessionSource::Cli); + assert_eq!(item.reasoning_effort, Some(ReasoningEffort::Medium)); + assert_eq!( + item.git_info.as_ref().and_then(|git| git.branch.as_deref()), + Some("main") + ); + + let _ = shutdown_tx.send(()); + server.await.expect("join server").expect("server"); + } + + #[test] + fn stored_thread_proto_roundtrips_through_domain_type() { + let thread = proto::StoredThread { + thread_id: "11111111-1111-1111-1111-111111111111".to_string(), + forked_from_id: Some("22222222-2222-2222-2222-222222222222".to_string()), + preview: "preview text".to_string(), + name: Some("named thread".to_string()), + model_provider: "openai".to_string(), + model: Some("gpt-5".to_string()), + created_at: 100, + updated_at: 200, + archived_at: Some(300), + cwd: "/workspace/project".to_string(), + cli_version: "1.2.3".to_string(), + source: Some(proto::SessionSource { + kind: proto::SessionSourceKind::SubAgentThreadSpawn.into(), + sub_agent_parent_thread_id: Some( + "33333333-3333-3333-3333-333333333333".to_string(), + ), + sub_agent_depth: Some(2), + sub_agent_path: Some("/root/review/backend".to_string()), + sub_agent_nickname: Some("Navigator".to_string()), + sub_agent_role: Some("explorer".to_string()), + ..Default::default() + }), + git_info: Some(proto::GitInfo { + sha: Some("abc123".to_string()), + branch: Some("main".to_string()), + origin_url: Some("https://example.test/repo.git".to_string()), + }), + agent_nickname: Some("Navigator".to_string()), + agent_role: Some("explorer".to_string()), + agent_path: Some("/root/review/backend".to_string()), + reasoning_effort: Some("high".to_string()), + first_user_message: Some("first message".to_string()), + }; + + let stored = stored_thread_from_proto(thread.clone()).expect("proto to stored thread"); + + assert_eq!(stored.rollout_path, None); + assert!(stored.history.is_none()); + assert_eq!(stored_thread_to_proto(stored), thread); + } +} diff --git a/codex-rs/thread-store/src/remote/mod.rs b/codex-rs/thread-store/src/remote/mod.rs new file mode 100644 index 0000000000..2e8b9e5afb --- /dev/null +++ b/codex-rs/thread-store/src/remote/mod.rs @@ -0,0 +1,112 @@ +mod helpers; +mod list_threads; + +use async_trait::async_trait; + +use crate::AppendThreadItemsParams; +use crate::ArchiveThreadParams; +use crate::CreateThreadParams; +use crate::ListThreadsParams; +use crate::LoadThreadHistoryParams; +use crate::ReadThreadParams; +use crate::ResumeThreadRecorderParams; +use crate::SetThreadNameParams; +use crate::StoredThread; +use crate::StoredThreadHistory; +use crate::ThreadPage; +use crate::ThreadRecorder; +use crate::ThreadStore; +use crate::ThreadStoreError; +use crate::ThreadStoreResult; +use crate::UpdateThreadMetadataParams; +use proto::thread_store_client::ThreadStoreClient; + +#[path = "proto/codex.thread_store.v1.rs"] +mod proto; + +/// gRPC-backed [`ThreadStore`] implementation for deployments whose durable thread data lives +/// outside the app-server process. +#[derive(Clone, Debug)] +pub struct RemoteThreadStore { + endpoint: String, +} + +impl RemoteThreadStore { + pub fn new(endpoint: impl Into) -> Self { + Self { + endpoint: endpoint.into(), + } + } + + async fn client(&self) -> ThreadStoreResult> { + ThreadStoreClient::connect(self.endpoint.clone()) + .await + .map_err(|err| ThreadStoreError::Internal { + message: format!("failed to connect to remote thread store: {err}"), + }) + } +} + +#[async_trait] +impl ThreadStore for RemoteThreadStore { + async fn create_thread( + &self, + _params: CreateThreadParams, + ) -> ThreadStoreResult> { + Err(not_implemented("create_thread")) + } + + async fn resume_thread_recorder( + &self, + _params: ResumeThreadRecorderParams, + ) -> ThreadStoreResult> { + Err(not_implemented("resume_thread_recorder")) + } + + async fn append_items(&self, _params: AppendThreadItemsParams) -> ThreadStoreResult<()> { + Err(not_implemented("append_items")) + } + + async fn load_history( + &self, + _params: LoadThreadHistoryParams, + ) -> ThreadStoreResult { + Err(not_implemented("load_history")) + } + + async fn read_thread(&self, _params: ReadThreadParams) -> ThreadStoreResult { + Err(not_implemented("read_thread")) + } + + async fn list_threads(&self, params: ListThreadsParams) -> ThreadStoreResult { + list_threads::list_threads(self, params).await + } + + async fn set_thread_name(&self, _params: SetThreadNameParams) -> ThreadStoreResult<()> { + Err(not_implemented("set_thread_name")) + } + + async fn update_thread_metadata( + &self, + _params: UpdateThreadMetadataParams, + ) -> ThreadStoreResult { + Err(not_implemented("update_thread_metadata")) + } + + async fn archive_thread(&self, _params: ArchiveThreadParams) -> ThreadStoreResult<()> { + Err(not_implemented("archive_thread")) + } + + async fn unarchive_thread( + &self, + _params: ArchiveThreadParams, + ) -> ThreadStoreResult { + Err(not_implemented("unarchive_thread")) + } +} + +fn not_implemented(method: &str) -> ThreadStoreError { + ThreadStoreError::Internal { + message: format!("remote thread store does not implement {method} yet"), + } +} diff --git a/codex-rs/thread-store/src/remote/proto/codex.thread_store.v1.proto b/codex-rs/thread-store/src/remote/proto/codex.thread_store.v1.proto new file mode 100644 index 0000000000..f217027fcd --- /dev/null +++ b/codex-rs/thread-store/src/remote/proto/codex.thread_store.v1.proto @@ -0,0 +1,86 @@ +syntax = "proto3"; + +package codex.thread_store.v1; + +service ThreadStore { + rpc ListThreads(ListThreadsRequest) returns (ListThreadsResponse); +} + +message ListThreadsRequest { + uint32 page_size = 1; + optional string cursor = 2; + ThreadSortKey sort_key = 3; + repeated SessionSource allowed_sources = 4; + optional ModelProviderFilter model_provider_filter = 5; + bool archived = 6; + optional string search_term = 7; +} + +message ModelProviderFilter { + repeated string values = 1; +} + +enum ThreadSortKey { + THREAD_SORT_KEY_CREATED_AT = 0; + THREAD_SORT_KEY_UPDATED_AT = 1; +} + +message ListThreadsResponse { + repeated StoredThread threads = 1; + optional string next_cursor = 2; +} + +message StoredThread { + // Mirrors Rust's StoredThread. Domain types that are not protobuf-native, + // such as ThreadId, DateTime, and PathBuf, are represented as their + // stable scalar forms on the wire. + string thread_id = 1; + optional string forked_from_id = 2; + string preview = 3; + optional string name = 4; + string model_provider = 5; + optional string model = 6; + int64 created_at = 7; + int64 updated_at = 8; + optional int64 archived_at = 9; + string cwd = 10; + string cli_version = 11; + SessionSource source = 12; + optional GitInfo git_info = 13; + optional string agent_nickname = 14; + optional string agent_role = 15; + optional string agent_path = 16; + optional string reasoning_effort = 17; + optional string first_user_message = 18; +} + +message SessionSource { + SessionSourceKind kind = 1; + optional string custom = 2; + optional string sub_agent_parent_thread_id = 3; + optional int32 sub_agent_depth = 4; + optional string sub_agent_other = 5; + optional string sub_agent_path = 6; + optional string sub_agent_nickname = 7; + optional string sub_agent_role = 8; +} + +enum SessionSourceKind { + SESSION_SOURCE_KIND_UNKNOWN = 0; + SESSION_SOURCE_KIND_CLI = 1; + SESSION_SOURCE_KIND_VSCODE = 2; + SESSION_SOURCE_KIND_EXEC = 3; + SESSION_SOURCE_KIND_APP_SERVER = 4; + SESSION_SOURCE_KIND_CUSTOM = 5; + SESSION_SOURCE_KIND_SUB_AGENT_REVIEW = 6; + SESSION_SOURCE_KIND_SUB_AGENT_COMPACT = 7; + SESSION_SOURCE_KIND_SUB_AGENT_THREAD_SPAWN = 8; + SESSION_SOURCE_KIND_SUB_AGENT_MEMORY_CONSOLIDATION = 9; + SESSION_SOURCE_KIND_SUB_AGENT_OTHER = 10; +} + +message GitInfo { + optional string sha = 1; + optional string branch = 2; + optional string origin_url = 3; +} diff --git a/codex-rs/thread-store/src/remote/proto/codex.thread_store.v1.rs b/codex-rs/thread-store/src/remote/proto/codex.thread_store.v1.rs new file mode 100644 index 0000000000..6d0e410268 --- /dev/null +++ b/codex-rs/thread-store/src/remote/proto/codex.thread_store.v1.rs @@ -0,0 +1,460 @@ +// This file is @generated by prost-build. +#![allow(clippy::trivially_copy_pass_by_ref)] + +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ListThreadsRequest { + #[prost(uint32, tag = "1")] + pub page_size: u32, + #[prost(string, optional, tag = "2")] + pub cursor: ::core::option::Option<::prost::alloc::string::String>, + #[prost(enumeration = "ThreadSortKey", tag = "3")] + pub sort_key: i32, + #[prost(message, repeated, tag = "4")] + pub allowed_sources: ::prost::alloc::vec::Vec, + #[prost(message, optional, tag = "5")] + pub model_provider_filter: ::core::option::Option, + #[prost(bool, tag = "6")] + pub archived: bool, + #[prost(string, optional, tag = "7")] + pub search_term: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ModelProviderFilter { + #[prost(string, repeated, tag = "1")] + pub values: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ListThreadsResponse { + #[prost(message, repeated, tag = "1")] + pub threads: ::prost::alloc::vec::Vec, + #[prost(string, optional, tag = "2")] + pub next_cursor: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct StoredThread { + /// Mirrors Rust's StoredThread. Domain types that are not protobuf-native, + /// such as ThreadId, DateTime, and PathBuf, are represented as their + /// stable scalar forms on the wire. + #[prost(string, tag = "1")] + pub thread_id: ::prost::alloc::string::String, + #[prost(string, optional, tag = "2")] + pub forked_from_id: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, tag = "3")] + pub preview: ::prost::alloc::string::String, + #[prost(string, optional, tag = "4")] + pub name: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, tag = "5")] + pub model_provider: ::prost::alloc::string::String, + #[prost(string, optional, tag = "6")] + pub model: ::core::option::Option<::prost::alloc::string::String>, + #[prost(int64, tag = "7")] + pub created_at: i64, + #[prost(int64, tag = "8")] + pub updated_at: i64, + #[prost(int64, optional, tag = "9")] + pub archived_at: ::core::option::Option, + #[prost(string, tag = "10")] + pub cwd: ::prost::alloc::string::String, + #[prost(string, tag = "11")] + pub cli_version: ::prost::alloc::string::String, + #[prost(message, optional, tag = "12")] + pub source: ::core::option::Option, + #[prost(message, optional, tag = "13")] + pub git_info: ::core::option::Option, + #[prost(string, optional, tag = "14")] + pub agent_nickname: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "15")] + pub agent_role: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "16")] + pub agent_path: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "17")] + pub reasoning_effort: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "18")] + pub first_user_message: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct SessionSource { + #[prost(enumeration = "SessionSourceKind", tag = "1")] + pub kind: i32, + #[prost(string, optional, tag = "2")] + pub custom: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "3")] + pub sub_agent_parent_thread_id: ::core::option::Option<::prost::alloc::string::String>, + #[prost(int32, optional, tag = "4")] + pub sub_agent_depth: ::core::option::Option, + #[prost(string, optional, tag = "5")] + pub sub_agent_other: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "6")] + pub sub_agent_path: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "7")] + pub sub_agent_nickname: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "8")] + pub sub_agent_role: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct GitInfo { + #[prost(string, optional, tag = "1")] + pub sha: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "2")] + pub branch: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "3")] + pub origin_url: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum ThreadSortKey { + CreatedAt = 0, + UpdatedAt = 1, +} +impl ThreadSortKey { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::CreatedAt => "THREAD_SORT_KEY_CREATED_AT", + Self::UpdatedAt => "THREAD_SORT_KEY_UPDATED_AT", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "THREAD_SORT_KEY_CREATED_AT" => Some(Self::CreatedAt), + "THREAD_SORT_KEY_UPDATED_AT" => Some(Self::UpdatedAt), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum SessionSourceKind { + Unknown = 0, + Cli = 1, + Vscode = 2, + Exec = 3, + AppServer = 4, + Custom = 5, + SubAgentReview = 6, + SubAgentCompact = 7, + SubAgentThreadSpawn = 8, + SubAgentMemoryConsolidation = 9, + SubAgentOther = 10, +} +impl SessionSourceKind { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::Unknown => "SESSION_SOURCE_KIND_UNKNOWN", + Self::Cli => "SESSION_SOURCE_KIND_CLI", + Self::Vscode => "SESSION_SOURCE_KIND_VSCODE", + Self::Exec => "SESSION_SOURCE_KIND_EXEC", + Self::AppServer => "SESSION_SOURCE_KIND_APP_SERVER", + Self::Custom => "SESSION_SOURCE_KIND_CUSTOM", + Self::SubAgentReview => "SESSION_SOURCE_KIND_SUB_AGENT_REVIEW", + Self::SubAgentCompact => "SESSION_SOURCE_KIND_SUB_AGENT_COMPACT", + Self::SubAgentThreadSpawn => "SESSION_SOURCE_KIND_SUB_AGENT_THREAD_SPAWN", + Self::SubAgentMemoryConsolidation => { + "SESSION_SOURCE_KIND_SUB_AGENT_MEMORY_CONSOLIDATION" + } + Self::SubAgentOther => "SESSION_SOURCE_KIND_SUB_AGENT_OTHER", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "SESSION_SOURCE_KIND_UNKNOWN" => Some(Self::Unknown), + "SESSION_SOURCE_KIND_CLI" => Some(Self::Cli), + "SESSION_SOURCE_KIND_VSCODE" => Some(Self::Vscode), + "SESSION_SOURCE_KIND_EXEC" => Some(Self::Exec), + "SESSION_SOURCE_KIND_APP_SERVER" => Some(Self::AppServer), + "SESSION_SOURCE_KIND_CUSTOM" => Some(Self::Custom), + "SESSION_SOURCE_KIND_SUB_AGENT_REVIEW" => Some(Self::SubAgentReview), + "SESSION_SOURCE_KIND_SUB_AGENT_COMPACT" => Some(Self::SubAgentCompact), + "SESSION_SOURCE_KIND_SUB_AGENT_THREAD_SPAWN" => Some(Self::SubAgentThreadSpawn), + "SESSION_SOURCE_KIND_SUB_AGENT_MEMORY_CONSOLIDATION" => { + Some(Self::SubAgentMemoryConsolidation) + } + "SESSION_SOURCE_KIND_SUB_AGENT_OTHER" => Some(Self::SubAgentOther), + _ => None, + } + } +} +/// Generated client implementations. +pub mod thread_store_client { + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value + )] + use tonic::codegen::http::Uri; + use tonic::codegen::*; + #[derive(Debug, Clone)] + pub struct ThreadStoreClient { + inner: tonic::client::Grpc, + } + impl ThreadStoreClient { + /// Attempt to create a new client by connecting to a given endpoint. + pub async fn connect(dst: D) -> Result + where + D: TryInto, + D::Error: Into, + { + let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; + Ok(Self::new(conn)) + } + } + impl ThreadStoreClient + where + T: tonic::client::GrpcService, + T::Error: Into, + T::ResponseBody: Body + std::marker::Send + 'static, + ::Error: Into + std::marker::Send, + { + pub fn new(inner: T) -> Self { + let inner = tonic::client::Grpc::new(inner); + Self { inner } + } + pub fn with_origin(inner: T, origin: Uri) -> Self { + let inner = tonic::client::Grpc::with_origin(inner, origin); + Self { inner } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> ThreadStoreClient> + where + F: tonic::service::Interceptor, + T::ResponseBody: Default, + T: tonic::codegen::Service< + http::Request, + Response = http::Response< + >::ResponseBody, + >, + >, + >>::Error: + Into + std::marker::Send + std::marker::Sync, + { + ThreadStoreClient::new(InterceptedService::new(inner, interceptor)) + } + /// Compress requests with the given encoding. + /// + /// This requires the server to support it otherwise it might respond with an + /// error. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.send_compressed(encoding); + self + } + /// Enable decompressing responses. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.accept_compressed(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_decoding_message_size(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_encoding_message_size(limit); + self + } + pub async fn list_threads( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> + { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/codex.thread_store.v1.ThreadStore/ListThreads", + ); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new( + "codex.thread_store.v1.ThreadStore", + "ListThreads", + )); + self.inner.unary(req, path, codec).await + } + } +} +/// Generated server implementations. +pub mod thread_store_server { + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value + )] + use tonic::codegen::*; + /// Generated trait containing gRPC methods that should be implemented for use with ThreadStoreServer. + #[async_trait] + pub trait ThreadStore: std::marker::Send + std::marker::Sync + 'static { + async fn list_threads( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + } + #[derive(Debug)] + pub struct ThreadStoreServer { + inner: Arc, + accept_compression_encodings: EnabledCompressionEncodings, + send_compression_encodings: EnabledCompressionEncodings, + max_decoding_message_size: Option, + max_encoding_message_size: Option, + } + impl ThreadStoreServer { + pub fn new(inner: T) -> Self { + Self::from_arc(Arc::new(inner)) + } + pub fn from_arc(inner: Arc) -> Self { + Self { + inner, + accept_compression_encodings: Default::default(), + send_compression_encodings: Default::default(), + max_decoding_message_size: None, + max_encoding_message_size: None, + } + } + pub fn with_interceptor(inner: T, interceptor: F) -> InterceptedService + where + F: tonic::service::Interceptor, + { + InterceptedService::new(Self::new(inner), interceptor) + } + /// Enable decompressing requests with the given encoding. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.accept_compression_encodings.enable(encoding); + self + } + /// Compress responses with the given encoding, if the client supports it. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.send_compression_encodings.enable(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.max_decoding_message_size = Some(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.max_encoding_message_size = Some(limit); + self + } + } + impl tonic::codegen::Service> for ThreadStoreServer + where + T: ThreadStore, + B: Body + std::marker::Send + 'static, + B::Error: Into + std::marker::Send + 'static, + { + type Response = http::Response; + type Error = std::convert::Infallible; + type Future = BoxFuture; + fn poll_ready( + &mut self, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + fn call(&mut self, req: http::Request) -> Self::Future { + match req.uri().path() { + "/codex.thread_store.v1.ThreadStore/ListThreads" => { + #[allow(non_camel_case_types)] + struct ListThreadsSvc(pub Arc); + impl tonic::server::UnaryService for ListThreadsSvc { + type Response = super::ListThreadsResponse; + type Future = BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::list_threads(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = ListThreadsSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + _ => Box::pin(async move { + let mut response = http::Response::new(tonic::body::Body::default()); + let headers = response.headers_mut(); + headers.insert( + tonic::Status::GRPC_STATUS, + (tonic::Code::Unimplemented as i32).into(), + ); + headers.insert( + http::header::CONTENT_TYPE, + tonic::metadata::GRPC_CONTENT_TYPE, + ); + Ok(response) + }), + } + } + } + impl Clone for ThreadStoreServer { + fn clone(&self) -> Self { + let inner = self.inner.clone(); + Self { + inner, + accept_compression_encodings: self.accept_compression_encodings, + send_compression_encodings: self.send_compression_encodings, + max_decoding_message_size: self.max_decoding_message_size, + max_encoding_message_size: self.max_encoding_message_size, + } + } + } + /// Generated gRPC service name + pub const SERVICE_NAME: &str = "codex.thread_store.v1.ThreadStore"; + impl tonic::server::NamedService for ThreadStoreServer { + const NAME: &'static str = SERVICE_NAME; + } +} From 2ca270d08dfda06e44df579b86d62d96e388019d Mon Sep 17 00:00:00 2001 From: Ahmed Ibrahim Date: Thu, 16 Apr 2026 10:30:10 -0700 Subject: [PATCH 25/59] [2/8] Support piped stdin in exec process API (#18086) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Add an explicit stdin mode to process/start. - Keep normal non-interactive exec stdin closed while allowing pipe-backed processes. ## Stack ```text o #18027 [8/8] Fail exec client operations after disconnect │ o #18025 [7/8] Cover MCP stdio tests with executor placement │ o #18089 [6/8] Wire remote MCP stdio through executor │ o #18088 [5/8] Add executor process transport for MCP stdio │ o #18087 [4/8] Abstract MCP stdio server launching │ o #18020 [3/8] Add pushed exec process events │ @ #18086 [2/8] Support piped stdin in exec process API │ o #18085 [1/8] Add MCP server environment config │ o main ``` Co-authored-by: Codex --- .../core/src/unified_exec/process_manager.rs | 1 + codex-rs/exec-server/README.md | 13 +-- codex-rs/exec-server/src/environment.rs | 1 + codex-rs/exec-server/src/local_process.rs | 14 ++- codex-rs/exec-server/src/protocol.rs | 3 + .../exec-server/src/server/handler/tests.rs | 1 + codex-rs/exec-server/src/server/processor.rs | 1 + codex-rs/exec-server/tests/exec_process.rs | 91 +++++++++++++++++ codex-rs/exec-server/tests/process.rs | 97 +++++++++++++++++++ 9 files changed, 215 insertions(+), 7 deletions(-) diff --git a/codex-rs/core/src/unified_exec/process_manager.rs b/codex-rs/core/src/unified_exec/process_manager.rs index 5214892681..aaaa7076f4 100644 --- a/codex-rs/core/src/unified_exec/process_manager.rs +++ b/codex-rs/core/src/unified_exec/process_manager.rs @@ -152,6 +152,7 @@ fn exec_server_params_for_request( env_policy, env, tty, + pipe_stdin: false, arg0: request.arg0.clone(), } } diff --git a/codex-rs/exec-server/README.md b/codex-rs/exec-server/README.md index 0047449d39..78b92e1a71 100644 --- a/codex-rs/exec-server/README.md +++ b/codex-rs/exec-server/README.md @@ -85,6 +85,7 @@ Request params: "PATH": "/usr/bin:/bin" }, "tty": true, + "pipeStdin": false, "arg0": null } ``` @@ -95,8 +96,8 @@ Field definitions: - `argv`: command vector. It must be non-empty. - `cwd`: absolute working directory used for the child process. - `env`: environment variables passed to the child process. -- `tty`: when `true`, spawn a PTY-backed interactive process; when `false`, - spawn a pipe-backed process with closed stdin. +- `tty`: when `true`, spawn a PTY-backed interactive process. +- `pipeStdin`: when `true`, keep non-PTY stdin writable via `process/write`. - `arg0`: optional argv0 override forwarded to `codex-utils-pty`. Response: @@ -111,7 +112,7 @@ Behavior notes: - Reusing an existing `processId` is rejected. - PTY-backed processes accept later writes through `process/write`. -- Pipe-backed processes are launched with stdin closed and reject writes. +- Non-PTY processes reject writes unless `pipeStdin` is `true`. - Output is streamed asynchronously via `process/output`. - Exit is reported asynchronously via `process/exited`. @@ -153,7 +154,7 @@ Response: ### `process/write` -Writes raw bytes to a running PTY-backed process stdin. +Writes raw bytes to a running process stdin. Request params: @@ -177,7 +178,7 @@ Response: Behavior notes: - Writes to an unknown `processId` are rejected. -- Writes to a non-PTY process are rejected because stdin is already closed. +- Writes to a non-PTY process are rejected unless it started with `pipeStdin`. ### `process/terminate` @@ -325,7 +326,7 @@ Initialize: Start a process: ```json -{"id":2,"method":"process/start","params":{"processId":"proc-1","argv":["bash","-lc","printf 'ready\\n'; while IFS= read -r line; do printf 'echo:%s\\n' \"$line\"; done"],"cwd":"/tmp","env":{"PATH":"/usr/bin:/bin"},"tty":true,"arg0":null}} +{"id":2,"method":"process/start","params":{"processId":"proc-1","argv":["bash","-lc","printf 'ready\\n'; while IFS= read -r line; do printf 'echo:%s\\n' \"$line\"; done"],"cwd":"/tmp","env":{"PATH":"/usr/bin:/bin"},"tty":true,"pipeStdin":false,"arg0":null}} {"id":2,"result":{"processId":"proc-1"}} {"method":"process/output","params":{"processId":"proc-1","seq":1,"stream":"stdout","chunk":"cmVhZHkK"}} ``` diff --git a/codex-rs/exec-server/src/environment.rs b/codex-rs/exec-server/src/environment.rs index 77ead87a84..afe0720196 100644 --- a/codex-rs/exec-server/src/environment.rs +++ b/codex-rs/exec-server/src/environment.rs @@ -346,6 +346,7 @@ mod tests { env_policy: None, env: Default::default(), tty: false, + pipe_stdin: false, arg0: None, }) .await diff --git a/codex-rs/exec-server/src/local_process.rs b/codex-rs/exec-server/src/local_process.rs index bf38aa360b..2608f892b8 100644 --- a/codex-rs/exec-server/src/local_process.rs +++ b/codex-rs/exec-server/src/local_process.rs @@ -59,6 +59,7 @@ struct RetainedOutputChunk { struct RunningProcess { session: ExecCommandSession, tty: bool, + pipe_stdin: bool, output: VecDeque, retained_bytes: usize, next_seq: u64, @@ -165,6 +166,15 @@ impl LocalProcess { TerminalSize::default(), ) .await + } else if params.pipe_stdin { + codex_utils_pty::spawn_pipe_process( + program, + args, + params.cwd.as_path(), + &env, + ¶ms.arg0, + ) + .await } else { codex_utils_pty::spawn_pipe_process_no_stdin( program, @@ -195,6 +205,7 @@ impl LocalProcess { ProcessEntry::Running(Box::new(RunningProcess { session: spawned.session, tty: params.tty, + pipe_stdin: params.pipe_stdin, output: VecDeque::new(), retained_bytes: 0, next_seq: 1, @@ -339,7 +350,7 @@ impl LocalProcess { status: WriteStatus::Starting, }); }; - if !process.tty { + if !process.tty && !process.pipe_stdin { return Ok(WriteResponse { status: WriteStatus::StdinClosed, }); @@ -667,6 +678,7 @@ mod tests { env_policy: None, env, tty: false, + pipe_stdin: false, arg0: None, } } diff --git a/codex-rs/exec-server/src/protocol.rs b/codex-rs/exec-server/src/protocol.rs index 5d29348897..9f4234e649 100644 --- a/codex-rs/exec-server/src/protocol.rs +++ b/codex-rs/exec-server/src/protocol.rs @@ -69,6 +69,9 @@ pub struct ExecParams { pub env_policy: Option, pub env: HashMap, pub tty: bool, + /// Keep non-tty stdin writable through `process/write`. + #[serde(default)] + pub pipe_stdin: bool, pub arg0: Option, } diff --git a/codex-rs/exec-server/src/server/handler/tests.rs b/codex-rs/exec-server/src/server/handler/tests.rs index 321bf243a9..7f2cdba629 100644 --- a/codex-rs/exec-server/src/server/handler/tests.rs +++ b/codex-rs/exec-server/src/server/handler/tests.rs @@ -30,6 +30,7 @@ fn exec_params_with_argv(process_id: &str, argv: Vec) -> ExecParams { env_policy: None, env: inherited_path_env(), tty: false, + pipe_stdin: false, arg0: None, } } diff --git a/codex-rs/exec-server/src/server/processor.rs b/codex-rs/exec-server/src/server/processor.rs index 87b9704502..1153bc83e2 100644 --- a/codex-rs/exec-server/src/server/processor.rs +++ b/codex-rs/exec-server/src/server/processor.rs @@ -393,6 +393,7 @@ mod tests { env_policy: None, env, tty: false, + pipe_stdin: false, arg0: None, } } diff --git a/codex-rs/exec-server/tests/exec_process.rs b/codex-rs/exec-server/tests/exec_process.rs index 72f029231c..862e4e1c8b 100644 --- a/codex-rs/exec-server/tests/exec_process.rs +++ b/codex-rs/exec-server/tests/exec_process.rs @@ -12,6 +12,7 @@ use codex_exec_server::ExecProcess; use codex_exec_server::ProcessId; use codex_exec_server::ReadResponse; use codex_exec_server::StartedExecProcess; +use codex_exec_server::WriteStatus; use pretty_assertions::assert_eq; use test_case::test_case; use tokio::sync::watch; @@ -54,6 +55,7 @@ async fn assert_exec_process_starts_and_exits(use_remote: bool) -> Result<()> { env_policy: /*env_policy*/ None, env: Default::default(), tty: false, + pipe_stdin: false, arg0: None, }) .await?; @@ -131,6 +133,7 @@ async fn assert_exec_process_streams_output(use_remote: bool) -> Result<()> { env_policy: /*env_policy*/ None, env: Default::default(), tty: false, + pipe_stdin: false, arg0: None, }) .await?; @@ -164,6 +167,7 @@ async fn assert_exec_process_write_then_read(use_remote: bool) -> Result<()> { env_policy: /*env_policy*/ None, env: Default::default(), tty: true, + pipe_stdin: false, arg0: None, }) .await?; @@ -184,6 +188,73 @@ async fn assert_exec_process_write_then_read(use_remote: bool) -> Result<()> { Ok(()) } +async fn assert_exec_process_write_then_read_without_tty(use_remote: bool) -> Result<()> { + let context = create_process_context(use_remote).await?; + let process_id = "proc-stdin-pipe".to_string(); + let session = context + .backend + .start(ExecParams { + process_id: process_id.clone().into(), + argv: vec![ + "/bin/sh".to_string(), + "-c".to_string(), + "IFS= read line; printf 'from-stdin:%s\\n' \"$line\"".to_string(), + ], + cwd: std::env::current_dir()?, + env_policy: /*env_policy*/ None, + env: Default::default(), + tty: false, + pipe_stdin: true, + arg0: None, + }) + .await?; + assert_eq!(session.process.process_id().as_str(), process_id); + + tokio::time::sleep(Duration::from_millis(200)).await; + let write_response = session.process.write(b"hello\n".to_vec()).await?; + assert_eq!(write_response.status, WriteStatus::Accepted); + let StartedExecProcess { process } = session; + let wake_rx = process.subscribe_wake(); + let actual = collect_process_output_from_reads(process, wake_rx).await?; + + assert_eq!(actual, ("from-stdin:hello\n".to_string(), Some(0), true)); + Ok(()) +} + +async fn assert_exec_process_rejects_write_without_pipe_stdin(use_remote: bool) -> Result<()> { + let context = create_process_context(use_remote).await?; + let process_id = "proc-stdin-closed".to_string(); + let session = context + .backend + .start(ExecParams { + process_id: process_id.clone().into(), + argv: vec![ + "/bin/sh".to_string(), + "-c".to_string(), + "sleep 0.3; if IFS= read -r line; then printf 'read:%s\\n' \"$line\"; else printf 'eof\\n'; fi".to_string(), + ], + cwd: std::env::current_dir()?, + env_policy: /*env_policy*/ None, + env: Default::default(), + tty: false, + pipe_stdin: false, + arg0: None, + }) + .await?; + assert_eq!(session.process.process_id().as_str(), process_id); + + let write_response = session.process.write(b"ignored\n".to_vec()).await?; + assert_eq!(write_response.status, WriteStatus::StdinClosed); + let StartedExecProcess { process } = session; + let wake_rx = process.subscribe_wake(); + let (output, exit_code, closed) = collect_process_output_from_reads(process, wake_rx).await?; + + assert_eq!(output, "eof\n"); + assert_eq!(exit_code, Some(0)); + assert!(closed); + Ok(()) +} + async fn assert_exec_process_preserves_queued_events_before_subscribe( use_remote: bool, ) -> Result<()> { @@ -201,6 +272,7 @@ async fn assert_exec_process_preserves_queued_events_before_subscribe( env_policy: /*env_policy*/ None, env: Default::default(), tty: false, + pipe_stdin: false, arg0: None, }) .await?; @@ -234,6 +306,7 @@ async fn remote_exec_process_reports_transport_disconnect() -> Result<()> { env_policy: /*env_policy*/ None, env: Default::default(), tty: false, + pipe_stdin: false, arg0: None, }) .await?; @@ -289,6 +362,24 @@ async fn exec_process_write_then_read(use_remote: bool) -> Result<()> { assert_exec_process_write_then_read(use_remote).await } +#[test_case(false ; "local")] +#[test_case(true ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +// Serialize tests that launch a real exec-server process through the full CLI. +#[serial_test::serial(remote_exec_server)] +async fn exec_process_write_then_read_without_tty(use_remote: bool) -> Result<()> { + assert_exec_process_write_then_read_without_tty(use_remote).await +} + +#[test_case(false ; "local")] +#[test_case(true ; "remote")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +// Serialize tests that launch a real exec-server process through the full CLI. +#[serial_test::serial(remote_exec_server)] +async fn exec_process_rejects_write_without_pipe_stdin(use_remote: bool) -> Result<()> { + assert_exec_process_rejects_write_without_pipe_stdin(use_remote).await +} + #[test_case(false ; "local")] #[test_case(true ; "remote")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] diff --git a/codex-rs/exec-server/tests/process.rs b/codex-rs/exec-server/tests/process.rs index cc96e6ef81..334e449224 100644 --- a/codex-rs/exec-server/tests/process.rs +++ b/codex-rs/exec-server/tests/process.rs @@ -10,6 +10,8 @@ use codex_exec_server::InitializeResponse; use codex_exec_server::ProcessId; use codex_exec_server::ReadResponse; use codex_exec_server::TerminateResponse; +use codex_exec_server::WriteResponse; +use codex_exec_server::WriteStatus; use common::exec_server::exec_server; use pretty_assertions::assert_eq; @@ -47,6 +49,7 @@ async fn exec_server_starts_process_over_websocket() -> anyhow::Result<()> { "cwd": std::env::current_dir()?, "env": {}, "tty": false, + "pipeStdin": false, "arg0": null }), ) @@ -75,6 +78,99 @@ async fn exec_server_starts_process_over_websocket() -> anyhow::Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn exec_server_defaults_omitted_pipe_stdin_to_closed_stdin() -> anyhow::Result<()> { + let mut server = exec_server().await?; + let initialize_id = server + .send_request( + "initialize", + serde_json::to_value(InitializeParams { + client_name: "exec-server-test".to_string(), + resume_session_id: None, + })?, + ) + .await?; + let _ = server + .wait_for_event(|event| { + matches!( + event, + JSONRPCMessage::Response(JSONRPCResponse { id, .. }) if id == &initialize_id + ) + }) + .await?; + + server + .send_notification("initialized", serde_json::json!({})) + .await?; + + let process_start_id = server + .send_request( + "process/start", + serde_json::json!({ + "processId": "proc-default-stdin", + "argv": [ + "/bin/sh", + "-c", + "sleep 0.3; if IFS= read -r line; then printf 'read:%s\\n' \"$line\"; else printf 'eof\\n'; fi" + ], + "cwd": std::env::current_dir()?, + "env": {}, + "tty": false, + "arg0": null + }), + ) + .await?; + let response = server + .wait_for_event(|event| { + matches!( + event, + JSONRPCMessage::Response(JSONRPCResponse { id, .. }) if id == &process_start_id + ) + }) + .await?; + let JSONRPCMessage::Response(JSONRPCResponse { result, .. }) = response else { + panic!("expected process/start response"); + }; + let process_start_response: ExecResponse = serde_json::from_value(result)?; + assert_eq!( + process_start_response, + ExecResponse { + process_id: ProcessId::from("proc-default-stdin") + } + ); + + let write_id = server + .send_request( + "process/write", + serde_json::json!({ + "processId": "proc-default-stdin", + "chunk": "aWdub3JlZAo=" + }), + ) + .await?; + let response = server + .wait_for_event(|event| { + matches!( + event, + JSONRPCMessage::Response(JSONRPCResponse { id, .. }) if id == &write_id + ) + }) + .await?; + let JSONRPCMessage::Response(JSONRPCResponse { result, .. }) = response else { + panic!("expected process/write response"); + }; + let write_response: WriteResponse = serde_json::from_value(result)?; + assert_eq!( + write_response, + WriteResponse { + status: WriteStatus::StdinClosed + } + ); + + server.shutdown().await?; + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn exec_server_resumes_detached_session_without_killing_processes() -> anyhow::Result<()> { let mut server = exec_server().await?; @@ -113,6 +209,7 @@ async fn exec_server_resumes_detached_session_without_killing_processes() -> any "cwd": std::env::current_dir()?, "env": {}, "tty": false, + "pipeStdin": false, "arg0": null }), ) From ff9744fd661988450a24d845cb3416ad2af71f7d Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Thu, 16 Apr 2026 10:30:28 -0700 Subject: [PATCH 26/59] Avoid fatal TUI errors on skills list failure (#18061) Addresses #17951 Problem: The TUI treated skills/list failures as fatal during refresh, so proxy/firewall responses that break plugin discovery could crash the session. Solution: Route startup and refresh skills/list responses through shared graceful handling that logs a warning and keeps the TUI running. --- codex-rs/tui/src/app.rs | 52 ++++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index 3fbcd64909..5641131566 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -2514,14 +2514,16 @@ impl App { Ok(true) } AppCommandView::ListSkills { cwds, force_reload } => { - let response = app_server - .skills_list(codex_app_server_protocol::SkillsListParams { - cwds: cwds.to_vec(), - force_reload, - per_cwd_extra_user_roots: None, - }) - .await?; - self.handle_skills_list_response(response); + self.handle_skills_list_result( + app_server + .skills_list(codex_app_server_protocol::SkillsListParams { + cwds: cwds.to_vec(), + force_reload, + per_cwd_extra_user_roots: None, + }) + .await, + "failed to refresh skills", + ); Ok(true) } AppCommandView::Compact => { @@ -2595,6 +2597,19 @@ impl App { } } + fn handle_skills_list_result( + &mut self, + result: Result, + failure_message: &str, + ) { + match result { + Ok(response) => self.handle_skills_list_response(response), + Err(err) => { + tracing::warn!("{failure_message}: {err:#}"); + } + } + } + async fn try_resolve_app_server_request( &mut self, app_server: &AppServerSession, @@ -4002,17 +4017,16 @@ impl App { app.enqueue_primary_thread_session(started.session, started.turns) .await?; } - match app_server - .skills_list(codex_app_server_protocol::SkillsListParams { - cwds: vec![app.config.cwd.to_path_buf()], - force_reload: true, - per_cwd_extra_user_roots: None, - }) - .await - { - Ok(response) => app.handle_skills_list_response(response), - Err(err) => tracing::warn!("failed to load skills on startup: {err:#}"), - } + app.handle_skills_list_result( + app_server + .skills_list(codex_app_server_protocol::SkillsListParams { + cwds: vec![app.config.cwd.to_path_buf()], + force_reload: true, + per_cwd_extra_user_roots: None, + }) + .await, + "failed to load skills on startup", + ); // On startup, if Agent mode (workspace-write) or ReadOnly is active, warn about world-writable dirs on Windows. #[cfg(target_os = "windows")] From 6862b9c745066e6b7f73ea7483c9abb0df696594 Mon Sep 17 00:00:00 2001 From: viyatb-oai Date: Thu, 16 Apr 2026 10:31:51 -0700 Subject: [PATCH 27/59] feat(permissions): add glob deny-read policy support (#15979) ## Summary - adds first-class filesystem policy entries for deny-read glob patterns - parses config such as :project_roots { "**/*.env" = "none" } into pattern entries - enforces deny-read patterns in direct read/list helpers - fails closed for sandbox execution until platform backends enforce glob patterns in #18096 - preserves split filesystem policy in turn context only when it cannot be reconstructed from legacy sandbox policy ## Stack 1. This PR - glob deny-read policy/config/direct-tool support 2. #18096 - macOS and Linux sandbox enforcement 3. #17740 - managed deny-read requirements ## Verification - just fmt - cargo check -p codex-core -p codex-sandboxing --tests --------- Co-authored-by: Codex --- codex-rs/Cargo.lock | 1 + codex-rs/core/src/codex.rs | 22 +- .../src/codex/rollout_reconstruction_tests.rs | 8 + codex-rs/core/src/codex_tests.rs | 92 ++++ codex-rs/core/src/config/config_tests.rs | 68 +++ codex-rs/core/src/config/mod.rs | 4 +- codex-rs/core/src/config/permissions.rs | 181 ++++++- codex-rs/core/src/config/permissions_tests.rs | 92 ++++ .../core/src/context_manager/history_tests.rs | 1 + codex-rs/core/src/exec.rs | 6 + codex-rs/core/src/exec_tests.rs | 47 ++ codex-rs/core/src/tools/handlers/list_dir.rs | 34 +- .../core/src/tools/handlers/list_dir_tests.rs | 71 +++ codex-rs/core/tests/suite/resume_warning.rs | 1 + codex-rs/protocol/Cargo.toml | 1 + codex-rs/protocol/src/permissions.rs | 484 +++++++++++++++--- codex-rs/protocol/src/protocol.rs | 24 + codex-rs/rollout/src/recorder_tests.rs | 1 + codex-rs/sandboxing/src/lib.rs | 3 + codex-rs/sandboxing/src/manager.rs | 14 + codex-rs/state/src/extract.rs | 3 + codex-rs/tui/src/app.rs | 1 + codex-rs/tui/src/lib.rs | 1 + 23 files changed, 1081 insertions(+), 79 deletions(-) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 3e21398f85..a6d1ec2e47 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -2614,6 +2614,7 @@ dependencies = [ "codex-utils-string", "codex-utils-template", "encoding_rs", + "globset", "http 1.4.0", "icu_decimal", "icu_locale_core", diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 00c5c144cd..5f821eb6b4 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -1083,6 +1083,18 @@ impl TurnContext { } pub(crate) fn to_turn_context_item(&self) -> TurnContextItem { + let legacy_file_system_sandbox_policy = FileSystemSandboxPolicy::from_legacy_sandbox_policy( + self.sandbox_policy.get(), + &self.cwd, + ); + // Omit the derived split filesystem policy when it is equivalent to + // the legacy sandbox policy. This keeps turn-context payloads stable + // while both fields exist; once callers consume only the split policy, + // this comparison and the legacy projection should go away. + let file_system_sandbox_policy = (self.file_system_sandbox_policy + != legacy_file_system_sandbox_policy) + .then(|| self.file_system_sandbox_policy.clone()); + TurnContextItem { turn_id: Some(self.sub_id.clone()), trace_id: self.trace_id.clone(), @@ -1092,6 +1104,7 @@ impl TurnContext { approval_policy: self.approval_policy.value(), sandbox_policy: self.sandbox_policy.get().clone(), network: self.turn_context_network_item(), + file_system_sandbox_policy, model: self.model_info.slug.clone(), personality: self.personality, collaboration_mode: Some(self.collaboration_mode.clone()), @@ -1287,7 +1300,14 @@ impl SessionConfiguration { let cwd_changed = absolute_cwd.as_path() != self.cwd.as_path(); next_configuration.cwd = absolute_cwd; - if sandbox_policy_changed || (cwd_changed && file_system_policy_matches_legacy) { + if sandbox_policy_changed { + next_configuration.file_system_sandbox_policy = + FileSystemSandboxPolicy::from_legacy_sandbox_policy_preserving_deny_entries( + next_configuration.sandbox_policy.get(), + &next_configuration.cwd, + &self.file_system_sandbox_policy, + ); + } else if cwd_changed && file_system_policy_matches_legacy { // Preserve richer split policies across cwd-only updates; only // rederive when the session is already using the legacy bridge. next_configuration.file_system_sandbox_policy = diff --git a/codex-rs/core/src/codex/rollout_reconstruction_tests.rs b/codex-rs/core/src/codex/rollout_reconstruction_tests.rs index 753244ac2b..432cfa7a40 100644 --- a/codex-rs/core/src/codex/rollout_reconstruction_tests.rs +++ b/codex-rs/core/src/codex/rollout_reconstruction_tests.rs @@ -68,6 +68,7 @@ async fn record_initial_history_resumed_bare_turn_context_does_not_hydrate_previ approval_policy: turn_context.approval_policy.value(), sandbox_policy: turn_context.sandbox_policy.get().clone(), network: None, + file_system_sandbox_policy: None, model: previous_model.to_string(), personality: turn_context.personality, collaboration_mode: Some(turn_context.collaboration_mode.clone()), @@ -107,6 +108,7 @@ async fn record_initial_history_resumed_hydrates_previous_turn_settings_from_lif approval_policy: turn_context.approval_policy.value(), sandbox_policy: turn_context.sandbox_policy.get().clone(), network: None, + file_system_sandbox_policy: None, model: previous_model.to_string(), personality: turn_context.personality, collaboration_mode: Some(turn_context.collaboration_mode.clone()), @@ -901,6 +903,7 @@ async fn record_initial_history_resumed_turn_context_after_compaction_reestablis approval_policy: turn_context.approval_policy.value(), sandbox_policy: turn_context.sandbox_policy.get().clone(), network: None, + file_system_sandbox_policy: None, model: previous_model.to_string(), personality: turn_context.personality, collaboration_mode: Some(turn_context.collaboration_mode.clone()), @@ -976,6 +979,7 @@ async fn record_initial_history_resumed_turn_context_after_compaction_reestablis approval_policy: turn_context.approval_policy.value(), sandbox_policy: turn_context.sandbox_policy.get().clone(), network: None, + file_system_sandbox_policy: None, model: previous_model.to_string(), personality: turn_context.personality, collaboration_mode: Some(turn_context.collaboration_mode.clone()), @@ -1005,6 +1009,7 @@ async fn record_initial_history_resumed_aborted_turn_without_id_clears_active_tu approval_policy: turn_context.approval_policy.value(), sandbox_policy: turn_context.sandbox_policy.get().clone(), network: None, + file_system_sandbox_policy: None, model: previous_model.to_string(), personality: turn_context.personality, collaboration_mode: Some(turn_context.collaboration_mode.clone()), @@ -1117,6 +1122,7 @@ async fn record_initial_history_resumed_unmatched_abort_preserves_active_turn_fo approval_policy: turn_context.approval_policy.value(), sandbox_policy: turn_context.sandbox_policy.get().clone(), network: None, + file_system_sandbox_policy: None, model: current_model.to_string(), personality: turn_context.personality, collaboration_mode: Some(turn_context.collaboration_mode.clone()), @@ -1227,6 +1233,7 @@ async fn record_initial_history_resumed_trailing_incomplete_turn_compaction_clea approval_policy: turn_context.approval_policy.value(), sandbox_policy: turn_context.sandbox_policy.get().clone(), network: None, + file_system_sandbox_policy: None, model: previous_model.to_string(), personality: turn_context.personality, collaboration_mode: Some(turn_context.collaboration_mode.clone()), @@ -1376,6 +1383,7 @@ async fn record_initial_history_resumed_replaced_incomplete_compacted_turn_clear approval_policy: turn_context.approval_policy.value(), sandbox_policy: turn_context.sandbox_policy.get().clone(), network: None, + file_system_sandbox_policy: None, model: previous_model.to_string(), personality: turn_context.personality, collaboration_mode: Some(turn_context.collaboration_mode.clone()), diff --git a/codex-rs/core/src/codex_tests.rs b/codex-rs/core/src/codex_tests.rs index 4805b03d01..6f4276b077 100644 --- a/codex-rs/core/src/codex_tests.rs +++ b/codex-rs/core/src/codex_tests.rs @@ -1639,6 +1639,7 @@ async fn record_initial_history_forked_hydrates_previous_turn_settings() { approval_policy: turn_context.approval_policy.value(), sandbox_policy: turn_context.sandbox_policy.get().clone(), network: None, + file_system_sandbox_policy: None, model: previous_model.to_string(), personality: turn_context.personality, collaboration_mode: Some(turn_context.collaboration_mode.clone()), @@ -4714,6 +4715,47 @@ async fn build_initial_context_restates_realtime_start_when_reference_context_is ); } +fn file_system_policy_with_unreadable_glob(turn_context: &TurnContext) -> FileSystemSandboxPolicy { + let mut policy = FileSystemSandboxPolicy::from_legacy_sandbox_policy( + turn_context.sandbox_policy.get(), + &turn_context.cwd, + ); + policy.entries.push(FileSystemSandboxEntry { + path: FileSystemPath::GlobPattern { + pattern: format!("{}/**/*.env", turn_context.cwd.as_path().display()), + }, + access: FileSystemAccessMode::None, + }); + policy +} + +#[tokio::test] +async fn turn_context_item_omits_legacy_equivalent_file_system_sandbox_policy() { + let (_session, mut turn_context) = make_session_and_context().await; + turn_context.file_system_sandbox_policy = FileSystemSandboxPolicy::from_legacy_sandbox_policy( + turn_context.sandbox_policy.get(), + &turn_context.cwd, + ); + + let item = turn_context.to_turn_context_item(); + + assert_eq!(item.file_system_sandbox_policy, None); +} + +#[tokio::test] +async fn turn_context_item_stores_split_file_system_sandbox_policy_when_different() { + let (_session, mut turn_context) = make_session_and_context().await; + let file_system_sandbox_policy = file_system_policy_with_unreadable_glob(&turn_context); + turn_context.file_system_sandbox_policy = file_system_sandbox_policy.clone(); + + let item = turn_context.to_turn_context_item(); + + assert_eq!( + item.file_system_sandbox_policy, + Some(file_system_sandbox_policy) + ); +} + #[tokio::test] async fn record_context_updates_and_set_reference_context_item_injects_full_context_when_baseline_missing() { @@ -4852,6 +4894,56 @@ async fn record_context_updates_and_set_reference_context_item_persists_baseline ); } +#[tokio::test] +async fn record_context_updates_and_set_reference_context_item_persists_split_file_system_policy_to_rollout() + { + let (session, mut turn_context) = make_session_and_context().await; + let file_system_sandbox_policy = file_system_policy_with_unreadable_glob(&turn_context); + turn_context.file_system_sandbox_policy = file_system_sandbox_policy.clone(); + let config = session.get_config().await; + let recorder = RolloutRecorder::new( + config.as_ref(), + RolloutRecorderParams::new( + ThreadId::default(), + /*forked_from_id*/ None, + SessionSource::Exec, + BaseInstructions::default(), + Vec::new(), + EventPersistenceMode::Limited, + ), + /*state_db_ctx*/ None, + /*state_builder*/ None, + ) + .await + .expect("create rollout recorder"); + let rollout_path = recorder.rollout_path().to_path_buf(); + { + let mut rollout = session.services.rollout.lock().await; + *rollout = Some(recorder); + } + + session + .record_context_updates_and_set_reference_context_item(&turn_context) + .await; + session.ensure_rollout_materialized().await; + session.flush_rollout().await.expect("rollout should flush"); + + let InitialHistory::Resumed(resumed) = RolloutRecorder::get_rollout_history(&rollout_path) + .await + .expect("read rollout history") + else { + panic!("expected resumed rollout history"); + }; + let persisted_file_system_sandbox_policy = resumed.history.iter().find_map(|item| match item { + RolloutItem::TurnContext(ctx) => ctx.file_system_sandbox_policy.clone(), + _ => None, + }); + assert_eq!( + persisted_file_system_sandbox_policy, + Some(file_system_sandbox_policy) + ); +} + #[tokio::test] async fn build_initial_context_prepends_model_switch_message() { let (session, turn_context) = make_session_and_context().await; diff --git a/codex-rs/core/src/config/config_tests.rs b/codex-rs/core/src/config/config_tests.rs index 9243235d71..a7a5e88352 100644 --- a/codex-rs/core/src/config/config_tests.rs +++ b/codex-rs/core/src/config/config_tests.rs @@ -659,6 +659,74 @@ async fn default_permissions_profile_populates_runtime_sandbox_policy() -> std:: Ok(()) } +#[tokio::test] +async fn project_root_glob_none_compiles_to_filesystem_pattern_entry() -> std::io::Result<()> { + let codex_home = TempDir::new()?; + let cwd = TempDir::new()?; + tokio::fs::write(cwd.path().join(".git"), "gitdir: nowhere").await?; + + let config = Config::load_from_base_config_with_overrides( + ConfigToml { + default_permissions: Some("workspace".to_string()), + permissions: Some(PermissionsToml { + entries: BTreeMap::from([( + "workspace".to_string(), + PermissionProfileToml { + filesystem: Some(FilesystemPermissionsToml { + entries: BTreeMap::from([( + ":project_roots".to_string(), + FilesystemPermissionToml::Scoped(BTreeMap::from([ + (".".to_string(), FileSystemAccessMode::Write), + ("**/*.env".to_string(), FileSystemAccessMode::None), + ])), + )]), + }), + network: None, + }, + )]), + }), + ..Default::default() + }, + ConfigOverrides { + cwd: Some(cwd.path().to_path_buf()), + ..Default::default() + }, + codex_home.abs(), + ) + .await?; + + let expected_pattern = AbsolutePathBuf::resolve_path_against_base("**/*.env", cwd.path()) + .to_string_lossy() + .into_owned(); + assert!( + config + .permissions + .file_system_sandbox_policy + .entries + .contains(&FileSystemSandboxEntry { + path: FileSystemPath::GlobPattern { + pattern: expected_pattern, + }, + access: FileSystemAccessMode::None, + }) + ); + assert!( + !config + .permissions + .file_system_sandbox_policy + .entries + .iter() + .any(|entry| matches!( + &entry.path, + FileSystemPath::Special { + value: FileSystemSpecialPath::ProjectRoots { subpath: Some(subpath) }, + } if subpath == std::path::Path::new("**/*.env") + )), + "glob should compile to a filesystem pattern entry, not a literal filesystem entry" + ); + Ok(()) +} + #[tokio::test] async fn permissions_profiles_require_default_permissions() -> std::io::Result<()> { let codex_home = TempDir::new()?; diff --git a/codex-rs/core/src/config/mod.rs b/codex-rs/core/src/config/mod.rs index adfbe45baa..87e888e46e 100644 --- a/codex-rs/core/src/config/mod.rs +++ b/codex-rs/core/src/config/mod.rs @@ -1622,6 +1622,7 @@ impl Config { compile_permission_profile( permissions, default_permissions, + resolved_cwd.as_path(), &mut startup_warnings, )?; let mut sandbox_policy = file_system_sandbox_policy @@ -2006,9 +2007,10 @@ impl Config { if effective_sandbox_policy == original_sandbox_policy { file_system_sandbox_policy } else { - FileSystemSandboxPolicy::from_legacy_sandbox_policy( + FileSystemSandboxPolicy::from_legacy_sandbox_policy_preserving_deny_entries( &effective_sandbox_policy, resolved_cwd.as_path(), + &file_system_sandbox_policy, ) }; let effective_file_system_sandbox_policy = effective_file_system_sandbox_policy diff --git a/codex-rs/core/src/config/permissions.rs b/codex-rs/core/src/config/permissions.rs index 826d0139bf..1609b696d8 100644 --- a/codex-rs/core/src/config/permissions.rs +++ b/codex-rs/core/src/config/permissions.rs @@ -5,12 +5,14 @@ use std::path::Path; use std::path::PathBuf; use codex_config::permissions_toml::FilesystemPermissionToml; +use codex_config::permissions_toml::FilesystemPermissionsToml; use codex_config::permissions_toml::NetworkToml; use codex_config::permissions_toml::PermissionProfileToml; use codex_config::permissions_toml::PermissionsToml; use codex_network_proxy::NetworkProxyConfig; #[cfg(test)] use codex_network_proxy::NetworkUnixSocketPermission as ProxyNetworkUnixSocketPermission; +use codex_protocol::permissions::FileSystemAccessMode; use codex_protocol::permissions::FileSystemPath; use codex_protocol::permissions::FileSystemSandboxEntry; use codex_protocol::permissions::FileSystemSandboxPolicy; @@ -42,6 +44,7 @@ pub(crate) fn resolve_permission_profile<'a>( pub(crate) fn compile_permission_profile( permissions: &PermissionsToml, profile_name: &str, + policy_cwd: &Path, startup_warnings: &mut Vec, ) -> io::Result<(FileSystemSandboxPolicy, NetworkSandboxPolicy)> { let profile = resolve_permission_profile(permissions, profile_name)?; @@ -54,8 +57,23 @@ pub(crate) fn compile_permission_profile( missing_filesystem_entries_warning(profile_name), ); } else { + if cfg!(not(target_os = "macos")) { + for pattern in unsupported_read_write_glob_paths(filesystem) { + push_warning( + startup_warnings, + format!( + "Filesystem glob `{pattern}` uses `read` or `write` access, which is not fully supported by this platform's sandboxing. Use an exact path or trailing `/**` subtree rule instead. `none` deny-read globs are supported." + ), + ); + } + } for (path, permission) in &filesystem.entries { - compile_filesystem_permission(path, permission, &mut entries, startup_warnings)?; + entries.extend(compile_filesystem_permission( + path, + permission, + policy_cwd, + startup_warnings, + )?); } } } else { @@ -66,7 +84,6 @@ pub(crate) fn compile_permission_profile( } let network_sandbox_policy = compile_network_sandbox_policy(profile.network.as_ref()); - Ok(( FileSystemSandboxPolicy::restricted(entries), network_sandbox_policy, @@ -118,24 +135,71 @@ fn compile_network_sandbox_policy(network: Option<&NetworkToml>) -> NetworkSandb fn compile_filesystem_permission( path: &str, permission: &FilesystemPermissionToml, - entries: &mut Vec, + policy_cwd: &Path, startup_warnings: &mut Vec, -) -> io::Result<()> { +) -> io::Result> { + let mut entries = Vec::new(); match permission { - FilesystemPermissionToml::Access(access) => entries.push(FileSystemSandboxEntry { - path: compile_filesystem_path(path, startup_warnings)?, - access: *access, - }), + FilesystemPermissionToml::Access(access) => { + entries.push(FileSystemSandboxEntry { + path: compile_filesystem_access_path(path, *access, startup_warnings)?, + access: *access, + }); + } FilesystemPermissionToml::Scoped(scoped_entries) => { for (subpath, access) in scoped_entries { - entries.push(FileSystemSandboxEntry { - path: compile_scoped_filesystem_path(path, subpath, startup_warnings)?, - access: *access, - }); + let has_glob = contains_glob_chars(subpath); + let can_compile_as_pattern = match parse_special_path(path) { + Some(FileSystemSpecialPath::ProjectRoots { .. }) | None => true, + Some(_) => false, + }; + if has_glob && *access == FileSystemAccessMode::None && can_compile_as_pattern { + // Scoped glob syntax is a first-class filesystem policy + // pattern entry. Literal scoped paths continue through the + // exact-path parser so existing path semantics stay intact. + let entry = FileSystemSandboxEntry { + path: FileSystemPath::GlobPattern { + pattern: compile_scoped_filesystem_pattern( + path, subpath, *access, policy_cwd, + )?, + }, + access: *access, + }; + entries.push(entry); + } else { + let subpath = compile_read_write_glob_path(subpath, *access)?; + entries.push(FileSystemSandboxEntry { + path: compile_scoped_filesystem_path(path, subpath, startup_warnings)?, + access: *access, + }); + } } } } - Ok(()) + Ok(entries) +} + +fn compile_filesystem_access_path( + path: &str, + access: FileSystemAccessMode, + startup_warnings: &mut Vec, +) -> io::Result { + if !contains_glob_chars(path) { + return compile_filesystem_path(path, startup_warnings); + } + + if access == FileSystemAccessMode::None { + // At this point `path` is an unscoped filesystem table key. Top-level + // glob deny entries still go through the absolute-path parser before + // becoming policy patterns; relative project-root glob syntax is + // handled by `compile_scoped_filesystem_pattern`. + return Ok(FileSystemPath::GlobPattern { + pattern: parse_absolute_path(path)?.to_string_lossy().into_owned(), + }); + } + + let path = compile_read_write_glob_path(path, access)?; + compile_filesystem_path(path, startup_warnings) } fn compile_filesystem_path( @@ -186,6 +250,97 @@ fn compile_scoped_filesystem_path( Ok(FileSystemPath::Path { path }) } +fn compile_scoped_filesystem_pattern( + path: &str, + subpath: &str, + access: FileSystemAccessMode, + policy_cwd: &Path, +) -> io::Result { + // Pattern entries currently mean deny-read only. Supporting broader access + // modes here would imply glob-based read/write allow semantics that the + // sandbox policy does not express yet. + if access != FileSystemAccessMode::None { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("filesystem glob subpath `{subpath}` only supports `none` access"), + )); + } + let subpath = parse_relative_subpath(subpath)?; + + match parse_special_path(path) { + Some(FileSystemSpecialPath::ProjectRoots { .. }) => { + // `:project_roots` is represented as a special path, but current + // filesystem-policy resolution defines it relative to the session + // cwd. Use the same policy cwd here so glob entries and exact + // scoped entries resolve consistently. + Ok( + AbsolutePathBuf::resolve_path_against_base(&subpath, policy_cwd) + .to_string_lossy() + .to_string(), + ) + } + Some(_) => Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("filesystem path `{path}` does not support nested entries"), + )), + None => { + let base = parse_absolute_path(path)?; + Ok(base.join(&subpath).to_string_lossy().to_string()) + } + } +} + +fn compile_read_write_glob_path(path: &str, access: FileSystemAccessMode) -> io::Result<&str> { + if !contains_glob_chars(path) { + return Ok(path); + } + + let path_without_trailing_glob = remove_trailing_glob_suffix(path); + if !contains_glob_chars(path_without_trailing_glob) { + return Ok(path_without_trailing_glob); + } + + Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!( + "filesystem glob path `{path}` only supports `none` access; use an exact path or trailing `/**` for `{access}` subtree access" + ), + )) +} + +fn unsupported_read_write_glob_paths(filesystem: &FilesystemPermissionsToml) -> Vec { + let mut patterns = Vec::new(); + for (path, permission) in &filesystem.entries { + match permission { + FilesystemPermissionToml::Access(access) => { + if *access != FileSystemAccessMode::None + && contains_glob_chars(remove_trailing_glob_suffix(path)) + { + patterns.push(path.clone()); + } + } + FilesystemPermissionToml::Scoped(scoped_entries) => { + for (subpath, access) in scoped_entries { + if *access != FileSystemAccessMode::None + && contains_glob_chars(remove_trailing_glob_suffix(subpath)) + { + patterns.push(format!("{path}/{subpath}")); + } + } + } + } + } + patterns +} + +fn contains_glob_chars(path: &str) -> bool { + path.chars().any(|ch| matches!(ch, '*' | '?' | '[' | ']')) +} + +fn remove_trailing_glob_suffix(path: &str) -> &str { + path.strip_suffix("/**").unwrap_or(path) +} + // WARNING: keep this parser forward-compatible. // Adding a new `:special_path` must not make older Codex versions reject the // config. Unknown values intentionally round-trip through diff --git a/codex-rs/core/src/config/permissions_tests.rs b/codex-rs/core/src/config/permissions_tests.rs index 34d22c2bfb..7116e32e70 100644 --- a/codex-rs/core/src/config/permissions_tests.rs +++ b/codex-rs/core/src/config/permissions_tests.rs @@ -2,6 +2,7 @@ use super::*; use crate::config::Config; use crate::config::ConfigOverrides; use codex_config::config_toml::ConfigToml; +use codex_config::permissions_toml::FilesystemPermissionToml; use codex_config::permissions_toml::FilesystemPermissionsToml; use codex_config::permissions_toml::NetworkDomainPermissionToml; use codex_config::permissions_toml::NetworkDomainPermissionsToml; @@ -10,6 +11,11 @@ use codex_config::permissions_toml::NetworkUnixSocketPermissionToml; use codex_config::permissions_toml::NetworkUnixSocketPermissionsToml; use codex_config::permissions_toml::PermissionProfileToml; use codex_config::permissions_toml::PermissionsToml; +use codex_protocol::permissions::FileSystemAccessMode; +use codex_protocol::permissions::FileSystemPath; +use codex_protocol::permissions::FileSystemSandboxEntry; +use codex_protocol::permissions::FileSystemSandboxPolicy; +use codex_protocol::permissions::FileSystemSpecialPath; use codex_utils_absolute_path::AbsolutePathBuf; use pretty_assertions::assert_eq; use std::collections::BTreeMap; @@ -216,3 +222,89 @@ fn network_toml_overlays_unix_socket_permissions_by_path() { }) ); } + +#[test] +fn read_write_glob_warnings_skip_supported_deny_read_globs_and_trailing_subpaths() { + let filesystem = FilesystemPermissionsToml { + entries: BTreeMap::from([ + ( + "/tmp/**/*.log".to_string(), + FilesystemPermissionToml::Access(FileSystemAccessMode::Read), + ), + ( + "/tmp/cache/**".to_string(), + FilesystemPermissionToml::Access(FileSystemAccessMode::Write), + ), + ( + ":project_roots".to_string(), + FilesystemPermissionToml::Scoped(BTreeMap::from([ + ("**/*.env".to_string(), FileSystemAccessMode::None), + ("docs/**".to_string(), FileSystemAccessMode::Read), + ("src/**/*.rs".to_string(), FileSystemAccessMode::Write), + ])), + ), + ]), + }; + + assert_eq!( + unsupported_read_write_glob_paths(&filesystem), + vec![ + "/tmp/**/*.log".to_string(), + ":project_roots/src/**/*.rs".to_string() + ], + "`none` glob patterns are supported as deny-read rules; only `read`/`write` globs should warn" + ); +} + +#[test] +fn read_write_trailing_glob_suffix_compiles_as_subpath() -> std::io::Result<()> { + let cwd = TempDir::new()?; + let mut startup_warnings = Vec::new(); + let (file_system_policy, _) = compile_permission_profile( + &PermissionsToml { + entries: BTreeMap::from([( + "workspace".to_string(), + PermissionProfileToml { + filesystem: Some(FilesystemPermissionsToml { + entries: BTreeMap::from([( + ":project_roots".to_string(), + FilesystemPermissionToml::Scoped(BTreeMap::from([( + "docs/**".to_string(), + FileSystemAccessMode::Read, + )])), + )]), + }), + network: None, + }, + )]), + }, + "workspace", + cwd.path(), + &mut startup_warnings, + )?; + + assert_eq!( + file_system_policy, + FileSystemSandboxPolicy::restricted(vec![FileSystemSandboxEntry { + path: FileSystemPath::Special { + value: FileSystemSpecialPath::project_roots(Some("docs".into())), + }, + access: FileSystemAccessMode::Read, + }]), + "trailing /** should compile as a subtree path instead of a glob pattern" + ); + Ok(()) +} + +#[test] +fn read_write_glob_patterns_still_reject_non_subpath_globs() { + let err = compile_read_write_glob_path("src/**/*.rs", FileSystemAccessMode::Read) + .expect_err("non-subpath read/write glob should be rejected"); + + assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput); + assert!( + err.to_string() + .contains("filesystem glob path `src/**/*.rs` only supports `none` access"), + "{err}" + ); +} diff --git a/codex-rs/core/src/context_manager/history_tests.rs b/codex-rs/core/src/context_manager/history_tests.rs index 9720492c1f..ff71b797be 100644 --- a/codex-rs/core/src/context_manager/history_tests.rs +++ b/codex-rs/core/src/context_manager/history_tests.rs @@ -133,6 +133,7 @@ fn reference_context_item() -> TurnContextItem { approval_policy: AskForApproval::OnRequest, sandbox_policy: SandboxPolicy::new_read_only_policy(), network: None, + file_system_sandbox_policy: None, model: "gpt-test".to_string(), personality: None, collaboration_mode: None, diff --git a/codex-rs/core/src/exec.rs b/codex-rs/core/src/exec.rs index 10a1dc40ff..396b7701be 100644 --- a/codex-rs/core/src/exec.rs +++ b/codex-rs/core/src/exec.rs @@ -981,6 +981,9 @@ pub(crate) fn resolve_windows_restricted_token_filesystem_overrides( if !file_system_sandbox_policy .get_unreadable_roots_with_cwd(sandbox_policy_cwd) .is_empty() + || !file_system_sandbox_policy + .get_unreadable_globs_with_cwd(sandbox_policy_cwd) + .is_empty() { return Err( "windows unelevated restricted-token sandbox cannot enforce unreadable split filesystem carveouts directly; refusing to run unsandboxed" @@ -1096,6 +1099,9 @@ pub(crate) fn resolve_windows_elevated_filesystem_overrides( if !file_system_sandbox_policy .get_unreadable_roots_with_cwd(sandbox_policy_cwd) .is_empty() + || !file_system_sandbox_policy + .get_unreadable_globs_with_cwd(sandbox_policy_cwd) + .is_empty() { return Err( "windows elevated sandbox cannot enforce unreadable split filesystem carveouts directly; refusing to run unsandboxed" diff --git a/codex-rs/core/src/exec_tests.rs b/codex-rs/core/src/exec_tests.rs index acc92f5585..1cfa87ff3f 100644 --- a/codex-rs/core/src/exec_tests.rs +++ b/codex-rs/core/src/exec_tests.rs @@ -806,6 +806,53 @@ fn windows_elevated_rejects_unreadable_split_carveouts() { ); } +#[test] +fn windows_elevated_rejects_unreadable_globs() { + let temp_dir = tempfile::TempDir::new().expect("tempdir"); + let policy = SandboxPolicy::WorkspaceWrite { + writable_roots: vec![], + read_only_access: codex_protocol::protocol::ReadOnlyAccess::FullAccess, + network_access: false, + exclude_tmpdir_env_var: true, + exclude_slash_tmp: true, + }; + let file_system_policy = FileSystemSandboxPolicy::restricted(vec![ + codex_protocol::permissions::FileSystemSandboxEntry { + path: codex_protocol::permissions::FileSystemPath::Special { + value: codex_protocol::permissions::FileSystemSpecialPath::Root, + }, + access: codex_protocol::permissions::FileSystemAccessMode::Read, + }, + codex_protocol::permissions::FileSystemSandboxEntry { + path: codex_protocol::permissions::FileSystemPath::Special { + value: codex_protocol::permissions::FileSystemSpecialPath::CurrentWorkingDirectory, + }, + access: codex_protocol::permissions::FileSystemAccessMode::Write, + }, + codex_protocol::permissions::FileSystemSandboxEntry { + path: codex_protocol::permissions::FileSystemPath::GlobPattern { + pattern: "**/*.env".to_string(), + }, + access: codex_protocol::permissions::FileSystemAccessMode::None, + }, + ]); + + assert_eq!( + unsupported_windows_restricted_token_sandbox_reason( + SandboxType::WindowsRestrictedToken, + &policy, + &file_system_policy, + NetworkSandboxPolicy::Restricted, + &temp_dir.path().abs(), + WindowsSandboxLevel::Elevated, + ), + Some( + "windows elevated sandbox cannot enforce unreadable split filesystem carveouts directly; refusing to run unsandboxed" + .to_string() + ) + ); +} + #[test] fn windows_elevated_rejects_reopened_writable_descendants() { let temp_dir = tempfile::TempDir::new().expect("tempdir"); diff --git a/codex-rs/core/src/tools/handlers/list_dir.rs b/codex-rs/core/src/tools/handlers/list_dir.rs index be4e356c5c..22909c1d18 100644 --- a/codex-rs/core/src/tools/handlers/list_dir.rs +++ b/codex-rs/core/src/tools/handlers/list_dir.rs @@ -4,6 +4,7 @@ use std::fs::FileType; use std::path::Path; use std::path::PathBuf; +use codex_protocol::permissions::ReadDenyMatcher; use codex_utils_string::take_bytes_at_char_boundary; use serde::Deserialize; use tokio::fs; @@ -18,6 +19,8 @@ use crate::tools::registry::ToolKind; pub struct ListDirHandler; +const DENY_READ_POLICY_MESSAGE: &str = + "access denied: reading this path is blocked by filesystem deny_read policy"; const MAX_ENTRY_LENGTH: usize = 500; const INDENTATION_SPACES: usize = 2; @@ -52,7 +55,7 @@ impl ToolHandler for ListDirHandler { } async fn handle(&self, invocation: ToolInvocation) -> Result { - let ToolInvocation { payload, .. } = invocation; + let ToolInvocation { payload, turn, .. } = invocation; let arguments = match payload { ToolPayload::Function { arguments } => arguments, @@ -96,8 +99,20 @@ impl ToolHandler for ListDirHandler { "dir_path must be an absolute path".to_string(), )); } + let read_deny_matcher = ReadDenyMatcher::new(&turn.file_system_sandbox_policy, &turn.cwd); + if read_deny_matcher + .as_ref() + .is_some_and(|matcher| matcher.is_read_denied(&path)) + { + return Err(FunctionCallError::RespondToModel(format!( + "{DENY_READ_POLICY_MESSAGE}: `{}`", + path.display() + ))); + } - let entries = list_dir_slice(&path, offset, limit, depth).await?; + let entries = + list_dir_slice_with_policy(&path, offset, limit, depth, read_deny_matcher.as_ref()) + .await?; let mut output = Vec::with_capacity(entries.len() + 1); output.push(format!("Absolute path: {}", path.display())); output.extend(entries); @@ -105,14 +120,15 @@ impl ToolHandler for ListDirHandler { } } -async fn list_dir_slice( +async fn list_dir_slice_with_policy( path: &Path, offset: usize, limit: usize, depth: usize, + read_deny_matcher: Option<&ReadDenyMatcher>, ) -> Result, FunctionCallError> { let mut entries = Vec::new(); - collect_entries(path, Path::new(""), depth, &mut entries).await?; + collect_entries(path, Path::new(""), depth, read_deny_matcher, &mut entries).await?; if entries.is_empty() { return Ok(Vec::new()); @@ -148,6 +164,7 @@ async fn collect_entries( dir_path: &Path, relative_prefix: &Path, depth: usize, + read_deny_matcher: Option<&ReadDenyMatcher>, entries: &mut Vec, ) -> Result<(), FunctionCallError> { let mut queue = VecDeque::new(); @@ -163,6 +180,13 @@ async fn collect_entries( while let Some(entry) = read_dir.next_entry().await.map_err(|err| { FunctionCallError::RespondToModel(format!("failed to read directory: {err}")) })? { + let entry_path = entry.path(); + if let Some(read_deny_matcher) = read_deny_matcher + && read_deny_matcher.is_read_denied(&entry_path) + { + continue; + } + let file_type = entry.file_type().await.map_err(|err| { FunctionCallError::RespondToModel(format!("failed to inspect entry: {err}")) })?; @@ -179,7 +203,7 @@ async fn collect_entries( let sort_key = format_entry_name(&relative_path); let kind = DirEntryKind::from(&file_type); dir_entries.push(( - entry.path(), + entry_path, relative_path, kind, DirEntry { diff --git a/codex-rs/core/src/tools/handlers/list_dir_tests.rs b/codex-rs/core/src/tools/handlers/list_dir_tests.rs index ccc30b5326..7de107a18d 100644 --- a/codex-rs/core/src/tools/handlers/list_dir_tests.rs +++ b/codex-rs/core/src/tools/handlers/list_dir_tests.rs @@ -1,7 +1,21 @@ use super::*; +use codex_protocol::permissions::FileSystemAccessMode; +use codex_protocol::permissions::FileSystemPath; +use codex_protocol::permissions::FileSystemSandboxEntry; +use codex_protocol::permissions::FileSystemSandboxPolicy; +use codex_protocol::permissions::ReadDenyMatcher; use pretty_assertions::assert_eq; use tempfile::tempdir; +async fn list_dir_slice( + path: &Path, + offset: usize, + limit: usize, + depth: usize, +) -> Result, FunctionCallError> { + list_dir_slice_with_policy(path, offset, limit, depth, /*read_deny_matcher*/ None).await +} + #[tokio::test] async fn lists_directory_entries() { let temp = tempdir().expect("create tempdir"); @@ -258,3 +272,60 @@ async fn truncation_respects_sorted_order() -> anyhow::Result<()> { Ok(()) } + +#[tokio::test] +async fn hides_denied_entries_and_prunes_denied_subtrees() { + let temp = tempdir().expect("create tempdir"); + let dir_path = temp.path(); + let visible_dir = dir_path.join("visible"); + let denied_dir = dir_path.join("private"); + tokio::fs::create_dir(&visible_dir) + .await + .expect("create visible dir"); + tokio::fs::create_dir(&denied_dir) + .await + .expect("create denied dir"); + tokio::fs::write(visible_dir.join("ok.txt"), b"ok") + .await + .expect("write visible file"); + tokio::fs::write(denied_dir.join("secret.txt"), b"secret") + .await + .expect("write denied file"); + tokio::fs::write(dir_path.join("top_secret.txt"), b"secret") + .await + .expect("write denied top-level file"); + + let policy = FileSystemSandboxPolicy::restricted(vec![ + FileSystemSandboxEntry { + path: FileSystemPath::Path { + path: denied_dir.try_into().expect("absolute denied dir"), + }, + access: FileSystemAccessMode::None, + }, + FileSystemSandboxEntry { + path: FileSystemPath::Path { + path: dir_path + .join("top_secret.txt") + .try_into() + .expect("absolute denied file"), + }, + access: FileSystemAccessMode::None, + }, + ]); + + let read_deny_matcher = ReadDenyMatcher::new(&policy, dir_path); + let entries = list_dir_slice_with_policy( + dir_path, + /*offset*/ 1, + /*limit*/ 20, + /*depth*/ 3, + read_deny_matcher.as_ref(), + ) + .await + .expect("list directory"); + + assert_eq!( + entries, + vec!["visible/".to_string(), " ok.txt".to_string(),] + ); +} diff --git a/codex-rs/core/tests/suite/resume_warning.rs b/codex-rs/core/tests/suite/resume_warning.rs index f9ad64c771..8ea2be3147 100644 --- a/codex-rs/core/tests/suite/resume_warning.rs +++ b/codex-rs/core/tests/suite/resume_warning.rs @@ -34,6 +34,7 @@ fn resume_history( approval_policy: config.permissions.approval_policy.value(), sandbox_policy: config.permissions.sandbox_policy.get().clone(), network: None, + file_system_sandbox_policy: None, model: previous_model.to_string(), personality: None, collaboration_mode: None, diff --git a/codex-rs/protocol/Cargo.toml b/codex-rs/protocol/Cargo.toml index c4249a13b4..f70dc94585 100644 --- a/codex-rs/protocol/Cargo.toml +++ b/codex-rs/protocol/Cargo.toml @@ -23,6 +23,7 @@ codex-utils-image = { workspace = true } codex-utils-string = { workspace = true } codex-utils-template = { workspace = true } encoding_rs = { workspace = true } +globset = { workspace = true } icu_decimal = { workspace = true } icu_locale_core = { workspace = true } icu_provider = { workspace = true, features = ["sync"] } diff --git a/codex-rs/protocol/src/permissions.rs b/codex-rs/protocol/src/permissions.rs index f101f8bef0..d774f6dc0d 100644 --- a/codex-rs/protocol/src/permissions.rs +++ b/codex-rs/protocol/src/permissions.rs @@ -6,6 +6,8 @@ use std::path::PathBuf; use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_absolute_path::canonicalize_preserving_symlinks; +use globset::GlobBuilder; +use globset::GlobMatcher; use schemars::JsonSchema; use serde::Deserialize; use serde::Serialize; @@ -154,14 +156,101 @@ struct FileSystemSemanticSignature { readable_roots: Vec, writable_roots: Vec, unreadable_roots: Vec, + unreadable_globs: Vec, +} + +/// Runtime matcher for read-deny entries in a filesystem sandbox policy. +pub struct ReadDenyMatcher { + denied_candidates: Vec>, + deny_read_matchers: Vec, + invalid_pattern: bool, +} + +impl ReadDenyMatcher { + /// Builds a matcher from exact deny-read roots and deny-read glob entries. + /// + /// Returns `None` when the policy has no deny-read restrictions, so callers + /// can skip read-deny checks without allocating matcher state. The `cwd` + /// resolves cwd-relative policy paths and special paths before matching. + pub fn new(file_system_sandbox_policy: &FileSystemSandboxPolicy, cwd: &Path) -> Option { + if !file_system_sandbox_policy.has_denied_read_restrictions() { + return None; + } + + // Exact roots are stored as all meaningful path spellings we can derive + // cheaply. This lets direct tool checks catch both a symlink path and + // its canonical target without changing the policy entries themselves. + let denied_candidates = file_system_sandbox_policy + .get_unreadable_roots_with_cwd(cwd) + .into_iter() + .map(|path| normalized_and_canonical_candidates(path.as_path())) + .collect(); + // Pattern entries stay as policy-level globs. They are matched at read + // time here instead of being snapshotted to startup filesystem state. + let mut invalid_pattern = false; + let deny_read_matchers = file_system_sandbox_policy + .get_unreadable_globs_with_cwd(cwd) + .into_iter() + .filter_map(|pattern| match build_glob_matcher(&pattern) { + Some(matcher) => Some(matcher), + None => { + invalid_pattern = true; + None + } + }) + .collect(); + Some(Self { + denied_candidates, + deny_read_matchers, + invalid_pattern, + }) + } + + /// Returns whether `path` is denied by the policy used to build this matcher. + pub fn is_read_denied(&self, path: &Path) -> bool { + if self.invalid_pattern { + // Direct tool reads fail closed on malformed deny patterns. Silent + // allow would turn a config typo into a policy bypass. + return true; + } + + // Check exact roots against each candidate spelling before evaluating + // glob matchers. Exact entries are subtree denies; glob entries match + // according to the pattern compiler's path-separator rules. + let path_candidates = normalized_and_canonical_candidates(path); + if self.denied_candidates.iter().any(|denied_candidates| { + path_candidates.iter().any(|candidate| { + denied_candidates.iter().any(|denied_candidate| { + candidate == denied_candidate || candidate.starts_with(denied_candidate) + }) + }) + }) { + return true; + } + + self.deny_read_matchers.iter().any(|matcher| { + path_candidates + .iter() + .any(|candidate| matcher.is_match(candidate)) + }) + } } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema, TS)] #[serde(tag = "type", rename_all = "snake_case")] #[ts(tag = "type")] pub enum FileSystemPath { - Path { path: AbsolutePathBuf }, - Special { value: FileSystemSpecialPath }, + Path { + path: AbsolutePathBuf, + }, + /// A git-style glob pattern. Pattern entries currently support + /// FileSystemAccessMode::None only. + GlobPattern { + pattern: String, + }, + Special { + value: FileSystemSpecialPath, + }, } impl Default for FileSystemSandboxPolicy { @@ -179,62 +268,6 @@ impl Default for FileSystemSandboxPolicy { } impl FileSystemSandboxPolicy { - fn has_root_access(&self, predicate: impl Fn(FileSystemAccessMode) -> bool) -> bool { - matches!(self.kind, FileSystemSandboxKind::Restricted) - && self.entries.iter().any(|entry| { - matches!( - &entry.path, - FileSystemPath::Special { value } - if matches!(value, FileSystemSpecialPath::Root) && predicate(entry.access) - ) - }) - } - - fn has_explicit_deny_entries(&self) -> bool { - matches!(self.kind, FileSystemSandboxKind::Restricted) - && self - .entries - .iter() - .any(|entry| entry.access == FileSystemAccessMode::None) - } - - /// Returns true when a restricted policy contains any entry that really - /// reduces a broader `:root = write` grant. - /// - /// Raw entry presence is not enough here: an equally specific `write` - /// entry for the same target wins under the normal precedence rules, so a - /// shadowed `read` entry must not downgrade the policy out of full-disk - /// write mode. - fn has_write_narrowing_entries(&self) -> bool { - matches!(self.kind, FileSystemSandboxKind::Restricted) - && self.entries.iter().any(|entry| { - if entry.access.can_write() { - return false; - } - - match &entry.path { - FileSystemPath::Path { .. } => !self.has_same_target_write_override(entry), - FileSystemPath::Special { value } => match value { - FileSystemSpecialPath::Root => entry.access == FileSystemAccessMode::None, - FileSystemSpecialPath::Minimal | FileSystemSpecialPath::Unknown { .. } => { - false - } - _ => !self.has_same_target_write_override(entry), - }, - } - }) - } - - /// Returns true when a higher-priority `write` entry targets the same - /// location as `entry`, so `entry` cannot narrow effective write access. - fn has_same_target_write_override(&self, entry: &FileSystemSandboxEntry) -> bool { - self.entries.iter().any(|candidate| { - candidate.access.can_write() - && candidate.access > entry.access - && file_system_paths_share_target(&candidate.path, &entry.path) - }) - } - pub fn unrestricted() -> Self { Self { kind: FileSystemSandboxKind::Unrestricted, @@ -256,6 +289,86 @@ impl FileSystemSandboxPolicy { } } + fn has_root_access(&self, predicate: impl Fn(FileSystemAccessMode) -> bool) -> bool { + matches!(self.kind, FileSystemSandboxKind::Restricted) + && self.entries.iter().any(|entry| { + matches!( + &entry.path, + FileSystemPath::Special { value } + if matches!(value, FileSystemSpecialPath::Root) && predicate(entry.access) + ) + }) + } + + pub fn has_denied_read_restrictions(&self) -> bool { + matches!(self.kind, FileSystemSandboxKind::Restricted) + && self + .entries + .iter() + .any(|entry| entry.access == FileSystemAccessMode::None) + } + + pub fn from_legacy_sandbox_policy_preserving_deny_entries( + sandbox_policy: &SandboxPolicy, + cwd: &Path, + existing: &Self, + ) -> Self { + let mut rebuilt = Self::from_legacy_sandbox_policy(sandbox_policy, cwd); + if !matches!(rebuilt.kind, FileSystemSandboxKind::Restricted) { + return rebuilt; + } + + for deny_entry in existing + .entries + .iter() + .filter(|entry| entry.access == FileSystemAccessMode::None) + { + if !rebuilt.entries.iter().any(|entry| entry == deny_entry) { + rebuilt.entries.push(deny_entry.clone()); + } + } + + rebuilt + } + + /// Returns true when a restricted policy contains any entry that really + /// reduces a broader `:root = write` grant. + /// + /// Raw entry presence is not enough here: an equally specific `write` + /// entry for the same target wins under the normal precedence rules, so a + /// shadowed `read` entry must not downgrade the policy out of full-disk + /// write mode. + fn has_write_narrowing_entries(&self) -> bool { + matches!(self.kind, FileSystemSandboxKind::Restricted) + && self.entries.iter().any(|entry| { + if entry.access.can_write() { + return false; + } + + match &entry.path { + FileSystemPath::Path { .. } => !self.has_same_target_write_override(entry), + FileSystemPath::GlobPattern { .. } => true, + FileSystemPath::Special { value } => match value { + FileSystemSpecialPath::Root => entry.access == FileSystemAccessMode::None, + FileSystemSpecialPath::Minimal | FileSystemSpecialPath::Unknown { .. } => { + false + } + _ => !self.has_same_target_write_override(entry), + }, + } + }) + } + + /// Returns true when a higher-priority `write` entry targets the same + /// location as `entry`, so `entry` cannot narrow effective write access. + fn has_same_target_write_override(&self, entry: &FileSystemSandboxEntry) -> bool { + self.entries.iter().any(|candidate| { + candidate.access.can_write() + && candidate.access > entry.access + && file_system_paths_share_target(&candidate.path, &entry.path) + }) + } + /// Converts a legacy sandbox policy into an equivalent filesystem policy /// for the provided cwd. /// @@ -276,6 +389,7 @@ impl FileSystemSandboxPolicy { FileSystemPath::Path { path } => !legacy_writable_roots .iter() .any(|root| root.is_path_writable(path.as_path())), + FileSystemPath::GlobPattern { .. } => true, FileSystemPath::Special { .. } => true, } }); @@ -312,7 +426,7 @@ impl FileSystemSandboxPolicy { FileSystemSandboxKind::Unrestricted | FileSystemSandboxKind::ExternalSandbox => true, FileSystemSandboxKind::Restricted => { self.has_root_access(FileSystemAccessMode::can_read) - && !self.has_explicit_deny_entries() + && !self.has_denied_read_restrictions() } } } @@ -585,6 +699,29 @@ impl FileSystemSandboxPolicy { ) } + /// Returns unreadable glob patterns resolved against the provided cwd. + pub fn get_unreadable_globs_with_cwd(&self, cwd: &Path) -> Vec { + if !matches!(self.kind, FileSystemSandboxKind::Restricted) { + return Vec::new(); + } + + let mut patterns = self + .entries + .iter() + .filter(|entry| entry.access == FileSystemAccessMode::None) + .filter_map(|entry| match &entry.path { + FileSystemPath::GlobPattern { pattern } => { + Some(AbsolutePathBuf::resolve_path_against_base(pattern, cwd)) + } + FileSystemPath::Path { .. } | FileSystemPath::Special { .. } => None, + }) + .map(|pattern| pattern.to_string_lossy().into_owned()) + .collect::>(); + patterns.sort(); + patterns.dedup(); + patterns + } + pub fn to_legacy_sandbox_policy( &self, network_policy: NetworkSandboxPolicy, @@ -620,6 +757,7 @@ impl FileSystemSandboxPolicy { for entry in &self.entries { match &entry.path { + FileSystemPath::GlobPattern { .. } => {} FileSystemPath::Path { path } => { if entry.access.can_write() { if cwd_absolute.as_ref().is_some_and(|cwd| cwd == path) { @@ -770,6 +908,7 @@ impl FileSystemSandboxPolicy { readable_roots: self.get_readable_roots_with_cwd(cwd), writable_roots: self.get_writable_roots_with_cwd(cwd), unreadable_roots: self.get_unreadable_roots_with_cwd(cwd), + unreadable_globs: self.get_unreadable_globs_with_cwd(cwd), } } } @@ -905,6 +1044,7 @@ fn resolve_file_system_path( ) -> Option { match path { FileSystemPath::Path { path } => Some(path.clone()), + FileSystemPath::GlobPattern { .. } => None, FileSystemPath::Special { value } => resolve_file_system_special_path(value, cwd), } } @@ -947,6 +1087,11 @@ fn file_system_paths_share_target(left: &FileSystemPath, right: &FileSystemPath) | (FileSystemPath::Special { value }, FileSystemPath::Path { path }) => { special_path_matches_absolute_path(value, path) } + ( + FileSystemPath::GlobPattern { pattern: left }, + FileSystemPath::GlobPattern { pattern: right }, + ) => left == right, + (FileSystemPath::GlobPattern { .. }, _) | (_, FileSystemPath::GlobPattern { .. }) => false, } } @@ -1021,6 +1166,44 @@ fn absolute_root_path_for_cwd(cwd: &AbsolutePathBuf) -> AbsolutePathBuf { .unwrap_or_else(|err| panic!("cwd root must be an absolute path: {err}")) } +fn normalized_and_canonical_candidates(path: &Path) -> Vec { + // Compare the lexical absolute form plus the canonical target when it + // exists. Missing paths still need the lexical candidate so future-created + // denied paths remain blocked by direct tool checks. + let mut candidates = Vec::new(); + + if let Ok(normalized) = AbsolutePathBuf::from_absolute_path(path) { + push_unique(&mut candidates, normalized.to_path_buf()); + } else { + push_unique(&mut candidates, path.to_path_buf()); + } + + if let Ok(canonical) = path.canonicalize() + && let Ok(canonical_absolute) = AbsolutePathBuf::from_absolute_path(canonical) + { + push_unique(&mut candidates, canonical_absolute.to_path_buf()); + } + + candidates +} + +fn push_unique(candidates: &mut Vec, candidate: PathBuf) { + if !candidates.iter().any(|existing| existing == &candidate) { + candidates.push(candidate); + } +} + +fn build_glob_matcher(pattern: &str) -> Option { + // Keep `*` and `?` within a single path component and preserve an unclosed + // `[` as a literal so matcher behavior stays aligned with config parsing. + GlobBuilder::new(pattern) + .literal_separator(true) + .allow_unclosed_class(true) + .build() + .ok() + .map(|glob| glob.compile_matcher()) +} + fn resolve_file_system_special_path( value: &FileSystemSpecialPath, cwd: Option<&AbsolutePathBuf>, @@ -2061,4 +2244,187 @@ mod tests { assert!(FileSystemAccessMode::Write > FileSystemAccessMode::Read); assert!(FileSystemAccessMode::None > FileSystemAccessMode::Write); } + + #[test] + fn legacy_bridge_preserves_explicit_deny_entries() { + let denied = AbsolutePathBuf::try_from("/tmp/private").expect("absolute path"); + let existing = FileSystemSandboxPolicy::restricted(vec![FileSystemSandboxEntry { + path: FileSystemPath::Path { + path: denied.clone(), + }, + access: FileSystemAccessMode::None, + }]); + + let rebuilt = FileSystemSandboxPolicy::from_legacy_sandbox_policy_preserving_deny_entries( + &SandboxPolicy::new_workspace_write_policy(), + Path::new("/tmp/workspace"), + &existing, + ); + + assert!( + rebuilt.entries.iter().any(|entry| { + entry.path + == FileSystemPath::Path { + path: denied.clone(), + } + && entry.access == FileSystemAccessMode::None + }), + "expected explicit deny entry to be preserved" + ); + } + + fn deny_policy(path: &Path) -> FileSystemSandboxPolicy { + FileSystemSandboxPolicy::restricted(vec![FileSystemSandboxEntry { + path: FileSystemPath::Path { + path: AbsolutePathBuf::try_from(path).expect("absolute deny path"), + }, + access: FileSystemAccessMode::None, + }]) + } + + fn unreadable_glob_entry(pattern: String) -> FileSystemSandboxEntry { + FileSystemSandboxEntry { + path: FileSystemPath::GlobPattern { pattern }, + access: FileSystemAccessMode::None, + } + } + + fn default_policy_with_unreadable_glob(pattern: String) -> FileSystemSandboxPolicy { + let mut policy = FileSystemSandboxPolicy::default(); + policy.entries.push(unreadable_glob_entry(pattern)); + policy + } + + fn is_read_denied( + path: &Path, + file_system_sandbox_policy: &FileSystemSandboxPolicy, + cwd: &Path, + ) -> bool { + ReadDenyMatcher::new(file_system_sandbox_policy, cwd) + .is_some_and(|matcher| matcher.is_read_denied(path)) + } + + #[test] + fn exact_path_and_descendants_are_denied() { + let temp = TempDir::new().expect("tempdir"); + let denied_dir = temp.path().join("denied"); + let nested = denied_dir.join("nested.txt"); + std::fs::create_dir_all(&denied_dir).expect("create denied dir"); + std::fs::write(&nested, "secret").expect("write secret"); + + let policy = deny_policy(&denied_dir); + assert!(is_read_denied(&denied_dir, &policy, temp.path())); + assert!(is_read_denied(&nested, &policy, temp.path())); + assert!(!is_read_denied( + &temp.path().join("other.txt"), + &policy, + temp.path() + )); + } + + #[cfg(unix)] + #[test] + fn canonical_target_matches_denied_symlink_alias() { + let temp = TempDir::new().expect("tempdir"); + let real_dir = temp.path().join("real"); + let alias_dir = temp.path().join("alias"); + std::fs::create_dir_all(&real_dir).expect("create real dir"); + symlink_dir(&real_dir, &alias_dir).expect("symlink alias"); + + let secret = real_dir.join("secret.txt"); + std::fs::write(&secret, "secret").expect("write secret"); + let alias_secret = alias_dir.join("secret.txt"); + + let policy = deny_policy(&real_dir); + assert!(is_read_denied(&alias_secret, &policy, temp.path())); + } + + #[test] + fn literal_patterns_and_globs_are_denied() { + let temp = TempDir::new().expect("tempdir"); + let literal = temp.path().join("private"); + let other = temp.path().join("notes.txt"); + std::fs::create_dir_all(&literal).expect("create literal dir"); + std::fs::write(&other, "notes").expect("write notes"); + + let mut policy = deny_policy(&literal); + policy.entries.push(unreadable_glob_entry(format!( + "{}/**/*.txt", + temp.path().display() + ))); + + assert!(is_read_denied(&literal, &policy, temp.path())); + assert!(is_read_denied(&other, &policy, temp.path())); + } + + #[test] + fn glob_patterns_deny_matching_paths() { + let temp = TempDir::new().expect("tempdir"); + let denied = temp.path().join("private").join("secret1.txt"); + std::fs::create_dir_all(denied.parent().expect("parent")).expect("create parent"); + std::fs::write(&denied, "secret").expect("write secret"); + + let policy = default_policy_with_unreadable_glob(format!( + "{}/private/secret?.txt", + temp.path().display() + )); + + assert!(is_read_denied(&denied, &policy, temp.path())); + } + + #[test] + fn glob_patterns_do_not_cross_path_separators() { + let temp = TempDir::new().expect("tempdir"); + let matching = temp.path().join("app").join("file42.txt"); + let nested = temp.path().join("app").join("nested").join("file42.txt"); + let short = temp.path().join("app").join("file4.txt"); + let letters = temp.path().join("app").join("fileab.txt"); + std::fs::create_dir_all(nested.parent().expect("parent")).expect("create parent"); + std::fs::write(&matching, "secret").expect("write matching"); + std::fs::write(&nested, "secret").expect("write nested"); + std::fs::write(&short, "secret").expect("write short"); + std::fs::write(&letters, "secret").expect("write letters"); + + let policy = default_policy_with_unreadable_glob(format!( + "{}/*/file[0-9]?.txt", + temp.path().display() + )); + + assert!(is_read_denied(&matching, &policy, temp.path())); + assert!(!is_read_denied(&nested, &policy, temp.path())); + assert!(!is_read_denied(&short, &policy, temp.path())); + assert!(!is_read_denied(&letters, &policy, temp.path())); + } + + #[test] + fn globstar_patterns_deny_root_and_nested_matches() { + let temp = TempDir::new().expect("tempdir"); + let root_env = temp.path().join(".env"); + let nested_env = temp.path().join("app").join(".env"); + let other = temp.path().join("app").join("notes.txt"); + std::fs::create_dir_all(nested_env.parent().expect("parent")).expect("create parent"); + std::fs::write(&root_env, "secret").expect("write root env"); + std::fs::write(&nested_env, "secret").expect("write nested env"); + std::fs::write(&other, "notes").expect("write notes"); + + let policy = + default_policy_with_unreadable_glob(format!("{}/**/*.env", temp.path().display())); + + assert!(is_read_denied(&root_env, &policy, temp.path())); + assert!(is_read_denied(&nested_env, &policy, temp.path())); + assert!(!is_read_denied(&other, &policy, temp.path())); + } + + #[test] + fn unclosed_character_classes_match_literal_brackets() { + let temp = TempDir::new().expect("tempdir"); + let bracket_file = temp.path().join("["); + let other = temp.path().join("notes.txt"); + std::fs::write(&bracket_file, "secret").expect("write bracket file"); + std::fs::write(&other, "notes").expect("write notes"); + let policy = default_policy_with_unreadable_glob(format!("{}/[", temp.path().display())); + + assert!(is_read_denied(&bracket_file, &policy, temp.path())); + assert!(!is_read_denied(&other, &policy, temp.path())); + } } diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index fb1ea592da..8aea3544d4 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -2844,6 +2844,8 @@ pub struct TurnContextItem { pub sandbox_policy: SandboxPolicy, #[serde(skip_serializing_if = "Option::is_none")] pub network: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub file_system_sandbox_policy: Option, pub model: String, #[serde(skip_serializing_if = "Option::is_none")] pub personality: Option, @@ -4974,6 +4976,7 @@ mod tests { assert_eq!(item.trace_id, None); assert_eq!(item.network, None); + assert_eq!(item.file_system_sandbox_policy, None); Ok(()) } @@ -4991,6 +4994,14 @@ mod tests { allowed_domains: vec!["api.example.com".to_string()], denied_domains: vec!["blocked.example.com".to_string()], }), + file_system_sandbox_policy: Some(FileSystemSandboxPolicy::restricted(vec![ + FileSystemSandboxEntry { + path: FileSystemPath::GlobPattern { + pattern: "/tmp/private/**/*.txt".to_string(), + }, + access: FileSystemAccessMode::None, + }, + ])), model: "gpt-5".to_string(), personality: None, collaboration_mode: None, @@ -5011,6 +5022,19 @@ mod tests { "denied_domains": ["blocked.example.com"], }) ); + assert_eq!( + value["file_system_sandbox_policy"], + json!({ + "kind": "restricted", + "entries": [{ + "path": { + "type": "glob_pattern", + "pattern": "/tmp/private/**/*.txt" + }, + "access": "none" + }] + }) + ); Ok(()) } diff --git a/codex-rs/rollout/src/recorder_tests.rs b/codex-rs/rollout/src/recorder_tests.rs index 3e2ac52194..e3d82c4454 100644 --- a/codex-rs/rollout/src/recorder_tests.rs +++ b/codex-rs/rollout/src/recorder_tests.rs @@ -544,6 +544,7 @@ async fn resume_candidate_matches_cwd_reads_latest_turn_context() -> std::io::Re approval_policy: AskForApproval::Never, sandbox_policy: SandboxPolicy::new_read_only_policy(), network: None, + file_system_sandbox_policy: None, model: "test-model".to_string(), personality: None, collaboration_mode: None, diff --git a/codex-rs/sandboxing/src/lib.rs b/codex-rs/sandboxing/src/lib.rs index 38aed6ce24..dbb8323649 100644 --- a/codex-rs/sandboxing/src/lib.rs +++ b/codex-rs/sandboxing/src/lib.rs @@ -34,6 +34,9 @@ impl From for CodexErr { SandboxTransformError::MissingLinuxSandboxExecutable => { CodexErr::LandlockSandboxExecutableNotProvided } + SandboxTransformError::UnreadableGlobPatternsUnsupported => { + CodexErr::UnsupportedOperation(err.to_string()) + } #[cfg(target_os = "linux")] SandboxTransformError::Wsl1UnsupportedForBubblewrap => { CodexErr::UnsupportedOperation(crate::bwrap::WSL1_BWRAP_WARNING.to_string()) diff --git a/codex-rs/sandboxing/src/manager.rs b/codex-rs/sandboxing/src/manager.rs index 0836eb2565..1e3ea9c196 100644 --- a/codex-rs/sandboxing/src/manager.rs +++ b/codex-rs/sandboxing/src/manager.rs @@ -109,6 +109,7 @@ pub struct SandboxTransformRequest<'a> { #[derive(Debug)] pub enum SandboxTransformError { MissingLinuxSandboxExecutable, + UnreadableGlobPatternsUnsupported, #[cfg(target_os = "linux")] Wsl1UnsupportedForBubblewrap, #[cfg(not(target_os = "macos"))] @@ -121,6 +122,10 @@ impl std::fmt::Display for SandboxTransformError { Self::MissingLinuxSandboxExecutable => { write!(f, "missing codex-linux-sandbox executable path") } + Self::UnreadableGlobPatternsUnsupported => write!( + f, + "platform sandbox backend cannot enforce unreadable glob patterns" + ), #[cfg(target_os = "linux")] Self::Wsl1UnsupportedForBubblewrap => write!(f, "{WSL1_BWRAP_WARNING}"), #[cfg(not(target_os = "macos"))] @@ -196,6 +201,15 @@ impl SandboxManager { ); let effective_network_policy = effective_network_sandbox_policy(network_policy, additional_permissions.as_ref()); + if matches!( + sandbox, + SandboxType::MacosSeatbelt | SandboxType::LinuxSeccomp + ) && !effective_file_system_policy + .get_unreadable_globs_with_cwd(sandbox_policy_cwd) + .is_empty() + { + return Err(SandboxTransformError::UnreadableGlobPatternsUnsupported); + } let mut argv = Vec::with_capacity(1 + command.args.len()); argv.push(command.program); argv.extend(command.args.into_iter().map(OsString::from)); diff --git a/codex-rs/state/src/extract.rs b/codex-rs/state/src/extract.rs index 91c6755ffe..6d36f211a4 100644 --- a/codex-rs/state/src/extract.rs +++ b/codex-rs/state/src/extract.rs @@ -302,6 +302,7 @@ mod tests { approval_policy: AskForApproval::Never, sandbox_policy: SandboxPolicy::DangerFullAccess, network: None, + file_system_sandbox_policy: None, model: "gpt-5".to_string(), personality: None, collaboration_mode: None, @@ -340,6 +341,7 @@ mod tests { approval_policy: AskForApproval::OnRequest, sandbox_policy: SandboxPolicy::new_read_only_policy(), network: None, + file_system_sandbox_policy: None, model: "gpt-5".to_string(), personality: None, collaboration_mode: None, @@ -372,6 +374,7 @@ mod tests { approval_policy: AskForApproval::OnRequest, sandbox_policy: SandboxPolicy::new_read_only_policy(), network: None, + file_system_sandbox_policy: None, model: "gpt-5".to_string(), personality: None, collaboration_mode: None, diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index 5641131566..1db3dbc7ed 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -9239,6 +9239,7 @@ guardian_approval = true approval_policy: primary_session.approval_policy, sandbox_policy: primary_session.sandbox_policy.clone(), network: None, + file_system_sandbox_policy: None, model: "gpt-agent".to_string(), personality: None, collaboration_mode: None, diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index cf938545c6..a5efcd1ea2 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -2249,6 +2249,7 @@ mod tests { approval_policy: config.permissions.approval_policy.value(), sandbox_policy: config.permissions.sandbox_policy.get().clone(), network: None, + file_system_sandbox_policy: None, model, personality: None, collaboration_mode: None, From 109b22a8d024084cd86704dc183c6616891fac96 Mon Sep 17 00:00:00 2001 From: alexsong-oai Date: Thu, 16 Apr 2026 10:34:38 -0700 Subject: [PATCH 28/59] Improve external agent plugin migration for configured marketplaces (#18055) --- .../src/external_agent_config_api.rs | 1 + codex-rs/core/src/external_agent_config.rs | 253 +++-- .../core/src/external_agent_config_tests.rs | 989 ++++++++++++++++-- codex-rs/core/src/plugins/manager.rs | 2 +- codex-rs/core/src/plugins/marketplace_add.rs | 2 +- .../src/plugins/marketplace_add/source.rs | 4 +- codex-rs/core/src/plugins/mod.rs | 3 + 7 files changed, 1111 insertions(+), 143 deletions(-) diff --git a/codex-rs/app-server/src/external_agent_config_api.rs b/codex-rs/app-server/src/external_agent_config_api.rs index 3e20632db2..2f90a55a13 100644 --- a/codex-rs/app-server/src/external_agent_config_api.rs +++ b/codex-rs/app-server/src/external_agent_config_api.rs @@ -37,6 +37,7 @@ impl ExternalAgentConfigApi { include_home: params.include_home, cwds: params.cwds, }) + .await .map_err(map_io_error)?; Ok(ExternalAgentConfigDetectResponse { diff --git a/codex-rs/core/src/external_agent_config.rs b/codex-rs/core/src/external_agent_config.rs index 18305a6262..f0f4443a75 100644 --- a/codex-rs/core/src/external_agent_config.rs +++ b/codex-rs/core/src/external_agent_config.rs @@ -1,8 +1,15 @@ +use crate::config::Config; +use crate::config::ConfigBuilder; use crate::plugins::MarketplaceAddRequest; use crate::plugins::PluginId; use crate::plugins::PluginInstallRequest; use crate::plugins::PluginsManager; use crate::plugins::add_marketplace; +use crate::plugins::configured_plugins_from_stack; +use crate::plugins::find_marketplace_manifest_path; +use crate::plugins::parse_marketplace_source; +use codex_core_plugins::marketplace::MarketplacePluginInstallPolicy; +use codex_protocol::protocol::Product; use serde_json::Value as JsonValue; use std::collections::BTreeMap; use std::collections::HashSet; @@ -15,8 +22,8 @@ use toml::Value as TomlValue; const EXTERNAL_AGENT_CONFIG_DETECT_METRIC: &str = "codex.external_agent_config.detect"; const EXTERNAL_AGENT_CONFIG_IMPORT_METRIC: &str = "codex.external_agent_config.import"; -// Installed marketplace roots always expose their manifest at this relative path. -const INSTALLED_MARKETPLACE_MANIFEST_RELATIVE_PATH: &str = ".agents/plugins/marketplace.json"; +const EXTERNAL_AGENT_DIR: &str = ".claude"; +const EXTERNAL_AGENT_CONFIG_MD: &str = "CLAUDE.md"; #[derive(Debug, Clone, PartialEq, Eq)] pub struct ExternalAgentConfigDetectOptions { @@ -83,20 +90,21 @@ impl ExternalAgentConfigService { } } - pub fn detect( + pub async fn detect( &self, params: ExternalAgentConfigDetectOptions, ) -> io::Result> { let mut items = Vec::new(); if params.include_home { - self.detect_migrations(/*repo_root*/ None, &mut items)?; + self.detect_migrations(/*repo_root*/ None, &mut items) + .await?; } for cwd in params.cwds.as_deref().unwrap_or(&[]) { let Some(repo_root) = find_repo_root(Some(cwd))? else { continue; }; - self.detect_migrations(Some(&repo_root), &mut items)?; + self.detect_migrations(Some(&repo_root), &mut items).await?; } Ok(items) @@ -137,22 +145,11 @@ impl ExternalAgentConfigService { let cwd = migration_item.cwd; let details = migration_item.details; tokio::spawn(async move { - match service.import_plugins(cwd.as_deref(), details).await { - Ok(outcome) => { - tracing::info!( - succeeded_marketplaces = outcome.succeeded_marketplaces.len(), - succeeded_plugin_ids = outcome.succeeded_plugin_ids.len(), - failed_marketplaces = outcome.failed_marketplaces.len(), - failed_plugin_ids = outcome.failed_plugin_ids.len(), - "external agent config plugin import completed" - ); - } - Err(err) => { - tracing::warn!( - error = %err, - "external agent config plugin import failed" - ); - } + if let Err(err) = service.import_plugins(cwd.as_deref(), details).await { + tracing::warn!( + error = %err, + "external agent config plugin import failed" + ); } }); emit_migration_metric( @@ -168,7 +165,7 @@ impl ExternalAgentConfigService { Ok(()) } - fn detect_migrations( + async fn detect_migrations( &self, repo_root: Option<&Path>, items: &mut Vec, @@ -176,7 +173,7 @@ impl ExternalAgentConfigService { let cwd = repo_root.map(Path::to_path_buf); let source_settings = repo_root.map_or_else( || self.external_agent_home.join("settings.json"), - |repo_root| repo_root.join(".claude").join("settings.json"), + |repo_root| repo_root.join(EXTERNAL_AGENT_DIR).join("settings.json"), ); let settings = read_external_settings(&source_settings)?; let target_config = repo_root.map_or_else( @@ -219,16 +216,9 @@ impl ExternalAgentConfigService { } } - self.detect_plugin_migration( - source_settings.as_path(), - cwd.clone(), - settings.as_ref(), - items, - ); - let source_skills = repo_root.map_or_else( || self.external_agent_home.join("skills"), - |repo_root| repo_root.join(".claude").join("skills"), + |repo_root| repo_root.join(EXTERNAL_AGENT_DIR).join("skills"), ); let target_skills = repo_root.map_or_else( || self.home_target_skills_dir(), @@ -239,7 +229,7 @@ impl ExternalAgentConfigService { items.push(ExternalAgentConfigMigrationItem { item_type: ExternalAgentConfigMigrationItemType::Skills, description: format!( - "Copy skill folders from {} to {}", + "Migrate skills from {} to {}", source_skills.display(), target_skills.display() ), @@ -256,7 +246,7 @@ impl ExternalAgentConfigService { let source_agents_md = if let Some(repo_root) = repo_root { find_repo_agents_md_source(repo_root)? } else { - let path = self.external_agent_home.join("CLAUDE.md"); + let path = self.external_agent_home.join(EXTERNAL_AGENT_CONFIG_MD); is_non_empty_text_file(&path)?.then_some(path) }; let target_agents_md = repo_root.map_or_else( @@ -273,7 +263,7 @@ impl ExternalAgentConfigService { source_agents_md.display(), target_agents_md.display() ), - cwd, + cwd: cwd.clone(), details: None, }); emit_migration_metric( @@ -283,6 +273,43 @@ impl ExternalAgentConfigService { ); } + if let Some(settings) = settings.as_ref() { + match ConfigBuilder::default() + .codex_home(self.codex_home.clone()) + .fallback_cwd(Some(self.codex_home.clone())) + .build() + .await + { + Ok(config) => { + let configured_plugin_ids = + configured_plugins_from_stack(&config.config_layer_stack) + .into_keys() + .collect::>(); + let configured_marketplace_plugins = configured_marketplace_plugins( + &config, + &PluginsManager::new(self.codex_home.clone()), + )?; + if let Some(item) = self.detect_plugin_migration( + source_settings.as_path(), + repo_root.unwrap_or(self.external_agent_home.as_path()), + cwd.clone(), + settings, + &configured_plugin_ids, + &configured_marketplace_plugins, + ) { + items.push(item); + } + } + Err(err) => { + tracing::warn!( + error = %err, + settings_path = %source_settings.display(), + "skipping external agent plugin migration detection because config load failed" + ); + } + } + } + Ok(()) } @@ -296,25 +323,30 @@ impl ExternalAgentConfigService { fn detect_plugin_migration( &self, source_settings: &Path, + source_root: &Path, cwd: Option, - settings: Option<&JsonValue>, - items: &mut Vec, - ) { - let Some(plugin_details) = settings.and_then(extract_plugin_migration_details) else { - return; - }; - - items.push(ExternalAgentConfigMigrationItem { - item_type: ExternalAgentConfigMigrationItemType::Plugins, - description: format!("Import enabled plugins from {}", source_settings.display()), - cwd, - details: Some(plugin_details), - }); + settings: &JsonValue, + configured_plugin_ids: &HashSet, + configured_marketplace_plugins: &BTreeMap>, + ) -> Option { + let plugin_details = extract_plugin_migration_details( + settings, + source_root, + configured_plugin_ids, + configured_marketplace_plugins, + )?; emit_migration_metric( EXTERNAL_AGENT_CONFIG_DETECT_METRIC, ExternalAgentConfigMigrationItemType::Plugins, /*skills_count*/ None, ); + + Some(ExternalAgentConfigMigrationItem { + item_type: ExternalAgentConfigMigrationItemType::Plugins, + description: format!("Import enabled plugins from {}", source_settings.display()), + cwd, + details: Some(plugin_details), + }) } async fn import_plugins( @@ -336,8 +368,14 @@ impl ExternalAgentConfigService { .iter() .map(|plugin_name| format!("{plugin_name}@{marketplace_name}")) .collect::>(); - let import_source = - read_marketplace_import_source(cwd, &self.external_agent_home, &marketplace_name)?; + let source_settings = cwd.map_or_else( + || self.external_agent_home.join("settings.json"), + |cwd| cwd.join(EXTERNAL_AGENT_DIR).join("settings.json"), + ); + let source_root = cwd.unwrap_or(self.external_agent_home.as_path()); + let import_source = read_external_settings(&source_settings)?.and_then(|settings| { + collect_marketplace_import_sources(&settings, source_root).remove(&marketplace_name) + }); let Some(import_source) = import_source else { outcome.failed_marketplaces.push(marketplace_name); outcome.failed_plugin_ids.extend(plugin_ids); @@ -351,12 +389,17 @@ impl ExternalAgentConfigService { let add_marketplace_outcome = add_marketplace(self.codex_home.clone(), request).await; let marketplace_path = match add_marketplace_outcome { Ok(add_marketplace_outcome) => { + let Some(marketplace_path) = find_marketplace_manifest_path( + add_marketplace_outcome.installed_root.as_path(), + ) else { + outcome.failed_marketplaces.push(marketplace_name); + outcome.failed_plugin_ids.extend(plugin_ids); + continue; + }; outcome .succeeded_marketplaces .push(marketplace_name.clone()); - add_marketplace_outcome - .installed_root - .join(INSTALLED_MARKETPLACE_MANIFEST_RELATIVE_PATH) + marketplace_path } Err(_) => { outcome.failed_marketplaces.push(marketplace_name); @@ -388,7 +431,7 @@ impl ExternalAgentConfigService { fn import_config(&self, cwd: Option<&Path>) -> io::Result<()> { let (source_settings, target_config) = if let Some(repo_root) = find_repo_root(cwd)? { ( - repo_root.join(".claude").join("settings.json"), + repo_root.join(EXTERNAL_AGENT_DIR).join("settings.json"), repo_root.join(".codex").join("config.toml"), ) } else if cwd.is_some_and(|cwd| !cwd.as_os_str().is_empty()) { @@ -440,7 +483,7 @@ impl ExternalAgentConfigService { fn import_skills(&self, cwd: Option<&Path>) -> io::Result { let (source_skills, target_skills) = if let Some(repo_root) = find_repo_root(cwd)? { ( - repo_root.join(".claude").join("skills"), + repo_root.join(EXTERNAL_AGENT_DIR).join("skills"), repo_root.join(".agents").join("skills"), ) } else if cwd.is_some_and(|cwd| !cwd.as_os_str().is_empty()) { @@ -487,7 +530,7 @@ impl ExternalAgentConfigService { return Ok(()); } else { ( - self.external_agent_home.join("CLAUDE.md"), + self.external_agent_home.join(EXTERNAL_AGENT_CONFIG_MD), self.codex_home.join("AGENTS.md"), ) }; @@ -508,10 +551,10 @@ impl ExternalAgentConfigService { fn default_external_agent_home() -> PathBuf { if let Some(home) = std::env::var_os("HOME").or_else(|| std::env::var_os("USERPROFILE")) { - return PathBuf::from(home).join(".claude"); + return PathBuf::from(home).join(EXTERNAL_AGENT_DIR); } - PathBuf::from(".claude") + PathBuf::from(EXTERNAL_AGENT_DIR) } fn read_external_settings(path: &Path) -> io::Result> { @@ -525,12 +568,37 @@ fn read_external_settings(path: &Path) -> io::Result> { Ok(Some(settings)) } -fn extract_plugin_migration_details(settings: &JsonValue) -> Option { +fn extract_plugin_migration_details( + settings: &JsonValue, + source_root: &Path, + configured_plugin_ids: &HashSet, + configured_marketplace_plugins: &BTreeMap>, +) -> Option { + let loadable_marketplaces = collect_marketplace_import_sources(settings, source_root) + .into_iter() + .filter_map(|(marketplace_name, source)| { + parse_marketplace_source(&source.source, source.ref_name) + .ok() + .map(|_| marketplace_name) + }) + .collect::>(); let mut plugins = BTreeMap::new(); - for plugin_id in collect_enabled_plugins(settings) { + for plugin_id in collect_enabled_plugins(settings) + .into_iter() + .filter(|plugin_id| !configured_plugin_ids.contains(plugin_id)) + { let Ok(plugin_id) = PluginId::parse(&plugin_id) else { continue; }; + if let Some(installable_plugins) = + configured_marketplace_plugins.get(&plugin_id.marketplace_name) + { + if !installable_plugins.contains(&plugin_id.plugin_name) { + continue; + } + } else if !loadable_marketplaces.contains(&plugin_id.marketplace_name) { + continue; + } let plugin_group = plugins .entry(plugin_id.marketplace_name.clone()) .or_insert_with(|| PluginsMigration { @@ -579,8 +647,40 @@ fn collect_enabled_plugins(settings: &JsonValue) -> Vec { .collect() } +fn configured_marketplace_plugins( + config: &Config, + plugins_manager: &PluginsManager, +) -> io::Result>> { + let marketplaces = plugins_manager + .list_marketplaces_for_config(config, &[]) + .map_err(|err| { + invalid_data_error(format!("failed to list configured marketplaces: {err}")) + })?; + let mut marketplace_plugins = BTreeMap::new(); + for marketplace in marketplaces.marketplaces { + let plugins = marketplace + .plugins + .into_iter() + .filter(|plugin| { + plugin.policy.installation != MarketplacePluginInstallPolicy::NotAvailable + }) + .filter(|plugin| { + plugin + .policy + .products + .as_deref() + .is_none_or(|products| Product::Codex.matches_product_restriction(products)) + }) + .map(|plugin| plugin.name) + .collect::>(); + marketplace_plugins.insert(marketplace.name, plugins); + } + Ok(marketplace_plugins) +} + fn collect_marketplace_import_sources( settings: &JsonValue, + source_root: &Path, ) -> BTreeMap { let Some(extra_known_marketplaces) = settings .as_object() @@ -611,6 +711,7 @@ fn collect_marketplace_import_sources( if source.is_empty() { return None; } + let source = resolve_external_marketplace_source(&source, source_root); let ref_name = source_fields .get("ref") @@ -631,20 +732,16 @@ struct MarketplaceImportSource { ref_name: Option, } -fn read_marketplace_import_source( - cwd: Option<&Path>, - external_agent_home: &Path, - marketplace_name: &str, -) -> io::Result> { - let source_settings = cwd.map_or_else( - || external_agent_home.join("settings.json"), - |cwd| cwd.join(".claude").join("settings.json"), - ); - let Some(settings) = read_external_settings(&source_settings)? else { - return Ok(None); - }; +fn resolve_external_marketplace_source(source: &str, source_root: &Path) -> String { + if !looks_like_relative_local_path(source) { + return source.to_string(); + } - Ok(collect_marketplace_import_sources(&settings).remove(marketplace_name)) + source_root.join(source).display().to_string() +} + +fn looks_like_relative_local_path(source: &str) -> bool { + source.starts_with("./") || source.starts_with("../") || source == "." || source == ".." } fn find_repo_root(cwd: Option<&Path>) -> io::Result> { @@ -729,8 +826,10 @@ fn is_non_empty_text_file(path: &Path) -> io::Result { fn find_repo_agents_md_source(repo_root: &Path) -> io::Result> { for candidate in [ - repo_root.join("CLAUDE.md"), - repo_root.join(".claude").join("CLAUDE.md"), + repo_root.join(EXTERNAL_AGENT_CONFIG_MD), + repo_root + .join(EXTERNAL_AGENT_DIR) + .join(EXTERNAL_AGENT_CONFIG_MD), ] { if is_non_empty_text_file(&candidate)? { return Ok(Some(candidate)); @@ -779,7 +878,11 @@ fn rewrite_and_copy_text_file(source: &Path, target: &Path) -> io::Result<()> { } fn rewrite_external_agent_terms(content: &str) -> String { - let mut rewritten = replace_case_insensitive_with_boundaries(content, "claude.md", "AGENTS.md"); + let mut rewritten = replace_case_insensitive_with_boundaries( + content, + &EXTERNAL_AGENT_CONFIG_MD.to_ascii_lowercase(), + "AGENTS.md", + ); for from in [ "claude code", "claude-code", diff --git a/codex-rs/core/src/external_agent_config_tests.rs b/codex-rs/core/src/external_agent_config_tests.rs index f2afab0170..f01a40e3d2 100644 --- a/codex-rs/core/src/external_agent_config_tests.rs +++ b/codex-rs/core/src/external_agent_config_tests.rs @@ -5,13 +5,16 @@ use tempfile::TempDir; fn fixture_paths() -> (TempDir, PathBuf, PathBuf) { let root = TempDir::new().expect("create tempdir"); - let claude_home = root.path().join(".claude"); + let external_agent_home = root.path().join(".claude"); let codex_home = root.path().join(".codex"); - (root, claude_home, codex_home) + (root, external_agent_home, codex_home) } -fn service_for_paths(claude_home: PathBuf, codex_home: PathBuf) -> ExternalAgentConfigService { - ExternalAgentConfigService::new_for_test(codex_home, claude_home) +fn service_for_paths( + external_agent_home: PathBuf, + codex_home: PathBuf, +) -> ExternalAgentConfigService { + ExternalAgentConfigService::new_for_test(codex_home, external_agent_home) } fn github_plugin_details() -> MigrationDetails { @@ -23,26 +26,28 @@ fn github_plugin_details() -> MigrationDetails { } } -#[test] -fn detect_home_lists_config_skills_and_agents_md() { - let (_root, claude_home, codex_home) = fixture_paths(); +#[tokio::test] +async fn detect_home_lists_config_skills_and_agents_md() { + let (_root, external_agent_home, codex_home) = fixture_paths(); let agents_skills = codex_home .parent() .map(|parent| parent.join(".agents").join("skills")) .unwrap_or_else(|| PathBuf::from(".agents").join("skills")); - fs::create_dir_all(claude_home.join("skills").join("skill-a")).expect("create skills"); - fs::write(claude_home.join("CLAUDE.md"), "claude rules").expect("write claude md"); + fs::create_dir_all(external_agent_home.join("skills").join("skill-a")).expect("create skills"); + fs::write(external_agent_home.join("CLAUDE.md"), "claude rules") + .expect("write external agent md"); fs::write( - claude_home.join("settings.json"), + external_agent_home.join("settings.json"), r#"{"model":"claude","env":{"FOO":"bar"}}"#, ) .expect("write settings"); - let items = service_for_paths(claude_home.clone(), codex_home.clone()) + let items = service_for_paths(external_agent_home.clone(), codex_home.clone()) .detect(ExternalAgentConfigDetectOptions { include_home: true, cwds: None, }) + .await .expect("detect"); let expected = vec![ @@ -50,7 +55,7 @@ fn detect_home_lists_config_skills_and_agents_md() { item_type: ExternalAgentConfigMigrationItemType::Config, description: format!( "Migrate {} into {}", - claude_home.join("settings.json").display(), + external_agent_home.join("settings.json").display(), codex_home.join("config.toml").display() ), cwd: None, @@ -59,8 +64,8 @@ fn detect_home_lists_config_skills_and_agents_md() { ExternalAgentConfigMigrationItem { item_type: ExternalAgentConfigMigrationItemType::Skills, description: format!( - "Copy skill folders from {} to {}", - claude_home.join("skills").display(), + "Migrate skills from {} to {}", + external_agent_home.join("skills").display(), agents_skills.display() ), cwd: None, @@ -70,7 +75,7 @@ fn detect_home_lists_config_skills_and_agents_md() { item_type: ExternalAgentConfigMigrationItemType::AgentsMd, description: format!( "Import {} to {}", - claude_home.join("CLAUDE.md").display(), + external_agent_home.join("CLAUDE.md").display(), codex_home.join("AGENTS.md").display() ), cwd: None, @@ -81,8 +86,8 @@ fn detect_home_lists_config_skills_and_agents_md() { assert_eq!(items, expected); } -#[test] -fn detect_repo_lists_agents_md_for_each_cwd() { +#[tokio::test] +async fn detect_repo_lists_agents_md_for_each_cwd() { let root = TempDir::new().expect("create tempdir"); let repo_root = root.path().join("repo"); let nested = repo_root.join("nested").join("child"); @@ -95,6 +100,7 @@ fn detect_repo_lists_agents_md_for_each_cwd() { include_home: false, cwds: Some(vec![nested, repo_root.clone()]), }) + .await .expect("detect"); let expected = vec![ @@ -123,27 +129,110 @@ fn detect_repo_lists_agents_md_for_each_cwd() { assert_eq!(items, expected); } +#[tokio::test] +async fn detect_repo_still_reports_non_plugin_items_when_home_config_is_invalid() { + let root = TempDir::new().expect("create tempdir"); + let repo_root = root.path().join("repo"); + let codex_home = root.path().join(".codex"); + fs::create_dir_all(repo_root.join(".git")).expect("create git dir"); + fs::create_dir_all(repo_root.join(".claude").join("skills").join("skill-a")) + .expect("create repo skills"); + fs::create_dir_all(&codex_home).expect("create codex home"); + fs::write(codex_home.join("config.toml"), "this is not valid = [toml") + .expect("write invalid codex config"); + fs::write( + repo_root.join(".claude").join("settings.json"), + r#"{"env":{"FOO":"bar"}}"#, + ) + .expect("write settings"); + fs::write( + repo_root + .join(".claude") + .join("skills") + .join("skill-a") + .join("SKILL.md"), + "Use Claude Code and CLAUDE utilities.", + ) + .expect("write skill"); + fs::write( + repo_root.join(".claude").join("CLAUDE.md"), + "Claude code guidance", + ) + .expect("write agents"); + + let items = service_for_paths(root.path().join(".claude"), codex_home) + .detect(ExternalAgentConfigDetectOptions { + include_home: false, + cwds: Some(vec![repo_root.clone()]), + }) + .await + .expect("detect"); + + assert_eq!( + items, + vec![ + ExternalAgentConfigMigrationItem { + item_type: ExternalAgentConfigMigrationItemType::Config, + description: format!( + "Migrate {} into {}", + repo_root.join(".claude").join("settings.json").display(), + repo_root.join(".codex").join("config.toml").display() + ), + cwd: Some(repo_root.clone()), + details: None, + }, + ExternalAgentConfigMigrationItem { + item_type: ExternalAgentConfigMigrationItemType::Skills, + description: format!( + "Migrate skills from {} to {}", + repo_root.join(".claude").join("skills").display(), + repo_root.join(".agents").join("skills").display() + ), + cwd: Some(repo_root.clone()), + details: None, + }, + ExternalAgentConfigMigrationItem { + item_type: ExternalAgentConfigMigrationItemType::AgentsMd, + description: format!( + "Import {} to {}", + repo_root.join(".claude").join("CLAUDE.md").display(), + repo_root.join("AGENTS.md").display(), + ), + cwd: Some(repo_root), + details: None, + }, + ] + ); +} + #[tokio::test] async fn import_home_migrates_supported_config_fields_skills_and_agents_md() { - let (_root, claude_home, codex_home) = fixture_paths(); + let (_root, external_agent_home, codex_home) = fixture_paths(); let agents_skills = codex_home .parent() .map(|parent| parent.join(".agents").join("skills")) .unwrap_or_else(|| PathBuf::from(".agents").join("skills")); - fs::create_dir_all(claude_home.join("skills").join("skill-a")).expect("create skills"); + fs::create_dir_all(external_agent_home.join("skills").join("skill-a")).expect("create skills"); fs::write( - claude_home.join("settings.json"), + external_agent_home.join("settings.json"), r#"{"model":"claude","permissions":{"ask":["git push"]},"env":{"FOO":"bar","CI":false,"MAX_RETRIES":3,"MY_TEAM":"codex","IGNORED":null,"LIST":["a","b"],"MAP":{"x":1}},"sandbox":{"enabled":true,"network":{"allowLocalBinding":true}}}"#, ) .expect("write settings"); fs::write( - claude_home.join("skills").join("skill-a").join("SKILL.md"), + external_agent_home + .join("skills") + .join("skill-a") + .join("SKILL.md"), "Use Claude Code and CLAUDE utilities.", ) .expect("write skill"); - fs::write(claude_home.join("CLAUDE.md"), "Claude code guidance").expect("write agents"); + fs::write( + external_agent_home.join("CLAUDE.md"), + "Claude code guidance", + ) + .expect("write agents"); - service_for_paths(claude_home, codex_home.clone()) + service_for_paths(external_agent_home, codex_home.clone()) .import(vec![ ExternalAgentConfigMigrationItem { item_type: ExternalAgentConfigMigrationItemType::AgentsMd, @@ -185,15 +274,15 @@ async fn import_home_migrates_supported_config_fields_skills_and_agents_md() { #[tokio::test] async fn import_home_skips_empty_config_migration() { - let (_root, claude_home, codex_home) = fixture_paths(); - fs::create_dir_all(&claude_home).expect("create claude home"); + let (_root, external_agent_home, codex_home) = fixture_paths(); + fs::create_dir_all(&external_agent_home).expect("create external agent home"); fs::write( - claude_home.join("settings.json"), + external_agent_home.join("settings.json"), r#"{"model":"claude","sandbox":{"enabled":false}}"#, ) .expect("write settings"); - service_for_paths(claude_home, codex_home.clone()) + service_for_paths(external_agent_home, codex_home.clone()) .import(vec![ExternalAgentConfigMigrationItem { item_type: ExternalAgentConfigMigrationItemType::Config, description: String::new(), @@ -206,13 +295,13 @@ async fn import_home_skips_empty_config_migration() { assert!(!codex_home.join("config.toml").exists()); } -#[test] -fn detect_home_skips_config_when_target_already_has_supported_fields() { - let (_root, claude_home, codex_home) = fixture_paths(); - fs::create_dir_all(&claude_home).expect("create claude home"); +#[tokio::test] +async fn detect_home_skips_config_when_target_already_has_supported_fields() { + let (_root, external_agent_home, codex_home) = fixture_paths(); + fs::create_dir_all(&external_agent_home).expect("create external agent home"); fs::create_dir_all(&codex_home).expect("create codex home"); fs::write( - claude_home.join("settings.json"), + external_agent_home.join("settings.json"), r#"{"env":{"FOO":"bar"},"sandbox":{"enabled":true}}"#, ) .expect("write settings"); @@ -230,31 +319,33 @@ fn detect_home_skips_config_when_target_already_has_supported_fields() { ) .expect("write config"); - let items = service_for_paths(claude_home, codex_home) + let items = service_for_paths(external_agent_home, codex_home) .detect(ExternalAgentConfigDetectOptions { include_home: true, cwds: None, }) + .await .expect("detect"); assert_eq!(items, Vec::::new()); } -#[test] -fn detect_home_skips_skills_when_all_skill_directories_exist() { - let (_root, claude_home, codex_home) = fixture_paths(); +#[tokio::test] +async fn detect_home_skips_skills_when_all_skill_directories_exist() { + let (_root, external_agent_home, codex_home) = fixture_paths(); let agents_skills = codex_home .parent() .map(|parent| parent.join(".agents").join("skills")) .unwrap_or_else(|| PathBuf::from(".agents").join("skills")); - fs::create_dir_all(claude_home.join("skills").join("skill-a")).expect("create source"); + fs::create_dir_all(external_agent_home.join("skills").join("skill-a")).expect("create source"); fs::create_dir_all(agents_skills.join("skill-a")).expect("create target"); - let items = service_for_paths(claude_home, codex_home) + let items = service_for_paths(external_agent_home, codex_home) .detect(ExternalAgentConfigDetectOptions { include_home: true, cwds: None, }) + .await .expect("detect"); assert_eq!(items, Vec::::new()); @@ -332,8 +423,8 @@ async fn import_repo_agents_md_overwrites_empty_targets() { ); } -#[test] -fn detect_repo_prefers_non_empty_dot_claude_agents_source() { +#[tokio::test] +async fn detect_repo_prefers_non_empty_external_agent_agents_source() { let root = TempDir::new().expect("create tempdir"); let repo_root = root.path().join("repo"); fs::create_dir_all(repo_root.join(".git")).expect("create git"); @@ -350,6 +441,7 @@ fn detect_repo_prefers_non_empty_dot_claude_agents_source() { include_home: false, cwds: Some(vec![repo_root.clone()]), }) + .await .expect("detect"); assert_eq!( @@ -368,7 +460,7 @@ fn detect_repo_prefers_non_empty_dot_claude_agents_source() { } #[tokio::test] -async fn import_repo_uses_non_empty_dot_claude_agents_source() { +async fn import_repo_uses_non_empty_external_agent_agents_source() { let root = TempDir::new().expect("create tempdir"); let repo_root = root.path().join("repo"); fs::create_dir_all(repo_root.join(".git")).expect("create git"); @@ -407,27 +499,33 @@ fn migration_metric_tags_for_skills_include_skills_count() { ); } -#[test] -fn detect_home_lists_enabled_plugins_from_settings() { - let (_root, claude_home, codex_home) = fixture_paths(); - fs::create_dir_all(&claude_home).expect("create claude home"); +#[tokio::test] +async fn detect_home_lists_enabled_plugins_from_settings() { + let (_root, external_agent_home, codex_home) = fixture_paths(); + fs::create_dir_all(&external_agent_home).expect("create external agent home"); fs::write( - claude_home.join("settings.json"), + external_agent_home.join("settings.json"), r#"{ "enabledPlugins": { "formatter@acme-tools": true, "deployer@acme-tools": true, "analyzer@security-plugins": false + }, + "extraKnownMarketplaces": { + "acme-tools": { + "source": "acme-corp/claude-plugins" + } } }"#, ) .expect("write settings"); - let items = service_for_paths(claude_home.clone(), codex_home) + let items = service_for_paths(external_agent_home.clone(), codex_home) .detect(ExternalAgentConfigDetectOptions { include_home: true, cwds: None, }) + .await .expect("detect"); assert_eq!( @@ -436,7 +534,7 @@ fn detect_home_lists_enabled_plugins_from_settings() { item_type: ExternalAgentConfigMigrationItemType::Plugins, description: format!( "Import enabled plugins from {}", - claude_home.join("settings.json").display() + external_agent_home.join("settings.json").display() ), cwd: None, details: Some(MigrationDetails { @@ -450,10 +548,155 @@ fn detect_home_lists_enabled_plugins_from_settings() { } #[tokio::test] -async fn import_plugins_requires_details() { - let (_root, claude_home, codex_home) = fixture_paths(); +async fn detect_repo_skips_plugins_that_are_already_configured_in_codex() { + let root = TempDir::new().expect("create tempdir"); + let external_agent_home = root.path().join(".claude"); + let codex_home = root.path().join(".codex"); + let repo_root = root.path().join("repo"); + fs::create_dir_all(repo_root.join(".git")).expect("create git dir"); + fs::create_dir_all(repo_root.join(".claude")).expect("create repo external agent dir"); + fs::create_dir_all(&codex_home).expect("create codex home"); + fs::write( + repo_root.join(".claude").join("settings.json"), + r#"{ + "enabledPlugins": { + "formatter@acme-tools": true, + "deployer@acme-tools": true + }, + "extraKnownMarketplaces": { + "acme-tools": { + "source": "acme-corp/claude-plugins" + } + } + }"#, + ) + .expect("write repo settings"); + fs::write( + codex_home.join("config.toml"), + r#" +[plugins."formatter@acme-tools"] +enabled = true +"#, + ) + .expect("write codex config"); - let err = service_for_paths(claude_home, codex_home) + let items = service_for_paths(external_agent_home, codex_home) + .detect(ExternalAgentConfigDetectOptions { + include_home: false, + cwds: Some(vec![repo_root.clone()]), + }) + .await + .expect("detect"); + + assert_eq!( + items, + vec![ExternalAgentConfigMigrationItem { + item_type: ExternalAgentConfigMigrationItemType::Plugins, + description: format!( + "Import enabled plugins from {}", + repo_root.join(".claude").join("settings.json").display() + ), + cwd: Some(repo_root), + details: Some(MigrationDetails { + plugins: vec![PluginsMigration { + marketplace_name: "acme-tools".to_string(), + plugin_names: vec!["deployer".to_string()], + }], + }), + }] + ); +} + +#[tokio::test] +async fn detect_repo_skips_plugins_that_are_disabled_in_codex() { + let root = TempDir::new().expect("create tempdir"); + let external_agent_home = root.path().join(".claude"); + let codex_home = root.path().join(".codex"); + let repo_root = root.path().join("repo"); + fs::create_dir_all(repo_root.join(".git")).expect("create git dir"); + fs::create_dir_all(repo_root.join(".claude")).expect("create repo external agent dir"); + fs::create_dir_all(&codex_home).expect("create codex home"); + fs::write( + repo_root.join(".claude").join("settings.json"), + r#"{ + "enabledPlugins": { + "formatter@acme-tools": true + }, + "extraKnownMarketplaces": { + "acme-tools": { + "source": "acme-corp/claude-plugins" + } + } + }"#, + ) + .expect("write repo settings"); + fs::write( + codex_home.join("config.toml"), + r#" +[plugins."formatter@acme-tools"] +enabled = false +"#, + ) + .expect("write codex config"); + + let items = service_for_paths(external_agent_home, codex_home) + .detect(ExternalAgentConfigDetectOptions { + include_home: false, + cwds: Some(vec![repo_root]), + }) + .await + .expect("detect"); + + assert_eq!(items, Vec::::new()); +} + +#[tokio::test] +async fn detect_repo_skips_plugins_without_explicit_enabled_in_codex() { + let root = TempDir::new().expect("create tempdir"); + let external_agent_home = root.path().join(".claude"); + let codex_home = root.path().join(".codex"); + let repo_root = root.path().join("repo"); + fs::create_dir_all(repo_root.join(".git")).expect("create git dir"); + fs::create_dir_all(repo_root.join(".claude")).expect("create repo external agent dir"); + fs::create_dir_all(&codex_home).expect("create codex home"); + fs::write( + repo_root.join(".claude").join("settings.json"), + r#"{ + "enabledPlugins": { + "formatter@acme-tools": true + }, + "extraKnownMarketplaces": { + "acme-tools": { + "source": "acme-corp/claude-plugins" + } + } + }"#, + ) + .expect("write repo settings"); + fs::write( + codex_home.join("config.toml"), + r#" +[plugins."formatter@acme-tools"] +"#, + ) + .expect("write codex config"); + + let items = service_for_paths(external_agent_home, codex_home) + .detect(ExternalAgentConfigDetectOptions { + include_home: false, + cwds: Some(vec![repo_root]), + }) + .await + .expect("detect"); + + assert_eq!(items, Vec::::new()); +} + +#[tokio::test] +async fn import_plugins_requires_details() { + let (_root, external_agent_home, codex_home) = fixture_paths(); + + let err = service_for_paths(external_agent_home, codex_home) .import_plugins(/*cwd*/ None, /*details*/ None) .await .expect_err("expected missing details error"); @@ -463,11 +706,251 @@ async fn import_plugins_requires_details() { } #[tokio::test] -async fn import_plugins_requires_source_marketplace_details() { - let (_root, claude_home, codex_home) = fixture_paths(); - fs::create_dir_all(&claude_home).expect("create claude home"); +async fn detect_repo_does_not_skip_plugins_only_configured_in_project_codex() { + let root = TempDir::new().expect("create tempdir"); + let external_agent_home = root.path().join(".claude"); + let codex_home = root.path().join(".codex"); + let repo_root = root.path().join("repo"); + fs::create_dir_all(repo_root.join(".git")).expect("create git dir"); + fs::create_dir_all(repo_root.join(".claude")).expect("create repo external agent dir"); + fs::create_dir_all(repo_root.join(".codex")).expect("create repo codex dir"); + fs::create_dir_all(&codex_home).expect("create codex home"); fs::write( - claude_home.join("settings.json"), + repo_root.join(".claude").join("settings.json"), + r#"{ + "enabledPlugins": { + "formatter@acme-tools": true + }, + "extraKnownMarketplaces": { + "acme-tools": { + "source": "acme-corp/claude-plugins" + } + } + }"#, + ) + .expect("write repo settings"); + fs::write( + repo_root.join(".codex").join("config.toml"), + r#" +[plugins."formatter@acme-tools"] +enabled = true +"#, + ) + .expect("write project codex config"); + + let items = service_for_paths(external_agent_home, codex_home) + .detect(ExternalAgentConfigDetectOptions { + include_home: false, + cwds: Some(vec![repo_root.clone()]), + }) + .await + .expect("detect"); + + assert_eq!( + items, + vec![ExternalAgentConfigMigrationItem { + item_type: ExternalAgentConfigMigrationItemType::Plugins, + description: format!( + "Import enabled plugins from {}", + repo_root.join(".claude").join("settings.json").display() + ), + cwd: Some(repo_root), + details: Some(MigrationDetails { + plugins: vec![PluginsMigration { + marketplace_name: "acme-tools".to_string(), + plugin_names: vec!["formatter".to_string()], + }], + }), + }] + ); +} + +#[tokio::test] +async fn detect_home_skips_plugins_without_marketplace_source() { + let (_root, external_agent_home, codex_home) = fixture_paths(); + fs::create_dir_all(&external_agent_home).expect("create external agent home"); + fs::write( + external_agent_home.join("settings.json"), + r#"{ + "enabledPlugins": { + "formatter@acme-tools": true + } + }"#, + ) + .expect("write settings"); + + let items = service_for_paths(external_agent_home, codex_home) + .detect(ExternalAgentConfigDetectOptions { + include_home: true, + cwds: None, + }) + .await + .expect("detect"); + + assert_eq!(items, Vec::::new()); +} + +#[tokio::test] +async fn detect_home_skips_plugins_with_invalid_marketplace_source() { + let (_root, external_agent_home, codex_home) = fixture_paths(); + fs::create_dir_all(&external_agent_home).expect("create external agent home"); + fs::write( + external_agent_home.join("settings.json"), + r#"{ + "enabledPlugins": { + "formatter@acme-tools": true + }, + "extraKnownMarketplaces": { + "acme-tools": { + "source": "github" + } + } + }"#, + ) + .expect("write settings"); + + let items = service_for_paths(external_agent_home, codex_home) + .detect(ExternalAgentConfigDetectOptions { + include_home: true, + cwds: None, + }) + .await + .expect("detect"); + + assert_eq!(items, Vec::::new()); +} + +#[tokio::test] +async fn detect_repo_filters_plugins_against_installed_marketplace() { + let root = TempDir::new().expect("create tempdir"); + let external_agent_home = root.path().join(".claude"); + let codex_home = root.path().join(".codex"); + let repo_root = root.path().join("repo"); + let marketplace_root = codex_home.join(".tmp").join("marketplaces").join("debug"); + fs::create_dir_all(repo_root.join(".git")).expect("create git dir"); + fs::create_dir_all(repo_root.join(".claude")).expect("create repo external agent dir"); + fs::create_dir_all(marketplace_root.join(".agents").join("plugins")) + .expect("create marketplace manifest dir"); + fs::create_dir_all( + marketplace_root + .join("plugins") + .join("sample") + .join(".codex-plugin"), + ) + .expect("create sample plugin"); + fs::create_dir_all( + marketplace_root + .join("plugins") + .join("available") + .join(".codex-plugin"), + ) + .expect("create available plugin"); + fs::write( + repo_root.join(".claude").join("settings.json"), + r#"{ + "enabledPlugins": { + "sample@debug": true, + "available@debug": true, + "missing@debug": true + }, + "extraKnownMarketplaces": { + "debug": { + "source": "owner/debug-marketplace" + } + } + }"#, + ) + .expect("write repo settings"); + fs::write( + codex_home.join("config.toml"), + r#" +[marketplaces.debug] +source_type = "git" +source = "owner/debug-marketplace" +"#, + ) + .expect("write codex config"); + fs::write( + marketplace_root + .join(".agents") + .join("plugins") + .join("marketplace.json"), + r#"{ + "name": "debug", + "plugins": [ + { + "name": "sample", + "source": { + "source": "local", + "path": "./plugins/sample" + }, + "policy": { + "installation": "NOT_AVAILABLE" + } + }, + { + "name": "available", + "source": { + "source": "local", + "path": "./plugins/available" + } + } + ] +}"#, + ) + .expect("write marketplace manifest"); + fs::write( + marketplace_root + .join("plugins") + .join("sample") + .join(".codex-plugin") + .join("plugin.json"), + r#"{"name":"sample"}"#, + ) + .expect("write sample plugin manifest"); + fs::write( + marketplace_root + .join("plugins") + .join("available") + .join(".codex-plugin") + .join("plugin.json"), + r#"{"name":"available"}"#, + ) + .expect("write available plugin manifest"); + + let items = service_for_paths(external_agent_home, codex_home) + .detect(ExternalAgentConfigDetectOptions { + include_home: false, + cwds: Some(vec![repo_root.clone()]), + }) + .await + .expect("detect"); + + assert_eq!( + items, + vec![ExternalAgentConfigMigrationItem { + item_type: ExternalAgentConfigMigrationItemType::Plugins, + description: format!( + "Import enabled plugins from {}", + repo_root.join(".claude").join("settings.json").display() + ), + cwd: Some(repo_root), + details: Some(MigrationDetails { + plugins: vec![PluginsMigration { + marketplace_name: "debug".to_string(), + plugin_names: vec!["available".to_string()], + }], + }), + }] + ); +} + +#[tokio::test] +async fn import_plugins_requires_source_marketplace_details() { + let (_root, external_agent_home, codex_home) = fixture_paths(); + fs::create_dir_all(&external_agent_home).expect("create external agent home"); + fs::write( + external_agent_home.join("settings.json"), r#"{ "enabledPlugins": { "formatter@acme-tools": true @@ -482,7 +965,7 @@ async fn import_plugins_requires_source_marketplace_details() { ) .expect("write settings"); - let outcome = service_for_paths(claude_home, codex_home) + let outcome = service_for_paths(external_agent_home, codex_home) .import_plugins( /*cwd*/ None, Some(MigrationDetails { @@ -508,10 +991,10 @@ async fn import_plugins_requires_source_marketplace_details() { #[tokio::test] async fn import_plugins_defers_marketplace_source_validation_to_add_marketplace() { - let (_root, claude_home, codex_home) = fixture_paths(); - fs::create_dir_all(&claude_home).expect("create claude home"); + let (_root, external_agent_home, codex_home) = fixture_paths(); + fs::create_dir_all(&external_agent_home).expect("create external agent home"); fs::write( - claude_home.join("settings.json"), + external_agent_home.join("settings.json"), r#"{ "enabledPlugins": { "formatter@acme-tools": true @@ -526,7 +1009,7 @@ async fn import_plugins_defers_marketplace_source_validation_to_add_marketplace( ) .expect("write settings"); - let outcome = service_for_paths(claude_home, codex_home) + let outcome = service_for_paths(external_agent_home, codex_home) .import_plugins(/*cwd*/ None, Some(github_plugin_details())) .await .expect("import plugins"); @@ -542,18 +1025,396 @@ async fn import_plugins_defers_marketplace_source_validation_to_add_marketplace( ); } +#[tokio::test] +async fn import_plugins_supports_external_agent_plugin_marketplace_layout() { + let (_root, external_agent_home, codex_home) = fixture_paths(); + let marketplace_root = external_agent_home.join("my-marketplace"); + let plugin_root = marketplace_root.join("plugins").join("cloudflare"); + fs::create_dir_all(marketplace_root.join(".claude-plugin")) + .expect("create marketplace manifest dir"); + fs::create_dir_all(plugin_root.join(".codex-plugin")).expect("create plugin manifest dir"); + fs::create_dir_all(&codex_home).expect("create codex home"); + + fs::write( + external_agent_home.join("settings.json"), + serde_json::to_string_pretty(&serde_json::json!({ + "enabledPlugins": { + "cloudflare@my-plugins": true + }, + "extraKnownMarketplaces": { + "my-plugins": { + "source": "local", + "path": marketplace_root + } + } + })) + .expect("serialize settings"), + ) + .expect("write settings"); + fs::write( + marketplace_root + .join(".claude-plugin") + .join("marketplace.json"), + r#"{ + "name": "my-plugins", + "plugins": [ + { + "name": "cloudflare", + "source": "./plugins/cloudflare" + } + ] + }"#, + ) + .expect("write marketplace manifest"); + fs::write( + plugin_root.join(".codex-plugin").join("plugin.json"), + r#"{"name":"cloudflare","version":"0.1.0"}"#, + ) + .expect("write plugin manifest"); + + let outcome = service_for_paths(external_agent_home, codex_home.clone()) + .import_plugins( + /*cwd*/ None, + Some(MigrationDetails { + plugins: vec![PluginsMigration { + marketplace_name: "my-plugins".to_string(), + plugin_names: vec!["cloudflare".to_string()], + }], + }), + ) + .await + .expect("import plugins"); + + assert_eq!( + outcome, + PluginImportOutcome { + succeeded_marketplaces: vec!["my-plugins".to_string()], + succeeded_plugin_ids: vec!["cloudflare@my-plugins".to_string()], + failed_marketplaces: Vec::new(), + failed_plugin_ids: Vec::new(), + } + ); + let config = fs::read_to_string(codex_home.join("config.toml")).expect("read config"); + assert!(config.contains(r#"[plugins."cloudflare@my-plugins"]"#)); + assert!(config.contains("enabled = true")); +} + +#[tokio::test] +async fn detect_home_supports_relative_external_agent_plugin_marketplace_path() { + let (_root, external_agent_home, codex_home) = fixture_paths(); + let marketplace_root = external_agent_home.join("my-marketplace"); + let plugin_root = marketplace_root.join("plugins").join("cloudflare"); + fs::create_dir_all(marketplace_root.join(".claude-plugin")) + .expect("create marketplace manifest dir"); + fs::create_dir_all(plugin_root.join(".codex-plugin")).expect("create plugin manifest dir"); + fs::create_dir_all(&codex_home).expect("create codex home"); + + fs::write( + external_agent_home.join("settings.json"), + r#"{ + "enabledPlugins": { + "cloudflare@my-plugins": true + }, + "extraKnownMarketplaces": { + "my-plugins": { + "source": "directory", + "path": "./my-marketplace" + } + } + }"#, + ) + .expect("write settings"); + fs::write( + marketplace_root + .join(".claude-plugin") + .join("marketplace.json"), + r#"{ + "name": "my-plugins", + "plugins": [ + { + "name": "cloudflare", + "source": "./plugins/cloudflare" + } + ] + }"#, + ) + .expect("write marketplace manifest"); + fs::write( + plugin_root.join(".codex-plugin").join("plugin.json"), + r#"{"name":"cloudflare","version":"0.1.0"}"#, + ) + .expect("write plugin manifest"); + + let items = service_for_paths(external_agent_home.clone(), codex_home) + .detect(ExternalAgentConfigDetectOptions { + include_home: true, + cwds: None, + }) + .await + .expect("detect"); + + assert_eq!( + items, + vec![ExternalAgentConfigMigrationItem { + item_type: ExternalAgentConfigMigrationItemType::Plugins, + description: format!( + "Import enabled plugins from {}", + external_agent_home.join("settings.json").display() + ), + cwd: None, + details: Some(MigrationDetails { + plugins: vec![PluginsMigration { + marketplace_name: "my-plugins".to_string(), + plugin_names: vec!["cloudflare".to_string()], + }], + }), + }] + ); +} + +#[tokio::test] +async fn import_plugins_supports_relative_external_agent_plugin_marketplace_path() { + let (_root, external_agent_home, codex_home) = fixture_paths(); + let marketplace_root = external_agent_home.join("my-marketplace"); + let plugin_root = marketplace_root.join("plugins").join("cloudflare"); + fs::create_dir_all(marketplace_root.join(".claude-plugin")) + .expect("create marketplace manifest dir"); + fs::create_dir_all(plugin_root.join(".codex-plugin")).expect("create plugin manifest dir"); + fs::create_dir_all(&codex_home).expect("create codex home"); + + fs::write( + external_agent_home.join("settings.json"), + r#"{ + "enabledPlugins": { + "cloudflare@my-plugins": true + }, + "extraKnownMarketplaces": { + "my-plugins": { + "source": "directory", + "path": "./my-marketplace" + } + } + }"#, + ) + .expect("write settings"); + fs::write( + marketplace_root + .join(".claude-plugin") + .join("marketplace.json"), + r#"{ + "name": "my-plugins", + "plugins": [ + { + "name": "cloudflare", + "source": "./plugins/cloudflare" + } + ] + }"#, + ) + .expect("write marketplace manifest"); + fs::write( + plugin_root.join(".codex-plugin").join("plugin.json"), + r#"{"name":"cloudflare","version":"0.1.0"}"#, + ) + .expect("write plugin manifest"); + + let outcome = service_for_paths(external_agent_home, codex_home.clone()) + .import_plugins( + /*cwd*/ None, + Some(MigrationDetails { + plugins: vec![PluginsMigration { + marketplace_name: "my-plugins".to_string(), + plugin_names: vec!["cloudflare".to_string()], + }], + }), + ) + .await + .expect("import plugins"); + + assert_eq!( + outcome, + PluginImportOutcome { + succeeded_marketplaces: vec!["my-plugins".to_string()], + succeeded_plugin_ids: vec!["cloudflare@my-plugins".to_string()], + failed_marketplaces: Vec::new(), + failed_plugin_ids: Vec::new(), + } + ); + let config = fs::read_to_string(codex_home.join("config.toml")).expect("read config"); + assert!(config.contains(r#"[plugins."cloudflare@my-plugins"]"#)); + assert!(config.contains("enabled = true")); +} + +#[tokio::test] +async fn detect_repo_supports_project_relative_external_agent_plugin_marketplace_path() { + let root = TempDir::new().expect("create tempdir"); + let external_agent_home = root.path().join(".claude"); + let codex_home = root.path().join(".codex"); + let repo_root = root.path().join("repo"); + let marketplace_root = repo_root.join("my-marketplace"); + let plugin_root = marketplace_root.join("plugins").join("cloudflare"); + fs::create_dir_all(repo_root.join(".git")).expect("create git dir"); + fs::create_dir_all(repo_root.join(".claude")).expect("create repo external agent dir"); + fs::create_dir_all(marketplace_root.join(".claude-plugin")) + .expect("create marketplace manifest dir"); + fs::create_dir_all(plugin_root.join(".codex-plugin")).expect("create plugin manifest dir"); + fs::create_dir_all(&codex_home).expect("create codex home"); + + fs::write( + repo_root.join(".claude").join("settings.json"), + r#"{ + "enabledPlugins": { + "cloudflare@my-plugins": true + }, + "extraKnownMarketplaces": { + "my-plugins": { + "source": "directory", + "path": "./my-marketplace" + } + } + }"#, + ) + .expect("write settings"); + fs::write( + marketplace_root + .join(".claude-plugin") + .join("marketplace.json"), + r#"{ + "name": "my-plugins", + "plugins": [ + { + "name": "cloudflare", + "source": "./plugins/cloudflare" + } + ] + }"#, + ) + .expect("write marketplace manifest"); + fs::write( + plugin_root.join(".codex-plugin").join("plugin.json"), + r#"{"name":"cloudflare","version":"0.1.0"}"#, + ) + .expect("write plugin manifest"); + + let items = service_for_paths(external_agent_home, codex_home) + .detect(ExternalAgentConfigDetectOptions { + include_home: false, + cwds: Some(vec![repo_root.clone()]), + }) + .await + .expect("detect"); + + assert_eq!( + items, + vec![ExternalAgentConfigMigrationItem { + item_type: ExternalAgentConfigMigrationItemType::Plugins, + description: format!( + "Import enabled plugins from {}", + repo_root.join(".claude").join("settings.json").display() + ), + cwd: Some(repo_root), + details: Some(MigrationDetails { + plugins: vec![PluginsMigration { + marketplace_name: "my-plugins".to_string(), + plugin_names: vec!["cloudflare".to_string()], + }], + }), + }] + ); +} + +#[tokio::test] +async fn import_plugins_supports_project_relative_external_agent_plugin_marketplace_path() { + let root = TempDir::new().expect("create tempdir"); + let external_agent_home = root.path().join(".claude"); + let codex_home = root.path().join(".codex"); + let repo_root = root.path().join("repo"); + let marketplace_root = repo_root.join("my-marketplace"); + let plugin_root = marketplace_root.join("plugins").join("cloudflare"); + fs::create_dir_all(repo_root.join(".git")).expect("create git dir"); + fs::create_dir_all(repo_root.join(".claude")).expect("create repo external agent dir"); + fs::create_dir_all(marketplace_root.join(".claude-plugin")) + .expect("create marketplace manifest dir"); + fs::create_dir_all(plugin_root.join(".codex-plugin")).expect("create plugin manifest dir"); + fs::create_dir_all(&codex_home).expect("create codex home"); + + fs::write( + repo_root.join(".claude").join("settings.json"), + r#"{ + "enabledPlugins": { + "cloudflare@my-plugins": true + }, + "extraKnownMarketplaces": { + "my-plugins": { + "source": "directory", + "path": "./my-marketplace" + } + } + }"#, + ) + .expect("write settings"); + fs::write( + marketplace_root + .join(".claude-plugin") + .join("marketplace.json"), + r#"{ + "name": "my-plugins", + "plugins": [ + { + "name": "cloudflare", + "source": "./plugins/cloudflare" + } + ] + }"#, + ) + .expect("write marketplace manifest"); + fs::write( + plugin_root.join(".codex-plugin").join("plugin.json"), + r#"{"name":"cloudflare","version":"0.1.0"}"#, + ) + .expect("write plugin manifest"); + + let outcome = service_for_paths(external_agent_home, codex_home.clone()) + .import_plugins( + Some(repo_root.as_path()), + Some(MigrationDetails { + plugins: vec![PluginsMigration { + marketplace_name: "my-plugins".to_string(), + plugin_names: vec!["cloudflare".to_string()], + }], + }), + ) + .await + .expect("import plugins"); + + assert_eq!( + outcome, + PluginImportOutcome { + succeeded_marketplaces: vec!["my-plugins".to_string()], + succeeded_plugin_ids: vec!["cloudflare@my-plugins".to_string()], + failed_marketplaces: Vec::new(), + failed_plugin_ids: Vec::new(), + } + ); + let config = fs::read_to_string(codex_home.join("config.toml")).expect("read config"); + assert!(config.contains(r#"[plugins."cloudflare@my-plugins"]"#)); + assert!(config.contains("enabled = true")); +} + #[test] fn import_skills_returns_only_new_skill_directory_count() { - let (_root, claude_home, codex_home) = fixture_paths(); + let (_root, external_agent_home, codex_home) = fixture_paths(); let agents_skills = codex_home .parent() .map(|parent| parent.join(".agents").join("skills")) .unwrap_or_else(|| PathBuf::from(".agents").join("skills")); - fs::create_dir_all(claude_home.join("skills").join("skill-a")).expect("create source a"); - fs::create_dir_all(claude_home.join("skills").join("skill-b")).expect("create source b"); + fs::create_dir_all(external_agent_home.join("skills").join("skill-a")) + .expect("create source a"); + fs::create_dir_all(external_agent_home.join("skills").join("skill-b")) + .expect("create source b"); fs::create_dir_all(agents_skills.join("skill-a")).expect("create existing target"); - let copied_count = service_for_paths(claude_home, codex_home) + let copied_count = service_for_paths(external_agent_home, codex_home) .import_skills(/*cwd*/ None) .expect("import skills"); diff --git a/codex-rs/core/src/plugins/manager.rs b/codex-rs/core/src/plugins/manager.rs index 50c62d5823..3449ecffdf 100644 --- a/codex-rs/core/src/plugins/manager.rs +++ b/codex-rs/core/src/plugins/manager.rs @@ -1346,7 +1346,7 @@ impl PluginUninstallError { } } -fn configured_plugins_from_stack( +pub(crate) fn configured_plugins_from_stack( config_layer_stack: &ConfigLayerStack, ) -> HashMap { // Plugin entries remain persisted user config only. diff --git a/codex-rs/core/src/plugins/marketplace_add.rs b/codex-rs/core/src/plugins/marketplace_add.rs index b8d1b34d41..38fc04c199 100644 --- a/codex-rs/core/src/plugins/marketplace_add.rs +++ b/codex-rs/core/src/plugins/marketplace_add.rs @@ -20,7 +20,7 @@ use metadata::find_marketplace_root_by_name; use metadata::installed_marketplace_root_for_source; use metadata::record_added_marketplace_entry; use source::MarketplaceSource; -use source::parse_marketplace_source; +pub(crate) use source::parse_marketplace_source; use source::stage_marketplace_source; use source::validate_marketplace_source_root; diff --git a/codex-rs/core/src/plugins/marketplace_add/source.rs b/codex-rs/core/src/plugins/marketplace_add/source.rs index 9a0b666c60..98bdc2b8bb 100644 --- a/codex-rs/core/src/plugins/marketplace_add/source.rs +++ b/codex-rs/core/src/plugins/marketplace_add/source.rs @@ -5,7 +5,7 @@ use std::path::Path; use std::path::PathBuf; #[derive(Debug, Clone, PartialEq, Eq)] -pub(super) enum MarketplaceSource { +pub(crate) enum MarketplaceSource { Git { url: String, ref_name: Option, @@ -15,7 +15,7 @@ pub(super) enum MarketplaceSource { }, } -pub(super) fn parse_marketplace_source( +pub(crate) fn parse_marketplace_source( source: &str, explicit_ref: Option, ) -> Result { diff --git a/codex-rs/core/src/plugins/mod.rs b/codex-rs/core/src/plugins/mod.rs index 57997eb134..2e86643f34 100644 --- a/codex-rs/core/src/plugins/mod.rs +++ b/codex-rs/core/src/plugins/mod.rs @@ -22,6 +22,7 @@ pub use codex_plugin::validate_plugin_segment; pub type LoadedPlugin = codex_plugin::LoadedPlugin; pub type PluginLoadOutcome = codex_plugin::PluginLoadOutcome; +pub(crate) use codex_core_plugins::marketplace::find_marketplace_manifest_path; pub(crate) use discoverable::list_tool_suggest_discoverable_plugins; pub(crate) use injection::build_plugin_injections; pub use installed_marketplaces::INSTALLED_MARKETPLACES_DIR; @@ -40,10 +41,12 @@ pub use manager::PluginRemoteSyncError; pub use manager::PluginUninstallError; pub use manager::PluginsManager; pub use manager::RemotePluginSyncResult; +pub(crate) use manager::configured_plugins_from_stack; pub use marketplace_add::MarketplaceAddError; pub use marketplace_add::MarketplaceAddOutcome; pub use marketplace_add::MarketplaceAddRequest; pub use marketplace_add::add_marketplace; +pub(crate) use marketplace_add::parse_marketplace_source; pub(crate) use render::render_explicit_plugin_instructions; pub(crate) use render::render_plugins_section; pub(crate) use startup_sync::curated_plugins_repo_path; From faf48489f3ade607701de064300ee127b1fb71d8 Mon Sep 17 00:00:00 2001 From: xli-oai Date: Thu, 16 Apr 2026 10:36:34 -0700 Subject: [PATCH 29/59] Auto-upgrade configured marketplaces (#17425) ## Summary - Add best-effort auto-upgrade for user-configured Git marketplaces recorded in `config.toml`. - Track the last activated Git revision with `last_revision` so unchanged marketplace sources skip clone work. - Trigger the upgrade from plugin startup and `plugin/list`, while preserving existing fail-open plugin behavior with warning logs rather than new user-visible errors. ## Details - Remote configured marketplaces use `git ls-remote` to compare the source/ref against the recorded revision. - Upgrades clone into a staging directory, validate that `.agents/plugins/marketplace.json` exists and that the manifest name matches the configured marketplace key, then atomically activate the new root. - Local `.agents/plugins/marketplace.json` marketplaces remain live filesystem state and are not auto-pulled. - Existing non-curated plugin cache refresh is kicked after successful marketplace root upgrades. ## Validation - `just write-config-schema` - `cargo test -p codex-core marketplace_upgrade` - `cargo check -p codex-cli -p codex-app-server` - `just fix -p codex-core` Did not run the complete `cargo test` suite because the repo instructions require asking before a full core workspace run. --- codex-rs/Cargo.lock | 1 + .../app-server/src/codex_message_processor.rs | 2 +- codex-rs/cli/src/marketplace_cmd.rs | 81 ++++- codex-rs/config/src/marketplace_edit.rs | 4 + codex-rs/config/src/types.rs | 3 + codex-rs/core-plugins/Cargo.toml | 1 + codex-rs/core-plugins/src/lib.rs | 1 + codex-rs/core-plugins/src/loader.rs | 33 +- .../core-plugins/src/marketplace_upgrade.rs | 298 ++++++++++++++++++ .../src/marketplace_upgrade/activation.rs | 167 ++++++++++ .../src/marketplace_upgrade/git.rs | 238 ++++++++++++++ codex-rs/core/config.schema.json | 5 + codex-rs/core/src/plugins/manager.rs | 209 ++++++++++-- codex-rs/core/src/plugins/manager_tests.rs | 64 ++++ .../src/plugins/marketplace_add/metadata.rs | 1 + .../src/plugins/marketplace_add/source.rs | 18 ++ codex-rs/core/src/plugins/mod.rs | 2 + 17 files changed, 1094 insertions(+), 34 deletions(-) create mode 100644 codex-rs/core-plugins/src/marketplace_upgrade.rs create mode 100644 codex-rs/core-plugins/src/marketplace_upgrade/activation.rs create mode 100644 codex-rs/core-plugins/src/marketplace_upgrade/git.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index a6d1ec2e47..a499a511ca 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -2019,6 +2019,7 @@ dependencies = [ name = "codex-core-plugins" version = "0.0.0" dependencies = [ + "chrono", "codex-app-server-protocol", "codex-config", "codex-core-skills", diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index 4d12df89fa..25892e40eb 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -6178,7 +6178,7 @@ impl CodexMessageProcessor { force_remote_sync, } = params; let roots = cwds.unwrap_or_default(); - plugins_manager.maybe_start_non_curated_plugin_cache_refresh_for_roots(&roots); + plugins_manager.maybe_start_non_curated_plugin_cache_refresh(&roots); let mut config = match self.load_latest_config(/*fallback_cwd*/ None).await { Ok(config) => config, diff --git a/codex-rs/cli/src/marketplace_cmd.rs b/codex-rs/cli/src/marketplace_cmd.rs index cde50a2bf4..ce1f99390a 100644 --- a/codex-rs/cli/src/marketplace_cmd.rs +++ b/codex-rs/cli/src/marketplace_cmd.rs @@ -1,8 +1,12 @@ use anyhow::Context; use anyhow::Result; +use anyhow::bail; use clap::Parser; +use codex_core::config::Config; use codex_core::config::find_codex_home; use codex_core::plugins::MarketplaceAddRequest; +use codex_core::plugins::PluginMarketplaceUpgradeOutcome; +use codex_core::plugins::PluginsManager; use codex_core::plugins::add_marketplace; use codex_utils_cli::CliConfigOverrides; @@ -17,8 +21,8 @@ pub struct MarketplaceCli { #[derive(Debug, clap::Subcommand)] enum MarketplaceSubcommand { - /// Add a remote marketplace repository. Add(AddMarketplaceArgs), + Upgrade(UpgradeMarketplaceArgs), } #[derive(Debug, Parser)] @@ -27,11 +31,9 @@ struct AddMarketplaceArgs { /// or local marketplace root directories. source: String, - /// Git ref to check out. Overrides any @ref or #ref suffix in SOURCE. #[arg(long = "ref", value_name = "REF")] ref_name: Option, - /// Sparse-checkout path to use while cloning git sources. Repeat to include multiple paths. #[arg( long = "sparse", value_name = "PATH", @@ -40,6 +42,11 @@ struct AddMarketplaceArgs { sparse_paths: Vec, } +#[derive(Debug, Parser)] +struct UpgradeMarketplaceArgs { + marketplace_name: Option, +} + impl MarketplaceCli { pub async fn run(self) -> Result<()> { let MarketplaceCli { @@ -47,14 +54,13 @@ impl MarketplaceCli { subcommand, } = self; - // Validate overrides now. This command writes to CODEX_HOME only; marketplace discovery - // happens from that cache root after the next plugin/list or app-server start. - config_overrides + let overrides = config_overrides .parse_overrides() .map_err(anyhow::Error::msg)?; match subcommand { MarketplaceSubcommand::Add(args) => run_add(args).await?, + MarketplaceSubcommand::Upgrade(args) => run_upgrade(overrides, args).await?, } Ok(()) @@ -98,6 +104,60 @@ async fn run_add(args: AddMarketplaceArgs) -> Result<()> { Ok(()) } +async fn run_upgrade( + overrides: Vec<(String, toml::Value)>, + args: UpgradeMarketplaceArgs, +) -> Result<()> { + let UpgradeMarketplaceArgs { marketplace_name } = args; + let config = Config::load_with_cli_overrides(overrides) + .await + .context("failed to load configuration")?; + let codex_home = find_codex_home().context("failed to resolve CODEX_HOME")?; + let manager = PluginsManager::new(codex_home.to_path_buf()); + let outcome = manager + .upgrade_configured_marketplaces_for_config(&config, marketplace_name.as_deref()) + .map_err(anyhow::Error::msg)?; + print_upgrade_outcome(&outcome, marketplace_name.as_deref()) +} + +fn print_upgrade_outcome( + outcome: &PluginMarketplaceUpgradeOutcome, + marketplace_name: Option<&str>, +) -> Result<()> { + for error in &outcome.errors { + eprintln!( + "Failed to upgrade marketplace `{}`: {}", + error.marketplace_name, error.message + ); + } + if !outcome.all_succeeded() { + bail!("{} upgrade failure(s) occurred.", outcome.errors.len()); + } + + let selection_label = marketplace_name.unwrap_or("all configured Git marketplaces"); + if outcome.selected_marketplaces.is_empty() { + println!("No configured Git marketplaces to upgrade."); + } else if outcome.upgraded_roots.is_empty() { + if marketplace_name.is_some() { + println!("Marketplace `{selection_label}` is already up to date."); + } else { + println!("All configured Git marketplaces are already up to date."); + } + } else if marketplace_name.is_some() { + println!("Upgraded marketplace `{selection_label}` to the latest configured revision."); + for root in &outcome.upgraded_roots { + println!("Installed marketplace root: {}", root.display()); + } + } else { + println!("Upgraded {} marketplace(s).", outcome.upgraded_roots.len()); + for root in &outcome.upgraded_roots { + println!("Installed marketplace root: {}", root.display()); + } + } + + Ok(()) +} + #[cfg(test)] mod tests { use super::*; @@ -132,4 +192,13 @@ mod tests { vec!["plugins/foo", "skills/bar"] ); } + + #[test] + fn upgrade_subcommand_parses_optional_marketplace_name() { + let upgrade_all = UpgradeMarketplaceArgs::try_parse_from(["upgrade"]).unwrap(); + assert_eq!(upgrade_all.marketplace_name, None); + + let upgrade_one = UpgradeMarketplaceArgs::try_parse_from(["upgrade", "debug"]).unwrap(); + assert_eq!(upgrade_one.marketplace_name.as_deref(), Some("debug")); + } } diff --git a/codex-rs/config/src/marketplace_edit.rs b/codex-rs/config/src/marketplace_edit.rs index 33cdd8e163..e20a75a02e 100644 --- a/codex-rs/config/src/marketplace_edit.rs +++ b/codex-rs/config/src/marketplace_edit.rs @@ -12,6 +12,7 @@ use crate::CONFIG_TOML_FILE; pub struct MarketplaceConfigUpdate<'a> { pub last_updated: &'a str, + pub last_revision: Option<&'a str>, pub source_type: &'a str, pub source: &'a str, pub ref_name: Option<&'a str>, @@ -63,6 +64,9 @@ fn upsert_marketplace( let mut entry = TomlTable::new(); entry.set_implicit(false); entry["last_updated"] = value(update.last_updated.to_string()); + if let Some(last_revision) = update.last_revision { + entry["last_revision"] = value(last_revision.to_string()); + } entry["source_type"] = value(update.source_type.to_string()); entry["source"] = value(update.source.to_string()); if let Some(ref_name) = update.ref_name { diff --git a/codex-rs/config/src/types.rs b/codex-rs/config/src/types.rs index 15c1a6fad5..b3576cb3c1 100644 --- a/codex-rs/config/src/types.rs +++ b/codex-rs/config/src/types.rs @@ -626,6 +626,9 @@ pub struct MarketplaceConfig { /// Last time Codex successfully added or refreshed this marketplace. #[serde(default)] pub last_updated: Option, + /// Git revision Codex last successfully activated for this marketplace. + #[serde(default)] + pub last_revision: Option, /// Source kind used to install this marketplace. #[serde(default)] pub source_type: Option, diff --git a/codex-rs/core-plugins/Cargo.toml b/codex-rs/core-plugins/Cargo.toml index 02e121f0e6..0372d9a14a 100644 --- a/codex-rs/core-plugins/Cargo.toml +++ b/codex-rs/core-plugins/Cargo.toml @@ -23,6 +23,7 @@ codex-plugin = { workspace = true } codex-protocol = { workspace = true } codex-utils-absolute-path = { workspace = true } codex-utils-plugins = { workspace = true } +chrono = { workspace = true } dirs = { workspace = true } reqwest = { workspace = true } serde = { workspace = true, features = ["derive"] } diff --git a/codex-rs/core-plugins/src/lib.rs b/codex-rs/core-plugins/src/lib.rs index e075158cb1..82ff4df3c7 100644 --- a/codex-rs/core-plugins/src/lib.rs +++ b/codex-rs/core-plugins/src/lib.rs @@ -1,6 +1,7 @@ pub mod loader; pub mod manifest; pub mod marketplace; +pub mod marketplace_upgrade; pub mod remote; pub mod store; pub mod toggles; diff --git a/codex-rs/core-plugins/src/loader.rs b/codex-rs/core-plugins/src/loader.rs index 206add5b53..ff3b8fc494 100644 --- a/codex-rs/core-plugins/src/loader.rs +++ b/codex-rs/core-plugins/src/loader.rs @@ -41,6 +41,12 @@ const DEFAULT_APP_CONFIG_FILE: &str = ".app.json"; const OPENAI_CURATED_MARKETPLACE_NAME: &str = "openai-curated"; const CONFIG_TOML_FILE: &str = "config.toml"; +#[derive(Clone, Copy, PartialEq, Eq)] +enum NonCuratedCacheRefreshMode { + IfVersionChanged, + ForceReinstall, +} + pub fn log_plugin_load_errors(outcome: &PluginLoadOutcome) { for plugin in outcome .plugins() @@ -180,6 +186,29 @@ pub fn refresh_curated_plugin_cache( pub fn refresh_non_curated_plugin_cache( codex_home: &Path, additional_roots: &[AbsolutePathBuf], +) -> Result { + refresh_non_curated_plugin_cache_with_mode( + codex_home, + additional_roots, + NonCuratedCacheRefreshMode::IfVersionChanged, + ) +} + +pub fn refresh_non_curated_plugin_cache_force_reinstall( + codex_home: &Path, + additional_roots: &[AbsolutePathBuf], +) -> Result { + refresh_non_curated_plugin_cache_with_mode( + codex_home, + additional_roots, + NonCuratedCacheRefreshMode::ForceReinstall, + ) +} + +fn refresh_non_curated_plugin_cache_with_mode( + codex_home: &Path, + additional_roots: &[AbsolutePathBuf], + mode: NonCuratedCacheRefreshMode, ) -> Result { let configured_non_curated_plugin_ids = non_curated_plugin_ids_from_config_keys(configured_plugins_from_codex_home( @@ -248,7 +277,9 @@ pub fn refresh_non_curated_plugin_cache( continue; }; - if store.active_plugin_version(&plugin_id).as_deref() == Some(plugin_version.as_str()) { + if mode == NonCuratedCacheRefreshMode::IfVersionChanged + && store.active_plugin_version(&plugin_id).as_deref() == Some(plugin_version.as_str()) + { continue; } diff --git a/codex-rs/core-plugins/src/marketplace_upgrade.rs b/codex-rs/core-plugins/src/marketplace_upgrade.rs new file mode 100644 index 0000000000..81474cf66f --- /dev/null +++ b/codex-rs/core-plugins/src/marketplace_upgrade.rs @@ -0,0 +1,298 @@ +mod activation; +mod git; + +use self::activation::activate_marketplace_root; +use self::activation::installed_marketplace_metadata_matches; +use self::activation::write_installed_marketplace_metadata; +use self::git::clone_git_source; +use self::git::git_remote_revision; +use crate::marketplace::validate_marketplace_root; +use codex_config::CONFIG_TOML_FILE; +use codex_config::ConfigLayerStack; +use codex_config::MarketplaceConfigUpdate; +use codex_config::record_user_marketplace; +use codex_config::types::MarketplaceConfig; +use codex_config::types::MarketplaceSourceType; +use codex_plugin::validate_plugin_segment; +use codex_utils_absolute_path::AbsolutePathBuf; +use std::collections::HashMap; +use std::path::Path; +use std::path::PathBuf; +use std::time::Duration; +use tracing::warn; + +const INSTALLED_MARKETPLACES_DIR: &str = ".tmp/marketplaces"; +const MARKETPLACE_UPGRADE_GIT_TIMEOUT: Duration = Duration::from_secs(30); + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ConfiguredMarketplaceUpgradeError { + pub marketplace_name: String, + pub message: String, +} + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct ConfiguredMarketplaceUpgradeOutcome { + pub selected_marketplaces: Vec, + pub upgraded_roots: Vec, + pub errors: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct ConfiguredGitMarketplace { + name: String, + source: String, + ref_name: Option, + sparse_paths: Vec, + last_revision: Option, +} + +impl ConfiguredMarketplaceUpgradeOutcome { + pub fn all_succeeded(&self) -> bool { + self.errors.is_empty() + } +} + +pub fn configured_git_marketplace_names(config_layer_stack: &ConfigLayerStack) -> Vec { + let mut names = configured_git_marketplaces(config_layer_stack) + .into_iter() + .map(|marketplace| marketplace.name) + .collect::>(); + names.sort_unstable(); + names +} + +pub fn upgrade_configured_git_marketplaces( + codex_home: &Path, + config_layer_stack: &ConfigLayerStack, + marketplace_name: Option<&str>, +) -> ConfiguredMarketplaceUpgradeOutcome { + let marketplaces = configured_git_marketplaces(config_layer_stack) + .into_iter() + .filter(|marketplace| marketplace_name.is_none_or(|name| marketplace.name.as_str() == name)) + .collect::>(); + if marketplaces.is_empty() { + return ConfiguredMarketplaceUpgradeOutcome::default(); + } + + let install_root = marketplace_install_root(codex_home); + let selected_marketplaces = marketplaces + .iter() + .map(|marketplace| marketplace.name.clone()) + .collect(); + let mut upgraded_roots = Vec::new(); + let mut errors = Vec::new(); + for marketplace in marketplaces { + match upgrade_configured_git_marketplace(codex_home, &install_root, &marketplace) { + Ok(Some(upgraded_root)) => upgraded_roots.push(upgraded_root), + Ok(None) => {} + Err(err) => { + errors.push(ConfiguredMarketplaceUpgradeError { + marketplace_name: marketplace.name, + message: err, + }); + } + } + } + + ConfiguredMarketplaceUpgradeOutcome { + selected_marketplaces, + upgraded_roots, + errors, + } +} + +fn marketplace_install_root(codex_home: &Path) -> PathBuf { + codex_home.join(INSTALLED_MARKETPLACES_DIR) +} + +fn configured_git_marketplaces( + config_layer_stack: &ConfigLayerStack, +) -> Vec { + let Some(user_layer) = config_layer_stack.get_user_layer() else { + return Vec::new(); + }; + let Some(marketplaces_value) = user_layer.config.get("marketplaces") else { + return Vec::new(); + }; + let marketplaces = match marketplaces_value + .clone() + .try_into::>() + { + Ok(marketplaces) => marketplaces, + Err(err) => { + warn!("invalid marketplaces config while preparing auto-upgrade: {err}"); + return Vec::new(); + } + }; + + let mut configured = marketplaces + .into_iter() + .filter_map(|(name, marketplace)| configured_git_marketplace_from_config(name, marketplace)) + .collect::>(); + configured.sort_unstable_by(|left, right| left.name.cmp(&right.name)); + configured +} + +fn configured_git_marketplace_from_config( + name: String, + marketplace: MarketplaceConfig, +) -> Option { + let MarketplaceConfig { + last_updated: _, + last_revision, + source_type, + source, + ref_name, + sparse_paths, + } = marketplace; + if source_type != Some(MarketplaceSourceType::Git) { + return None; + } + let Some(source) = source else { + warn!( + marketplace = name, + "ignoring configured Git marketplace without source" + ); + return None; + }; + Some(ConfiguredGitMarketplace { + name, + source, + ref_name, + sparse_paths: sparse_paths.unwrap_or_default(), + last_revision, + }) +} + +fn upgrade_configured_git_marketplace( + codex_home: &Path, + install_root: &Path, + marketplace: &ConfiguredGitMarketplace, +) -> Result, String> { + validate_plugin_segment(&marketplace.name, "marketplace name")?; + let remote_revision = git_remote_revision( + &marketplace.source, + marketplace.ref_name.as_deref(), + MARKETPLACE_UPGRADE_GIT_TIMEOUT, + )?; + let destination = install_root.join(&marketplace.name); + if destination + .join(".agents/plugins/marketplace.json") + .is_file() + && marketplace.last_revision.as_deref() == Some(remote_revision.as_str()) + && installed_marketplace_metadata_matches(&destination, marketplace, &remote_revision) + { + return Ok(None); + } + + let staging_parent = install_root.join(".staging"); + std::fs::create_dir_all(&staging_parent).map_err(|err| { + format!( + "failed to create marketplace upgrade staging directory {}: {err}", + staging_parent.display() + ) + })?; + let staged_dir = tempfile::Builder::new() + .prefix("marketplace-upgrade-") + .tempdir_in(&staging_parent) + .map_err(|err| { + format!( + "failed to create temporary marketplace upgrade directory in {}: {err}", + staging_parent.display() + ) + })?; + + let activated_revision = clone_git_source( + &marketplace.source, + marketplace.ref_name.as_deref(), + &marketplace.sparse_paths, + staged_dir.path(), + MARKETPLACE_UPGRADE_GIT_TIMEOUT, + )?; + let marketplace_name = validate_marketplace_root(staged_dir.path()) + .map_err(|err| format!("failed to validate upgraded marketplace root: {err}"))?; + if marketplace_name != marketplace.name { + return Err(format!( + "upgraded marketplace name `{marketplace_name}` does not match configured marketplace `{}`", + marketplace.name + )); + } + write_installed_marketplace_metadata(staged_dir.path(), marketplace, &activated_revision)?; + + let last_updated = chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true); + let update = MarketplaceConfigUpdate { + last_updated: &last_updated, + last_revision: Some(&activated_revision), + source_type: "git", + source: &marketplace.source, + ref_name: marketplace.ref_name.as_deref(), + sparse_paths: &marketplace.sparse_paths, + }; + activate_marketplace_root(&destination, staged_dir, || { + ensure_configured_git_marketplace_unchanged(codex_home, marketplace)?; + record_user_marketplace(codex_home, &marketplace.name, &update).map_err(|err| { + format!( + "failed to record upgraded marketplace `{}` in user config.toml: {err}", + marketplace.name + ) + }) + })?; + + AbsolutePathBuf::try_from(destination) + .map(Some) + .map_err(|err| format!("upgraded marketplace path is not absolute: {err}")) +} +fn ensure_configured_git_marketplace_unchanged( + codex_home: &Path, + expected: &ConfiguredGitMarketplace, +) -> Result<(), String> { + let current = read_configured_git_marketplace(codex_home, &expected.name)?; + match current { + Some(current) if current == *expected => Ok(()), + Some(_) => Err(format!( + "configured marketplace `{}` changed while auto-upgrade was in flight", + expected.name + )), + None => Err(format!( + "configured marketplace `{}` was removed or is no longer a Git marketplace", + expected.name + )), + } +} + +fn read_configured_git_marketplace( + codex_home: &Path, + marketplace_name: &str, +) -> Result, String> { + let config_path = codex_home.join(CONFIG_TOML_FILE); + let raw_config = match std::fs::read_to_string(&config_path) { + Ok(raw_config) => raw_config, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(err) => { + return Err(format!( + "failed to read user config {} while checking marketplace auto-upgrade: {err}", + config_path.display() + )); + } + }; + let config: toml::Value = toml::from_str(&raw_config).map_err(|err| { + format!( + "failed to parse user config {} while checking marketplace auto-upgrade: {err}", + config_path.display() + ) + })?; + let Some(marketplaces_value) = config.get("marketplaces") else { + return Ok(None); + }; + let mut marketplaces = marketplaces_value + .clone() + .try_into::>() + .map_err(|err| format!("invalid marketplaces config while checking auto-upgrade: {err}"))?; + let Some(marketplace) = marketplaces.remove(marketplace_name) else { + return Ok(None); + }; + Ok(configured_git_marketplace_from_config( + marketplace_name.to_string(), + marketplace, + )) +} diff --git a/codex-rs/core-plugins/src/marketplace_upgrade/activation.rs b/codex-rs/core-plugins/src/marketplace_upgrade/activation.rs new file mode 100644 index 0000000000..366b35fb43 --- /dev/null +++ b/codex-rs/core-plugins/src/marketplace_upgrade/activation.rs @@ -0,0 +1,167 @@ +use super::ConfiguredGitMarketplace; +use codex_config::types::MarketplaceSourceType; +use serde::Deserialize; +use serde::Serialize; +use std::path::Path; +use std::path::PathBuf; +use tempfile::TempDir; +use tracing::warn; + +const MARKETPLACE_INSTALL_METADATA_FILE: &str = ".codex-marketplace-install.json"; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +struct InstalledMarketplaceMetadata { + source_type: MarketplaceSourceType, + source: String, + ref_name: Option, + sparse_paths: Vec, + revision: String, +} + +pub(super) fn installed_marketplace_metadata_matches( + root: &Path, + marketplace: &ConfiguredGitMarketplace, + revision: &str, +) -> bool { + let metadata = match std::fs::read_to_string(installed_marketplace_metadata_path(root)) { + Ok(metadata) => metadata, + Err(_) => return false, + }; + let metadata = match serde_json::from_str::(&metadata) { + Ok(metadata) => metadata, + Err(err) => { + warn!( + marketplace = marketplace.name, + error = %err, + "failed to parse activated marketplace metadata" + ); + return false; + } + }; + metadata == installed_marketplace_metadata(marketplace, revision) +} + +pub(super) fn write_installed_marketplace_metadata( + root: &Path, + marketplace: &ConfiguredGitMarketplace, + revision: &str, +) -> Result<(), String> { + let metadata = installed_marketplace_metadata(marketplace, revision); + let contents = serde_json::to_string_pretty(&metadata) + .map_err(|err| format!("failed to serialize activated marketplace metadata: {err}"))?; + std::fs::write(installed_marketplace_metadata_path(root), contents) + .map_err(|err| format!("failed to write activated marketplace metadata: {err}")) +} + +pub(super) fn activate_marketplace_root( + destination: &Path, + staged_dir: TempDir, + after_activate: impl FnOnce() -> Result<(), String>, +) -> Result<(), String> { + let staged_root = staged_dir.path(); + let Some(parent) = destination.parent() else { + return Err(format!( + "failed to determine marketplace install parent for {}", + destination.display() + )); + }; + std::fs::create_dir_all(parent).map_err(|err| { + format!( + "failed to create marketplace install parent {}: {err}", + parent.display() + ) + })?; + + if destination.exists() { + let backup_dir = tempfile::Builder::new() + .prefix("marketplace-backup-") + .tempdir_in(parent) + .map_err(|err| { + format!( + "failed to create marketplace backup directory in {}: {err}", + parent.display() + ) + })?; + let backup_root = backup_dir.path().join("root"); + std::fs::rename(destination, &backup_root).map_err(|err| { + format!( + "failed to move previous marketplace root out of the way at {}: {err}", + destination.display() + ) + })?; + + if let Err(err) = std::fs::rename(staged_root, destination) { + let rollback_result = std::fs::rename(&backup_root, destination); + return match rollback_result { + Ok(()) => Err(format!( + "failed to activate upgraded marketplace at {}: {err}", + destination.display() + )), + Err(rollback_err) => { + let backup_path = backup_dir.keep().join("root"); + Err(format!( + "failed to activate upgraded marketplace at {}: {err}; failed to restore previous marketplace root (left at {}): {rollback_err}", + destination.display(), + backup_path.display() + )) + } + }; + } + + if let Err(err) = after_activate() { + let remove_result = std::fs::remove_dir_all(destination); + let rollback_result = + remove_result.and_then(|()| std::fs::rename(&backup_root, destination)); + return match rollback_result { + Ok(()) => Err(err), + Err(rollback_err) => { + let backup_path = backup_dir.keep().join("root"); + Err(format!( + "{err}; failed to restore previous marketplace root at {} (left at {}): {rollback_err}", + destination.display(), + backup_path.display() + )) + } + }; + } + + return Ok(()); + } + + std::fs::rename(staged_root, destination).map_err(|err| { + format!( + "failed to activate upgraded marketplace at {}: {err}", + destination.display() + ) + })?; + if let Err(err) = after_activate() { + let remove_result = std::fs::remove_dir_all(destination); + return match remove_result { + Ok(()) => Err(err), + Err(remove_err) => Err(format!( + "{err}; failed to remove newly activated marketplace root at {}: {remove_err}", + destination.display() + )), + }; + } + + Ok(()) +} + +fn installed_marketplace_metadata( + marketplace: &ConfiguredGitMarketplace, + revision: &str, +) -> InstalledMarketplaceMetadata { + InstalledMarketplaceMetadata { + source_type: MarketplaceSourceType::Git, + source: marketplace.source.clone(), + ref_name: marketplace.ref_name.clone(), + sparse_paths: marketplace.sparse_paths.clone(), + revision: revision.to_string(), + } +} + +fn installed_marketplace_metadata_path(root: &Path) -> PathBuf { + root.join(MARKETPLACE_INSTALL_METADATA_FILE) +} diff --git a/codex-rs/core-plugins/src/marketplace_upgrade/git.rs b/codex-rs/core-plugins/src/marketplace_upgrade/git.rs new file mode 100644 index 0000000000..80a7c68f68 --- /dev/null +++ b/codex-rs/core-plugins/src/marketplace_upgrade/git.rs @@ -0,0 +1,238 @@ +use std::path::Path; +use std::process::Command; +use std::process::Output; +use std::process::Stdio; +use std::time::Duration; + +pub(super) fn git_remote_revision( + source: &str, + ref_name: Option<&str>, + timeout: Duration, +) -> Result { + if let Some(ref_name) = ref_name + && is_full_git_sha(ref_name) + { + return Ok(ref_name.to_string()); + } + + let ref_name = ref_name.unwrap_or("HEAD"); + let output = run_git_command_with_timeout( + git_command().arg("ls-remote").arg(source).arg(ref_name), + "git ls-remote marketplace source", + timeout, + )?; + ensure_git_success(&output, "git ls-remote marketplace source")?; + + let stdout = String::from_utf8_lossy(&output.stdout); + let Some(first_line) = stdout.lines().next() else { + return Err("git ls-remote returned empty output for marketplace source".to_string()); + }; + let Some((revision, _)) = first_line.split_once('\t') else { + return Err(format!( + "unexpected git ls-remote output for marketplace source: {first_line}" + )); + }; + let revision = revision.trim(); + if revision.is_empty() { + return Err("git ls-remote returned empty revision for marketplace source".to_string()); + } + Ok(revision.to_string()) +} + +pub(super) fn clone_git_source( + source: &str, + ref_name: Option<&str>, + sparse_paths: &[String], + destination: &Path, + timeout: Duration, +) -> Result { + if sparse_paths.is_empty() { + let output = run_git_command_with_timeout( + git_command().arg("clone").arg(source).arg(destination), + "git clone marketplace source", + timeout, + )?; + ensure_git_success(&output, "git clone marketplace source")?; + if let Some(ref_name) = ref_name { + let output = run_git_command_with_timeout( + git_command() + .arg("-C") + .arg(destination) + .arg("checkout") + .arg(ref_name), + "git checkout marketplace ref", + timeout, + )?; + ensure_git_success(&output, "git checkout marketplace ref")?; + } + return git_worktree_revision(destination, timeout); + } + + let output = run_git_command_with_timeout( + git_command() + .arg("clone") + .arg("--filter=blob:none") + .arg("--no-checkout") + .arg(source) + .arg(destination), + "git clone marketplace source", + timeout, + )?; + ensure_git_success(&output, "git clone marketplace source")?; + + let mut sparse_checkout = git_command(); + sparse_checkout + .arg("-C") + .arg(destination) + .arg("sparse-checkout") + .arg("set") + .args(sparse_paths); + let output = run_git_command_with_timeout( + &mut sparse_checkout, + "git sparse-checkout marketplace source", + timeout, + )?; + ensure_git_success(&output, "git sparse-checkout marketplace source")?; + + let output = run_git_command_with_timeout( + git_command() + .arg("-C") + .arg(destination) + .arg("checkout") + .arg(ref_name.unwrap_or("HEAD")), + "git checkout marketplace ref", + timeout, + )?; + ensure_git_success(&output, "git checkout marketplace ref")?; + git_worktree_revision(destination, timeout) +} + +fn git_worktree_revision(destination: &Path, timeout: Duration) -> Result { + let output = run_git_command_with_timeout( + git_command() + .arg("-C") + .arg(destination) + .arg("rev-parse") + .arg("HEAD"), + "git rev-parse marketplace revision", + timeout, + )?; + ensure_git_success(&output, "git rev-parse marketplace revision")?; + + let revision = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if revision.is_empty() { + Err("git rev-parse returned empty revision for marketplace source".to_string()) + } else { + Ok(revision) + } +} + +fn is_full_git_sha(value: &str) -> bool { + value.len() == 40 && value.chars().all(|ch| ch.is_ascii_hexdigit()) +} + +fn git_command() -> Command { + let mut command = Command::new("git"); + command + .env("GIT_OPTIONAL_LOCKS", "0") + .env("GIT_TERMINAL_PROMPT", "0"); + command +} + +fn run_git_command_with_timeout( + command: &mut Command, + context: &str, + timeout: Duration, +) -> Result { + let mut child = command + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|err| format!("failed to run {context}: {err}"))?; + let start = std::time::Instant::now(); + loop { + match child.try_wait() { + Ok(Some(_)) => { + return child + .wait_with_output() + .map_err(|err| format!("failed to wait for {context}: {err}")); + } + Ok(None) => {} + Err(err) => return Err(format!("failed to poll {context}: {err}")), + } + + if start.elapsed() >= timeout { + let _ = child.kill(); + let output = child + .wait_with_output() + .map_err(|err| format!("failed to wait for {context} after timeout: {err}"))?; + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + return if stderr.is_empty() { + Err(format!("{context} timed out after {}s", timeout.as_secs())) + } else { + Err(format!( + "{context} timed out after {}s: {stderr}", + timeout.as_secs() + )) + }; + } + + std::thread::sleep(Duration::from_millis(100)); + } +} + +fn ensure_git_success(output: &Output, context: &str) -> Result<(), String> { + if output.status.success() { + return Ok(()); + } + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + if stderr.is_empty() { + Err(format!("{context} failed with status {}", output.status)) + } else { + Err(format!( + "{context} failed with status {}: {stderr}", + output.status + )) + } +} + +#[cfg(test)] +mod tests { + use super::git_command; + use super::is_full_git_sha; + use std::ffi::OsStr; + + #[test] + fn full_git_sha_ref_is_already_a_remote_revision() { + assert!(is_full_git_sha("0123456789abcdef0123456789abcdef01234567")); + assert!(!is_full_git_sha("main")); + assert!(!is_full_git_sha("0123456")); + } + + #[test] + fn git_command_uses_path_lookup_with_stable_noninteractive_env() { + let command = git_command(); + + assert_eq!(command.get_program(), OsStr::new("git")); + assert_eq!( + command_env(&command, "GIT_OPTIONAL_LOCKS"), + Some(Some(OsStr::new("0"))) + ); + assert_eq!( + command_env(&command, "GIT_TERMINAL_PROMPT"), + Some(Some(OsStr::new("0"))) + ); + assert_eq!(command_env(&command, "PATH"), None); + } + + fn command_env<'a>( + command: &'a std::process::Command, + name: &str, + ) -> Option> { + command + .get_envs() + .find(|(key, _)| key == &OsStr::new(name)) + .map(|(_, value)| value) + } +} diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.json index 2e381cd3f4..9c7eb853a9 100644 --- a/codex-rs/core/config.schema.json +++ b/codex-rs/core/config.schema.json @@ -779,6 +779,11 @@ "MarketplaceConfig": { "additionalProperties": false, "properties": { + "last_revision": { + "default": null, + "description": "Git revision Codex last successfully activated for this marketplace.", + "type": "string" + }, "last_updated": { "default": null, "description": "Last time Codex successfully added or refreshed this marketplace.", diff --git a/codex-rs/core/src/plugins/manager.rs b/codex-rs/core/src/plugins/manager.rs index 3449ecffdf..7a5991b03a 100644 --- a/codex-rs/core/src/plugins/manager.rs +++ b/codex-rs/core/src/plugins/manager.rs @@ -25,6 +25,7 @@ use codex_core_plugins::loader::log_plugin_load_errors; use codex_core_plugins::loader::plugin_telemetry_metadata_from_root; use codex_core_plugins::loader::refresh_curated_plugin_cache; use codex_core_plugins::loader::refresh_non_curated_plugin_cache; +use codex_core_plugins::loader::refresh_non_curated_plugin_cache_force_reinstall; use codex_core_plugins::manifest::PluginManifestInterface; use codex_core_plugins::manifest::load_plugin_manifest; use codex_core_plugins::marketplace::MarketplaceError; @@ -37,6 +38,10 @@ use codex_core_plugins::marketplace::ResolvedMarketplacePlugin; use codex_core_plugins::marketplace::list_marketplaces; use codex_core_plugins::marketplace::load_marketplace; use codex_core_plugins::marketplace::resolve_marketplace_plugin; +use codex_core_plugins::marketplace_upgrade::ConfiguredMarketplaceUpgradeError; +use codex_core_plugins::marketplace_upgrade::ConfiguredMarketplaceUpgradeOutcome; +use codex_core_plugins::marketplace_upgrade::configured_git_marketplace_names; +use codex_core_plugins::marketplace_upgrade::upgrade_configured_git_marketplaces; use codex_core_plugins::remote::RemotePluginFetchError; use codex_core_plugins::remote::RemotePluginMutationError; use codex_core_plugins::remote::RemotePluginServiceConfig; @@ -88,10 +93,27 @@ struct CachedFeaturedPluginIds { featured_plugin_ids: Vec, } +#[derive(Clone, PartialEq, Eq)] +struct NonCuratedCacheRefreshRequest { + roots: Vec, + mode: NonCuratedCacheRefreshMode, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum NonCuratedCacheRefreshMode { + IfVersionChanged, + ForceReinstall, +} + #[derive(Default)] struct NonCuratedCacheRefreshState { - requested_roots: Option>, - last_refreshed_roots: Option>, + requested: Option, + last_refreshed: Option, + in_flight: bool, +} + +#[derive(Default)] +struct ConfiguredMarketplaceUpgradeState { in_flight: bool, } @@ -314,6 +336,7 @@ pub struct PluginsManager { codex_home: PathBuf, store: PluginStore, featured_plugin_ids_cache: RwLock>, + configured_marketplace_upgrade_state: RwLock, non_curated_cache_refresh_state: RwLock, cached_enabled_outcome: RwLock>, remote_sync_lock: Mutex<()>, @@ -341,6 +364,9 @@ impl PluginsManager { codex_home: codex_home.clone(), store: PluginStore::new(codex_home), featured_plugin_ids_cache: RwLock::new(None), + configured_marketplace_upgrade_state: RwLock::new( + ConfiguredMarketplaceUpgradeState::default(), + ), non_curated_cache_refresh_state: RwLock::new(NonCuratedCacheRefreshState::default()), cached_enabled_outcome: RwLock::new(None), remote_sync_lock: Mutex::new(()), @@ -1074,6 +1100,57 @@ impl PluginsManager { ) { if config.features.enabled(Feature::Plugins) { self.start_curated_repo_sync(); + let should_spawn_marketplace_auto_upgrade = { + let mut state = match self.configured_marketplace_upgrade_state.write() { + Ok(state) => state, + Err(err) => err.into_inner(), + }; + if state.in_flight { + false + } else { + state.in_flight = true; + true + } + }; + if should_spawn_marketplace_auto_upgrade { + let manager = Arc::clone(self); + let config = config.clone(); + if let Err(err) = std::thread::Builder::new() + .name("plugins-marketplace-auto-upgrade".to_string()) + .spawn(move || { + let outcome = manager.upgrade_configured_marketplaces_for_config( + &config, /*marketplace_name*/ None, + ); + match outcome { + Ok(outcome) => { + for error in outcome.errors { + warn!( + marketplace = error.marketplace_name, + error = %error.message, + "failed to auto-upgrade configured marketplace" + ); + } + } + Err(err) => { + warn!("failed to auto-upgrade configured marketplaces: {err}"); + } + } + + let mut state = match manager.configured_marketplace_upgrade_state.write() { + Ok(state) => state, + Err(err) => err.into_inner(), + }; + state.in_flight = false; + }) + { + let mut state = match self.configured_marketplace_upgrade_state.write() { + Ok(state) => state, + Err(err) => err.into_inner(), + }; + state.in_flight = false; + warn!("failed to start configured marketplace auto-upgrade task: {err}"); + } + } start_startup_remote_plugin_sync_once( Arc::clone(self), self.codex_home.clone(), @@ -1098,9 +1175,66 @@ impl PluginsManager { } } - pub fn maybe_start_non_curated_plugin_cache_refresh_for_roots( + pub fn upgrade_configured_marketplaces_for_config( + &self, + config: &Config, + marketplace_name: Option<&str>, + ) -> Result { + if let Some(marketplace_name) = marketplace_name + && !configured_git_marketplace_names(&config.config_layer_stack) + .iter() + .any(|name| name == marketplace_name) + { + return Err(format!( + "marketplace `{marketplace_name}` is not configured as a Git marketplace" + )); + } + + let mut outcome = upgrade_configured_git_marketplaces( + self.codex_home.as_path(), + &config.config_layer_stack, + marketplace_name, + ); + if !outcome.upgraded_roots.is_empty() { + match refresh_non_curated_plugin_cache_force_reinstall( + self.codex_home.as_path(), + &outcome.upgraded_roots, + ) { + Ok(cache_refreshed) => { + if cache_refreshed { + self.clear_cache(); + } + } + Err(err) => { + self.clear_cache(); + outcome.errors.push(ConfiguredMarketplaceUpgradeError { + marketplace_name: marketplace_name + .unwrap_or("all configured marketplaces") + .to_string(), + message: format!( + "failed to refresh installed plugin cache after marketplace upgrade: {err}" + ), + }); + } + } + } + Ok(outcome) + } + + pub fn maybe_start_non_curated_plugin_cache_refresh( self: &Arc, roots: &[AbsolutePathBuf], + ) { + self.schedule_non_curated_plugin_cache_refresh( + roots, + NonCuratedCacheRefreshMode::IfVersionChanged, + ); + } + + fn schedule_non_curated_plugin_cache_refresh( + self: &Arc, + roots: &[AbsolutePathBuf], + mode: NonCuratedCacheRefreshMode, ) { let mut roots = roots.to_vec(); roots.sort_unstable(); @@ -1108,6 +1242,7 @@ impl PluginsManager { if roots.is_empty() { return; } + let request = NonCuratedCacheRefreshRequest { roots, mode }; let should_spawn = { let mut state = match self.non_curated_cache_refresh_state.write() { @@ -1115,13 +1250,25 @@ impl PluginsManager { Err(err) => err.into_inner(), }; // Collapse repeated plugin/list requests onto one worker and only queue another pass - // when the requested roots set actually changes. - if state.requested_roots.as_ref() == Some(&roots) - || (!state.in_flight && state.last_refreshed_roots.as_ref() == Some(&roots)) + // when the requested roots set actually changes. Forced reinstall requests are not + // deduped against the last completed pass because the same marketplace root path can + // point at newly activated files after an auto-upgrade. + if state.requested.as_ref() == Some(&request) + || (mode == NonCuratedCacheRefreshMode::IfVersionChanged + && !state.in_flight + && state.last_refreshed.as_ref() == Some(&request)) { return; } - state.requested_roots = Some(roots); + if mode == NonCuratedCacheRefreshMode::IfVersionChanged + && state.requested.as_ref().is_some_and(|requested| { + requested.mode == NonCuratedCacheRefreshMode::ForceReinstall + && requested.roots == request.roots + }) + { + return; + } + state.requested = Some(request); if state.in_flight { false } else { @@ -1143,7 +1290,7 @@ impl PluginsManager { Err(err) => err.into_inner(), }; state.in_flight = false; - state.requested_roots = None; + state.requested = None; warn!("failed to start non-curated plugin cache refresh task: {err}"); } } @@ -1192,15 +1339,15 @@ impl PluginsManager { fn run_non_curated_plugin_cache_refresh_loop(self: Arc) { loop { - let roots = { + let request = { let state = match self.non_curated_cache_refresh_state.read() { Ok(state) => state, Err(err) => err.into_inner(), }; - state.requested_roots.clone() + state.requested.clone() }; - let Some(roots) = roots else { + let Some(request) = request else { let mut state = match self.non_curated_cache_refresh_state.write() { Ok(state) => state, Err(err) => err.into_inner(), @@ -1209,30 +1356,40 @@ impl PluginsManager { return; }; - let refreshed = - match refresh_non_curated_plugin_cache(self.codex_home.as_path(), &roots) { - Ok(cache_refreshed) => { - if cache_refreshed { - self.clear_cache(); - } - true - } - Err(err) => { + let refresh_result = match request.mode { + NonCuratedCacheRefreshMode::IfVersionChanged => { + refresh_non_curated_plugin_cache(self.codex_home.as_path(), &request.roots) + } + NonCuratedCacheRefreshMode::ForceReinstall => { + refresh_non_curated_plugin_cache_force_reinstall( + self.codex_home.as_path(), + &request.roots, + ) + } + }; + let refreshed = match refresh_result { + Ok(cache_refreshed) => { + if cache_refreshed { self.clear_cache(); - warn!("failed to refresh non-curated plugin cache: {err}"); - false } - }; + true + } + Err(err) => { + self.clear_cache(); + warn!("failed to refresh non-curated plugin cache: {err}"); + false + } + }; let mut state = match self.non_curated_cache_refresh_state.write() { Ok(state) => state, Err(err) => err.into_inner(), }; if refreshed { - state.last_refreshed_roots = Some(roots.clone()); + state.last_refreshed = Some(request.clone()); } - if state.requested_roots.as_ref() == Some(&roots) { - state.requested_roots = None; + if state.requested.as_ref() == Some(&request) { + state.requested = None; state.in_flight = false; return; } diff --git a/codex-rs/core/src/plugins/manager_tests.rs b/codex-rs/core/src/plugins/manager_tests.rs index e12f52c82b..c82e1d174c 100644 --- a/codex-rs/core/src/plugins/manager_tests.rs +++ b/codex-rs/core/src/plugins/manager_tests.rs @@ -15,6 +15,8 @@ use crate::plugins::test_support::write_openai_curated_marketplace; use codex_app_server_protocol::ConfigLayerSource; use codex_config::McpServerConfig; use codex_config::types::McpServerTransportConfig; +use codex_core_plugins::loader::refresh_non_curated_plugin_cache; +use codex_core_plugins::loader::refresh_non_curated_plugin_cache_force_reinstall; use codex_core_plugins::marketplace::MarketplacePluginInstallPolicy; use codex_login::CodexAuth; use codex_protocol::protocol::Product; @@ -2676,6 +2678,68 @@ enabled = true ); } +#[test] +fn refresh_non_curated_plugin_cache_force_reinstalls_current_local_version() { + let tmp = tempfile::tempdir().unwrap(); + let repo_root = tmp.path().join("repo"); + fs::create_dir_all(repo_root.join(".git")).unwrap(); + fs::create_dir_all(repo_root.join(".agents/plugins")).unwrap(); + write_plugin(&repo_root, "sample-plugin", "sample-plugin"); + fs::write(repo_root.join("sample-plugin/skills/SKILL.md"), "new skill").unwrap(); + write_file( + &repo_root.join(".agents/plugins/marketplace.json"), + r#"{ + "name": "debug", + "plugins": [ + { + "name": "sample-plugin", + "source": { + "source": "local", + "path": "./sample-plugin" + } + } + ] +}"#, + ); + write_plugin( + &tmp.path().join("plugins/cache/debug"), + "sample-plugin/local", + "sample-plugin", + ); + fs::write( + tmp.path() + .join("plugins/cache/debug/sample-plugin/local/skills/SKILL.md"), + "old skill", + ) + .unwrap(); + write_file( + &tmp.path().join(CONFIG_TOML_FILE), + r#"[features] +plugins = true + +[plugins."sample-plugin@debug"] +enabled = true +"#, + ); + + assert!( + refresh_non_curated_plugin_cache_force_reinstall( + tmp.path(), + &[AbsolutePathBuf::try_from(repo_root).unwrap()], + ) + .expect("cache refresh should reinstall unchanged local version") + ); + + assert_eq!( + fs::read_to_string( + tmp.path() + .join("plugins/cache/debug/sample-plugin/local/skills/SKILL.md") + ) + .unwrap(), + "new skill" + ); +} + #[test] fn refresh_non_curated_plugin_cache_ignores_invalid_unconfigured_plugin_versions() { let tmp = tempfile::tempdir().unwrap(); diff --git a/codex-rs/core/src/plugins/marketplace_add/metadata.rs b/codex-rs/core/src/plugins/marketplace_add/metadata.rs index f21ed0faa2..ccded11c87 100644 --- a/codex-rs/core/src/plugins/marketplace_add/metadata.rs +++ b/codex-rs/core/src/plugins/marketplace_add/metadata.rs @@ -38,6 +38,7 @@ pub(super) fn record_added_marketplace_entry( let timestamp = utc_timestamp_now()?; let update = MarketplaceConfigUpdate { last_updated: ×tamp, + last_revision: None, source_type: install_metadata.config_source_type(), source: &source, ref_name: install_metadata.ref_name(), diff --git a/codex-rs/core/src/plugins/marketplace_add/source.rs b/codex-rs/core/src/plugins/marketplace_add/source.rs index 98bdc2b8bb..e723c8865a 100644 --- a/codex-rs/core/src/plugins/marketplace_add/source.rs +++ b/codex-rs/core/src/plugins/marketplace_add/source.rs @@ -125,6 +125,7 @@ fn normalize_git_url(url: &str) -> String { fn looks_like_local_path(source: &str) -> bool { Path::new(source).is_absolute() + || looks_like_windows_absolute_path(source) || source.starts_with("./") || source.starts_with(".\\") || source.starts_with("../") @@ -134,6 +135,15 @@ fn looks_like_local_path(source: &str) -> bool { || source == ".." } +fn looks_like_windows_absolute_path(source: &str) -> bool { + let bytes = source.as_bytes(); + bytes.len() >= 3 + && bytes[0].is_ascii_alphabetic() + && bytes[1] == b':' + && matches!(bytes[2], b'\\' | b'/') + || source.starts_with(r"\\") +} + fn resolve_local_source_path(source: &str) -> Result { let path = expand_tilde_path(source); let path = if path.is_absolute() { @@ -312,6 +322,14 @@ mod tests { assert!(path.is_absolute()); } + #[test] + fn windows_absolute_paths_look_like_local_paths_on_every_host() { + assert!(looks_like_local_path(r"C:\Users\alice\marketplace")); + assert!(looks_like_local_path("C:/Users/alice/marketplace")); + assert!(looks_like_local_path(r"\\server\share\marketplace")); + assert!(!looks_like_local_path(r"C:relative\path")); + } + #[test] fn local_file_source_is_rejected() { let tempdir = TempDir::new().unwrap(); diff --git a/codex-rs/core/src/plugins/mod.rs b/codex-rs/core/src/plugins/mod.rs index 2e86643f34..16cc8d08c4 100644 --- a/codex-rs/core/src/plugins/mod.rs +++ b/codex-rs/core/src/plugins/mod.rs @@ -11,6 +11,8 @@ mod startup_sync; #[cfg(test)] pub(crate) mod test_support; +pub use codex_core_plugins::marketplace_upgrade::ConfiguredMarketplaceUpgradeError as PluginMarketplaceUpgradeError; +pub use codex_core_plugins::marketplace_upgrade::ConfiguredMarketplaceUpgradeOutcome as PluginMarketplaceUpgradeOutcome; pub use codex_plugin::AppConnectorId; pub use codex_plugin::EffectiveSkillRoots; pub use codex_plugin::PluginCapabilitySummary; From ab97c9aaad7cb29a5d23c7485318e0ad6b20400c Mon Sep 17 00:00:00 2001 From: pakrym-oai Date: Thu, 16 Apr 2026 10:51:33 -0700 Subject: [PATCH 30/59] Refactor AGENTS.md discovery into AgentsMdManager (#18035) Encapsulate Agents MD processing a bit and drop user_instructions_path from config. --- codex-rs/app-server-client/src/lib.rs | 5 +- .../app-server/src/codex_message_processor.rs | 14 +- codex-rs/core/src/agents_md.rs | 367 ++++++++++++++++++ ...roject_doc_tests.rs => agents_md_tests.rs} | 92 +++-- codex-rs/core/src/codex.rs | 6 +- codex-rs/core/src/config/config_tests.rs | 36 +- codex-rs/core/src/config/mod.rs | 39 +- codex-rs/core/src/lib.rs | 9 +- codex-rs/core/src/project_doc.rs | 326 ---------------- codex-rs/tui/src/chatwidget.rs | 2 +- codex-rs/tui/src/chatwidget/slash_dispatch.rs | 4 +- .../src/chatwidget/tests/slash_commands.rs | 4 +- codex-rs/tui/src/status/helpers.rs | 14 +- 13 files changed, 464 insertions(+), 454 deletions(-) create mode 100644 codex-rs/core/src/agents_md.rs rename codex-rs/core/src/{project_doc_tests.rs => agents_md_tests.rs} (90%) delete mode 100644 codex-rs/core/src/project_doc.rs diff --git a/codex-rs/app-server-client/src/lib.rs b/codex-rs/app-server-client/src/lib.rs index 9818b2f61b..71a8784e0a 100644 --- a/codex-rs/app-server-client/src/lib.rs +++ b/codex-rs/app-server-client/src/lib.rs @@ -65,9 +65,9 @@ pub use crate::remote::RemoteAppServerConnectArgs; /// while legacy startup/config paths are migrated to RPCs. pub mod legacy_core { pub use codex_core::Cursor; - pub use codex_core::DEFAULT_PROJECT_DOC_FILENAME; + pub use codex_core::DEFAULT_AGENTS_MD_FILENAME; pub use codex_core::INTERACTIVE_SESSION_SOURCES; - pub use codex_core::LOCAL_PROJECT_DOC_FILENAME; + pub use codex_core::LOCAL_AGENTS_MD_FILENAME; pub use codex_core::McpManager; pub use codex_core::PLUGIN_TEXT_MENTION_SIGIL; pub use codex_core::RolloutRecorder; @@ -77,7 +77,6 @@ pub mod legacy_core { pub use codex_core::ThreadsPage; pub use codex_core::append_message_history_entry; pub use codex_core::check_execpolicy_for_warnings; - pub use codex_core::discover_project_doc_paths; pub use codex_core::find_thread_meta_by_name_str; pub use codex_core::find_thread_name_by_id; pub use codex_core::find_thread_names_by_ids; diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index 25892e40eb..410f22ece8 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -617,17 +617,9 @@ pub(crate) struct CodexMessageProcessorArgs { impl CodexMessageProcessor { async fn instruction_sources_from_config(config: &Config) -> Vec { - let mut paths: Vec = - config.user_instructions_path.iter().cloned().collect(); - match codex_core::discover_project_doc_paths(config, LOCAL_FS.as_ref()).await { - Ok(project_doc_paths) => { - paths.extend(project_doc_paths); - } - Err(err) => { - tracing::warn!(error = %err, "failed to discover project docs for thread response"); - } - } - paths + codex_core::AgentsMdManager::new(config) + .instruction_sources(LOCAL_FS.as_ref()) + .await } pub(crate) fn handle_config_mutation(&self) { diff --git a/codex-rs/core/src/agents_md.rs b/codex-rs/core/src/agents_md.rs new file mode 100644 index 0000000000..a1a883e839 --- /dev/null +++ b/codex-rs/core/src/agents_md.rs @@ -0,0 +1,367 @@ +//! AGENTS.md discovery and user instruction assembly. +//! +//! Project-level documentation is primarily stored in files named `AGENTS.md`. +//! Additional fallback filenames can be configured via `project_doc_fallback_filenames`. +//! We include the concatenation of all files found along the path from the +//! project root to the current working directory as follows: +//! +//! 1. Determine the project root by walking upwards from the current working +//! directory until a configured `project_root_markers` entry is found. +//! When `project_root_markers` is unset, the default marker list is used +//! (`.git`). If no marker is found, only the current working directory is +//! considered. An empty marker list disables parent traversal. +//! 2. Collect every `AGENTS.md` found from the project root down to the +//! current working directory (inclusive) and concatenate their contents in +//! that order. +//! 3. We do **not** walk past the project root. + +use crate::config::Config; +use crate::config_loader::ConfigLayerStackOrdering; +use crate::config_loader::default_project_root_markers; +use crate::config_loader::merge_toml_values; +use crate::config_loader::project_root_markers_from_config; +use codex_app_server_protocol::ConfigLayerSource; +use codex_exec_server::Environment; +use codex_exec_server::ExecutorFileSystem; +use codex_features::Feature; +use codex_utils_absolute_path::AbsolutePathBuf; +use dunce::canonicalize as normalize_path; +use std::io; +use toml::Value as TomlValue; +use tracing::error; + +pub(crate) const HIERARCHICAL_AGENTS_MESSAGE: &str = + include_str!("../hierarchical_agents_message.md"); + +/// Default filename scanned for AGENTS.md instructions. +pub const DEFAULT_AGENTS_MD_FILENAME: &str = "AGENTS.md"; +/// Preferred local override for AGENTS.md instructions. +pub const LOCAL_AGENTS_MD_FILENAME: &str = "AGENTS.override.md"; + +/// When both `Config::instructions` and AGENTS.md docs are present, they will +/// be concatenated with the following separator. +const AGENTS_MD_SEPARATOR: &str = "\n\n--- project-doc ---\n\n"; + +fn render_js_repl_instructions(config: &Config) -> Option { + if !config.features.enabled(Feature::JsRepl) { + return None; + } + + let mut section = String::from("## JavaScript REPL (Node)\n"); + section.push_str( + "- Use `js_repl` for Node-backed JavaScript with top-level await in a persistent kernel.\n", + ); + section.push_str("- `js_repl` is a freeform/custom tool. Direct `js_repl` calls must send raw JavaScript tool input (optionally with first-line `// codex-js-repl: timeout_ms=15000`). Do not wrap code in JSON (for example `{\"code\":\"...\"}`), quotes, or markdown code fences.\n"); + section.push_str( + "- Helpers: `codex.cwd`, `codex.homeDir`, `codex.tmpDir`, `codex.tool(name, args?)`, and `codex.emitImage(imageLike)`.\n", + ); + section.push_str("- `codex.tool` executes a normal tool call and resolves to the raw tool output object. Use it for shell and non-shell tools alike. Nested tool outputs stay inside JavaScript unless you emit them explicitly.\n"); + section.push_str("- `codex.emitImage(...)` adds one image to the outer `js_repl` function output each time you call it, so you can call it multiple times to emit multiple images. It accepts a data URL, a single `input_image` item, an object like `{ bytes, mimeType }`, or a raw tool response object with exactly one image and no text. It rejects mixed text-and-image content.\n"); + section.push_str("- `codex.tool(...)` and `codex.emitImage(...)` keep stable helper identities across cells. Saved references and persisted objects can reuse them in later cells, but async callbacks that fire after a cell finishes still fail because no exec is active.\n"); + section.push_str("- Request full-resolution image processing with `detail: \"original\"` only when the `view_image` tool schema includes a `detail` argument. The same availability applies to `codex.emitImage(...)`: if `view_image.detail` is present, you may also pass `detail: \"original\"` there. Use this when high-fidelity image perception or precise localization is needed, especially for CUA agents.\n"); + section.push_str("- Raw MCP image blocks can request the same behavior by returning `_meta: { \"codex/imageDetail\": \"original\" }` on the image content item.\n"); + section.push_str("- Example of sharing an in-memory Playwright screenshot: `await codex.emitImage({ bytes: await page.screenshot({ type: \"jpeg\", quality: 85 }), mimeType: \"image/jpeg\", detail: \"original\" })`.\n"); + section.push_str("- Example of sharing a local image tool result: `await codex.emitImage(codex.tool(\"view_image\", { path: \"/absolute/path\", detail: \"original\" }))`.\n"); + section.push_str("- When encoding an image to send with `codex.emitImage(...)` or `view_image`, prefer JPEG at about 85 quality when lossy compression is acceptable; use PNG when transparency or lossless detail matters. Smaller uploads are faster and less likely to hit size limits.\n"); + section.push_str("- Top-level bindings persist across cells. If a cell throws, prior bindings remain available and bindings that finished initializing before the throw often remain usable in later cells. For code you plan to reuse across cells, prefer declaring or assigning it in direct top-level statements before operations that might throw. If you hit `SyntaxError: Identifier 'x' has already been declared`, first reuse the existing binding, reassign a previously declared `let`, or pick a new descriptive name. Use `{ ... }` only for a short temporary block when you specifically need local scratch names; do not wrap an entire cell in block scope if you want those names reusable later. Reset the kernel with `js_repl_reset` only when you need a clean state.\n"); + section.push_str("- Top-level static import declarations (for example `import x from \"./file.js\"`) are currently unsupported in `js_repl`; use dynamic imports with `await import(\"pkg\")`, `await import(\"./file.js\")`, or `await import(\"/abs/path/file.mjs\")` instead. Imported local files must be ESM `.js`/`.mjs` files and run in the same REPL VM context. Bare package imports always resolve from REPL-global search roots (`CODEX_JS_REPL_NODE_MODULE_DIRS`, then cwd), not relative to the imported file location. Local files may statically import only other local relative/absolute/`file://` `.js`/`.mjs` files; package and builtin imports from local files must stay dynamic. `import.meta.resolve()` returns importable strings such as `file://...`, bare package names, and `node:...` specifiers. Local file modules reload between execs, while top-level bindings persist until `js_repl_reset`.\n"); + + if config.features.enabled(Feature::JsReplToolsOnly) { + section.push_str("- Do not call tools directly; use `js_repl` + `codex.tool(...)` for all tool calls, including shell commands.\n"); + section + .push_str("- MCP tools (if any) can also be called by name via `codex.tool(...)`.\n"); + } + + section.push_str("- Avoid direct access to `process.stdout` / `process.stderr` / `process.stdin`; it can corrupt the JSON line protocol. Use `console.log`, `codex.tool(...)`, and `codex.emitImage(...)`."); + + Some(section) +} + +/// Resolves AGENTS.md files into model-visible user instructions and source +/// paths. +pub struct AgentsMdManager<'a> { + config: &'a Config, +} + +pub(crate) struct LoadedAgentsMd { + pub(crate) contents: String, + pub(crate) path: AbsolutePathBuf, +} + +impl<'a> AgentsMdManager<'a> { + pub fn new(config: &'a Config) -> Self { + Self { config } + } + + pub(crate) fn load_global_instructions( + codex_dir: Option<&AbsolutePathBuf>, + ) -> Option { + let base = codex_dir?; + for candidate in [LOCAL_AGENTS_MD_FILENAME, DEFAULT_AGENTS_MD_FILENAME] { + let path = base.join(candidate); + if let Ok(contents) = std::fs::read_to_string(&path) { + let trimmed = contents.trim(); + if !trimmed.is_empty() { + return Some(LoadedAgentsMd { + contents: trimmed.to_string(), + path, + }); + } + } + } + None + } + + /// Combines configured user instructions and AGENTS.md content into a + /// single model-visible instruction string. + pub(crate) async fn user_instructions( + &self, + environment: Option<&Environment>, + ) -> Option { + let fs = environment?.get_filesystem(); + self.user_instructions_with_fs(fs.as_ref()).await + } + + pub(crate) async fn user_instructions_with_fs( + &self, + fs: &dyn ExecutorFileSystem, + ) -> Option { + let agents_md_docs = self.read_agents_md(fs).await; + + let mut output = String::new(); + + if let Some(instructions) = self.config.user_instructions.clone() { + output.push_str(&instructions); + } + + match agents_md_docs { + Ok(Some(docs)) => { + if !output.is_empty() { + output.push_str(AGENTS_MD_SEPARATOR); + } + output.push_str(&docs); + } + Ok(None) => {} + Err(e) => { + error!("error trying to find AGENTS.md docs: {e:#}"); + } + }; + + if let Some(js_repl_section) = render_js_repl_instructions(self.config) { + if !output.is_empty() { + output.push_str("\n\n"); + } + output.push_str(&js_repl_section); + } + + if self.config.features.enabled(Feature::ChildAgentsMd) { + if !output.is_empty() { + output.push_str("\n\n"); + } + output.push_str(HIERARCHICAL_AGENTS_MESSAGE); + } + + if !output.is_empty() { + Some(output) + } else { + None + } + } + + /// Returns all instruction source files included in the current config. + pub async fn instruction_sources(&self, fs: &dyn ExecutorFileSystem) -> Vec { + let mut paths = Self::load_global_instructions(Some(&self.config.codex_home)) + .map(|loaded| vec![loaded.path]) + .unwrap_or_default(); + match self.agents_md_paths(fs).await { + Ok(agents_md_paths) => paths.extend(agents_md_paths), + Err(err) => { + tracing::warn!(error = %err, "failed to discover AGENTS.md docs for instruction sources"); + } + } + paths + } + + /// Attempt to locate and load AGENTS.md documentation. + /// + /// On success returns `Ok(Some(contents))` where `contents` is the + /// concatenation of all discovered docs. If no documentation file is found + /// the function returns `Ok(None)`. Unexpected I/O failures bubble up as + /// `Err` so callers can decide how to handle them. + async fn read_agents_md(&self, fs: &dyn ExecutorFileSystem) -> io::Result> { + let max_total = self.config.project_doc_max_bytes; + + if max_total == 0 { + return Ok(None); + } + + let paths = self.agents_md_paths(fs).await?; + if paths.is_empty() { + return Ok(None); + } + + let mut remaining: u64 = max_total as u64; + let mut parts: Vec = Vec::new(); + + for p in paths { + if remaining == 0 { + break; + } + + match fs.get_metadata(&p, /*sandbox*/ None).await { + Ok(metadata) if !metadata.is_file => continue, + Ok(_) => {} + Err(err) if err.kind() == io::ErrorKind::NotFound => continue, + Err(err) => return Err(err), + } + + let mut data = match fs.read_file(&p, /*sandbox*/ None).await { + Ok(data) => data, + Err(err) if err.kind() == io::ErrorKind::NotFound => continue, + Err(err) => return Err(err), + }; + let size = data.len() as u64; + if size > remaining { + data.truncate(remaining as usize); + } + + if size > remaining { + tracing::warn!( + "Project doc `{}` exceeds remaining budget ({} bytes) - truncating.", + p.display(), + remaining, + ); + } + + let text = String::from_utf8_lossy(&data).to_string(); + if !text.trim().is_empty() { + parts.push(text); + remaining = remaining.saturating_sub(data.len() as u64); + } + } + + if parts.is_empty() { + Ok(None) + } else { + Ok(Some(parts.join("\n\n"))) + } + } + + /// Discover the list of AGENTS.md files using the same search rules as + /// `read_agents_md`, but return the file paths instead of concatenated + /// contents. The list is ordered from project root to the current working + /// directory (inclusive). Symlinks are allowed. When `project_doc_max_bytes` + /// is zero, returns an empty list. + async fn agents_md_paths( + &self, + fs: &dyn ExecutorFileSystem, + ) -> io::Result> { + if self.config.project_doc_max_bytes == 0 { + return Ok(Vec::new()); + } + + let mut dir = self.config.cwd.clone(); + if let Ok(canon) = normalize_path(&dir) { + dir = AbsolutePathBuf::try_from(canon)?; + } + + let mut merged = TomlValue::Table(toml::map::Map::new()); + for layer in self.config.config_layer_stack.get_layers( + ConfigLayerStackOrdering::LowestPrecedenceFirst, + /*include_disabled*/ false, + ) { + if matches!(layer.name, ConfigLayerSource::Project { .. }) { + continue; + } + merge_toml_values(&mut merged, &layer.config); + } + let project_root_markers = match project_root_markers_from_config(&merged) { + Ok(Some(markers)) => markers, + Ok(None) => default_project_root_markers(), + Err(err) => { + tracing::warn!("invalid project_root_markers: {err}"); + default_project_root_markers() + } + }; + let mut project_root = None; + if !project_root_markers.is_empty() { + for ancestor in dir.ancestors() { + for marker in &project_root_markers { + let marker_path = ancestor.join(marker); + let marker_exists = match fs.get_metadata(&marker_path, /*sandbox*/ None).await + { + Ok(_) => true, + Err(err) if err.kind() == io::ErrorKind::NotFound => false, + Err(err) => return Err(err), + }; + if marker_exists { + project_root = Some(ancestor.clone()); + break; + } + } + if project_root.is_some() { + break; + } + } + } + + let search_dirs: Vec = if let Some(root) = project_root { + let mut dirs = Vec::new(); + let mut cursor = dir.clone(); + loop { + dirs.push(cursor.clone()); + if cursor == root { + break; + } + let Some(parent) = cursor.parent() else { + break; + }; + cursor = parent; + } + dirs.reverse(); + dirs + } else { + vec![dir] + }; + + let mut found: Vec = Vec::new(); + let candidate_filenames = self.candidate_filenames(); + for d in search_dirs { + for name in &candidate_filenames { + let candidate = d.join(name); + match fs.get_metadata(&candidate, /*sandbox*/ None).await { + Ok(md) if md.is_file => { + found.push(candidate); + break; + } + Ok(_) => {} + Err(err) if err.kind() == io::ErrorKind::NotFound => continue, + Err(err) => return Err(err), + } + } + } + + Ok(found) + } + + fn candidate_filenames(&self) -> Vec<&str> { + let mut names: Vec<&str> = + Vec::with_capacity(2 + self.config.project_doc_fallback_filenames.len()); + names.push(LOCAL_AGENTS_MD_FILENAME); + names.push(DEFAULT_AGENTS_MD_FILENAME); + for candidate in &self.config.project_doc_fallback_filenames { + let candidate = candidate.as_str(); + if candidate.is_empty() { + continue; + } + if !names.contains(&candidate) { + names.push(candidate); + } + } + names + } +} + +#[cfg(test)] +#[path = "agents_md_tests.rs"] +mod tests; diff --git a/codex-rs/core/src/project_doc_tests.rs b/codex-rs/core/src/agents_md_tests.rs similarity index 90% rename from codex-rs/core/src/project_doc_tests.rs rename to codex-rs/core/src/agents_md_tests.rs index 31f73805f7..012724b43e 100644 --- a/codex-rs/core/src/project_doc_tests.rs +++ b/codex-rs/core/src/agents_md_tests.rs @@ -11,11 +11,15 @@ use std::path::PathBuf; use tempfile::TempDir; async fn get_user_instructions(config: &Config) -> Option { - super::get_user_instructions_with_fs(config, LOCAL_FS.as_ref()).await + AgentsMdManager::new(config) + .user_instructions_with_fs(LOCAL_FS.as_ref()) + .await } -async fn discover_project_doc_paths(config: &Config) -> std::io::Result> { - super::discover_project_doc_paths(config, LOCAL_FS.as_ref()).await +async fn agents_md_paths(config: &Config) -> std::io::Result> { + AgentsMdManager::new(config) + .agents_md_paths(LOCAL_FS.as_ref()) + .await } /// Helper that returns a `Config` pointing at `root` and using `limit` as @@ -101,7 +105,9 @@ async fn no_environment_returns_none() { let tmp = tempfile::tempdir().expect("tempdir"); let config = make_config(&tmp, /*limit*/ 4096, Some("user instructions")).await; - let res = super::get_user_instructions(&config, /*environment*/ None).await; + let res = AgentsMdManager::new(&config) + .user_instructions(/*environment*/ None) + .await; assert_eq!(res, None); } @@ -187,10 +193,9 @@ async fn zero_byte_limit_disables_discovery() { let tmp = tempfile::tempdir().expect("tempdir"); fs::write(tmp.path().join("AGENTS.md"), "something").unwrap(); - let discovery = - discover_project_doc_paths(&make_config(&tmp, /*limit*/ 0, /*instructions*/ None).await) - .await - .expect("discover paths"); + let discovery = agents_md_paths(&make_config(&tmp, /*limit*/ 0, /*instructions*/ None).await) + .await + .expect("discover paths"); assert_eq!(discovery, Vec::::new()); } @@ -228,10 +233,10 @@ async fn js_repl_tools_only_instructions_are_feature_gated() { assert_eq!(res, expected); } -/// When both system instructions *and* a project doc are present the two +/// When both system instructions and AGENTS.md docs are present the two /// should be concatenated with the separator. #[tokio::test] -async fn merges_existing_instructions_with_project_doc() { +async fn merges_existing_instructions_with_agents_md() { let tmp = tempfile::tempdir().expect("tempdir"); fs::write(tmp.path().join("AGENTS.md"), "proj doc").unwrap(); @@ -241,12 +246,12 @@ async fn merges_existing_instructions_with_project_doc() { .await .expect("should produce a combined instruction string"); - let expected = format!("{INSTRUCTIONS}{PROJECT_DOC_SEPARATOR}{}", "proj doc"); + let expected = format!("{INSTRUCTIONS}{AGENTS_MD_SEPARATOR}{}", "proj doc"); assert_eq!(res, expected); } -/// If there are existing system instructions but the project doc is +/// If there are existing system instructions but AGENTS.md docs are /// missing we expect the original instructions to be returned unchanged. #[tokio::test] async fn keeps_existing_instructions_when_doc_missing() { @@ -307,9 +312,7 @@ async fn project_root_markers_are_honored_for_agents_discovery() { .await; cfg.cwd = nested.abs(); - let discovery = discover_project_doc_paths(&cfg) - .await - .expect("discover paths"); + let discovery = agents_md_paths(&cfg).await.expect("discover paths"); let expected_parent = AbsolutePathBuf::try_from( dunce::canonicalize(root.path().join("AGENTS.md")).expect("canonical parent doc path"), ) @@ -326,12 +329,33 @@ async fn project_root_markers_are_honored_for_agents_discovery() { assert_eq!(res, "parent doc\n\nchild doc"); } +#[tokio::test] +async fn instruction_sources_include_global_before_agents_md_docs() { + let tmp = tempfile::tempdir().expect("tempdir"); + fs::write(tmp.path().join("AGENTS.md"), "project doc").unwrap(); + + let cfg = make_config(&tmp, /*limit*/ 4096, Some("global doc")).await; + let global_agents = cfg.codex_home.join(DEFAULT_AGENTS_MD_FILENAME); + fs::create_dir_all(&cfg.codex_home).unwrap(); + fs::write(&global_agents, "global doc").unwrap(); + + let sources = AgentsMdManager::new(&cfg) + .instruction_sources(LOCAL_FS.as_ref()) + .await; + let project_agents = AbsolutePathBuf::try_from( + dunce::canonicalize(cfg.cwd.join("AGENTS.md")).expect("canonical project doc path"), + ) + .expect("absolute project doc path"); + + assert_eq!(sources, vec![global_agents, project_agents]); +} + /// AGENTS.override.md is preferred over AGENTS.md when both are present. #[tokio::test] async fn agents_local_md_preferred() { let tmp = tempfile::tempdir().expect("tempdir"); - fs::write(tmp.path().join(DEFAULT_PROJECT_DOC_FILENAME), "versioned").unwrap(); - fs::write(tmp.path().join(LOCAL_PROJECT_DOC_FILENAME), "local").unwrap(); + fs::write(tmp.path().join(DEFAULT_AGENTS_MD_FILENAME), "versioned").unwrap(); + fs::write(tmp.path().join(LOCAL_AGENTS_MD_FILENAME), "local").unwrap(); let cfg = make_config(&tmp, /*limit*/ 4096, /*instructions*/ None).await; @@ -341,13 +365,11 @@ async fn agents_local_md_preferred() { assert_eq!(res, "local"); - let discovery = discover_project_doc_paths(&cfg) - .await - .expect("discover paths"); + let discovery = agents_md_paths(&cfg).await.expect("discover paths"); assert_eq!(discovery.len(), 1); assert_eq!( discovery[0].file_name().unwrap().to_string_lossy(), - LOCAL_PROJECT_DOC_FILENAME + LOCAL_AGENTS_MD_FILENAME ); } @@ -393,16 +415,14 @@ async fn agents_md_preferred_over_fallbacks() { assert_eq!(res, "primary"); - let discovery = discover_project_doc_paths(&cfg) - .await - .expect("discover paths"); + let discovery = agents_md_paths(&cfg).await.expect("discover paths"); assert_eq!(discovery.len(), 1); assert!( discovery[0] .file_name() .unwrap() .to_string_lossy() - .eq(DEFAULT_PROJECT_DOC_FILENAME) + .eq(DEFAULT_AGENTS_MD_FILENAME) ); } @@ -416,9 +436,7 @@ async fn agents_md_directory_is_ignored() { let res = get_user_instructions(&cfg).await; assert_eq!(res, None); - let discovery = discover_project_doc_paths(&cfg) - .await - .expect("discover paths"); + let discovery = agents_md_paths(&cfg).await.expect("discover paths"); assert_eq!(discovery, Vec::::new()); } @@ -441,17 +459,15 @@ async fn agents_md_special_file_is_ignored() { let res = get_user_instructions(&cfg).await; assert_eq!(res, None); - let discovery = discover_project_doc_paths(&cfg) - .await - .expect("discover paths"); + let discovery = agents_md_paths(&cfg).await.expect("discover paths"); assert_eq!(discovery, Vec::::new()); } #[tokio::test] async fn override_directory_falls_back_to_agents_md_file() { let tmp = tempfile::tempdir().expect("tempdir"); - fs::create_dir(tmp.path().join(LOCAL_PROJECT_DOC_FILENAME)).unwrap(); - fs::write(tmp.path().join(DEFAULT_PROJECT_DOC_FILENAME), "primary").unwrap(); + fs::create_dir(tmp.path().join(LOCAL_AGENTS_MD_FILENAME)).unwrap(); + fs::write(tmp.path().join(DEFAULT_AGENTS_MD_FILENAME), "primary").unwrap(); let cfg = make_config(&tmp, /*limit*/ 4096, /*instructions*/ None).await; @@ -460,21 +476,19 @@ async fn override_directory_falls_back_to_agents_md_file() { .expect("AGENTS.md should be used when override is a directory"); assert_eq!(res, "primary"); - let discovery = discover_project_doc_paths(&cfg) - .await - .expect("discover paths"); + let discovery = agents_md_paths(&cfg).await.expect("discover paths"); assert_eq!(discovery.len(), 1); assert_eq!( discovery[0] .file_name() .expect("file name") .to_string_lossy(), - DEFAULT_PROJECT_DOC_FILENAME + DEFAULT_AGENTS_MD_FILENAME ); } #[tokio::test] -async fn skills_are_not_appended_to_project_doc() { +async fn skills_are_not_appended_to_agents_md() { let tmp = tempfile::tempdir().expect("tempdir"); fs::write(tmp.path().join("AGENTS.md"), "base doc").unwrap(); @@ -504,7 +518,7 @@ async fn apps_feature_does_not_emit_user_instructions_by_itself() { } #[tokio::test] -async fn apps_feature_does_not_append_to_project_doc_user_instructions() { +async fn apps_feature_does_not_append_to_agents_md_user_instructions() { let tmp = tempfile::tempdir().expect("tempdir"); fs::write(tmp.path().join("AGENTS.md"), "base doc").unwrap(); diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 5f821eb6b4..50a4c25abd 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -265,6 +265,7 @@ use crate::SkillInjections; use crate::SkillLoadOutcome; use crate::SkillMetadata; use crate::SkillsManager; +use crate::agents_md::AgentsMdManager; use crate::build_skill_injections; use crate::collect_env_var_dependencies; use crate::collect_explicit_skill_mentions; @@ -293,7 +294,6 @@ use crate::network_policy_decision::execpolicy_network_rule_amendment; use crate::plugins::PluginsManager; use crate::plugins::build_plugin_injections; use crate::plugins::render_plugins_section; -use crate::project_doc::get_user_instructions; use crate::resolve_skill_dependencies_for_turn; use crate::rollout::RolloutRecorder; use crate::rollout::RolloutRecorderParams; @@ -556,7 +556,9 @@ impl Codex { config.startup_warnings.push(message); } - let user_instructions = get_user_instructions(&config, environment.as_deref()).await; + let user_instructions = AgentsMdManager::new(&config) + .user_instructions(environment.as_deref()) + .await; let exec_policy = if crate::guardian::is_guardian_reviewer_source(&session_source) { // Guardian review should rely on the built-in shell safety checks, diff --git a/codex-rs/core/src/config/config_tests.rs b/codex-rs/core/src/config/config_tests.rs index a7a5e88352..0001439816 100644 --- a/codex-rs/core/src/config/config_tests.rs +++ b/codex-rs/core/src/config/config_tests.rs @@ -1,3 +1,5 @@ +use crate::agents_md::DEFAULT_AGENTS_MD_FILENAME; +use crate::agents_md::LOCAL_AGENTS_MD_FILENAME; use crate::config::edit::ConfigEdit; use crate::config::edit::ConfigEditsBuilder; use crate::config::edit::apply_blocking; @@ -138,10 +140,12 @@ async fn load_config_normalizes_relative_cwd_override() -> std::io::Result<()> { } #[tokio::test] -async fn load_config_records_global_agents_path() -> std::io::Result<()> { +async fn load_config_loads_global_agents_instructions() -> std::io::Result<()> { let codex_home = tempdir()?; - let global_agents_path = codex_home.path().join(DEFAULT_PROJECT_DOC_FILENAME); - std::fs::write(&global_agents_path, "\n global instructions \n")?; + std::fs::write( + codex_home.path().join(DEFAULT_AGENTS_MD_FILENAME), + "\n global instructions \n", + )?; let config = Config::load_from_base_config_with_overrides( ConfigToml::default(), @@ -154,21 +158,17 @@ async fn load_config_records_global_agents_path() -> std::io::Result<()> { config.user_instructions.as_deref(), Some("global instructions") ); - assert_eq!( - config.user_instructions_path.as_deref(), - Some(global_agents_path.as_path()) - ); Ok(()) } #[tokio::test] -async fn load_config_records_preferred_global_agents_override_path() -> std::io::Result<()> { +async fn load_config_prefers_global_agents_override_instructions() -> std::io::Result<()> { let codex_home = tempdir()?; std::fs::write( - codex_home.path().join(DEFAULT_PROJECT_DOC_FILENAME), + codex_home.path().join(DEFAULT_AGENTS_MD_FILENAME), "global instructions", )?; - let global_agents_override_path = codex_home.path().join(LOCAL_PROJECT_DOC_FILENAME); + let global_agents_override_path = codex_home.path().join(LOCAL_AGENTS_MD_FILENAME); std::fs::write(&global_agents_override_path, "local override instructions")?; let config = Config::load_from_base_config_with_overrides( @@ -182,10 +182,6 @@ async fn load_config_records_preferred_global_agents_override_path() -> std::io: config.user_instructions.as_deref(), Some("local override instructions") ); - assert_eq!( - config.user_instructions_path.as_deref(), - Some(global_agents_override_path.as_path()) - ); Ok(()) } @@ -4721,7 +4717,6 @@ async fn test_precedence_fixture_with_o3_profile() -> std::io::Result<()> { approvals_reviewer: ApprovalsReviewer::User, enforce_residency: Constrained::allow_any(/*initial_value*/ None), user_instructions: None, - user_instructions_path: None, notify: None, cwd: fixture.cwd(), cli_auth_credentials_store_mode: Default::default(), @@ -4733,7 +4728,7 @@ async fn test_precedence_fixture_with_o3_profile() -> std::io::Result<()> { mcp_oauth_callback_port: None, mcp_oauth_callback_url: None, model_providers: fixture.model_provider_map.clone(), - project_doc_max_bytes: PROJECT_DOC_MAX_BYTES, + project_doc_max_bytes: AGENTS_MD_MAX_BYTES, project_doc_fallback_filenames: Vec::new(), tool_output_token_limit: None, agent_max_threads: DEFAULT_AGENT_MAX_THREADS, @@ -4872,7 +4867,6 @@ async fn test_precedence_fixture_with_gpt3_profile() -> std::io::Result<()> { approvals_reviewer: ApprovalsReviewer::User, enforce_residency: Constrained::allow_any(/*initial_value*/ None), user_instructions: None, - user_instructions_path: None, notify: None, cwd: fixture.cwd(), cli_auth_credentials_store_mode: Default::default(), @@ -4884,7 +4878,7 @@ async fn test_precedence_fixture_with_gpt3_profile() -> std::io::Result<()> { mcp_oauth_callback_port: None, mcp_oauth_callback_url: None, model_providers: fixture.model_provider_map.clone(), - project_doc_max_bytes: PROJECT_DOC_MAX_BYTES, + project_doc_max_bytes: AGENTS_MD_MAX_BYTES, project_doc_fallback_filenames: Vec::new(), tool_output_token_limit: None, agent_max_threads: DEFAULT_AGENT_MAX_THREADS, @@ -5021,7 +5015,6 @@ async fn test_precedence_fixture_with_zdr_profile() -> std::io::Result<()> { approvals_reviewer: ApprovalsReviewer::User, enforce_residency: Constrained::allow_any(/*initial_value*/ None), user_instructions: None, - user_instructions_path: None, notify: None, cwd: fixture.cwd(), cli_auth_credentials_store_mode: Default::default(), @@ -5033,7 +5026,7 @@ async fn test_precedence_fixture_with_zdr_profile() -> std::io::Result<()> { mcp_oauth_callback_port: None, mcp_oauth_callback_url: None, model_providers: fixture.model_provider_map.clone(), - project_doc_max_bytes: PROJECT_DOC_MAX_BYTES, + project_doc_max_bytes: AGENTS_MD_MAX_BYTES, project_doc_fallback_filenames: Vec::new(), tool_output_token_limit: None, agent_max_threads: DEFAULT_AGENT_MAX_THREADS, @@ -5155,7 +5148,6 @@ async fn test_precedence_fixture_with_gpt5_profile() -> std::io::Result<()> { approvals_reviewer: ApprovalsReviewer::User, enforce_residency: Constrained::allow_any(/*initial_value*/ None), user_instructions: None, - user_instructions_path: None, notify: None, cwd: fixture.cwd(), cli_auth_credentials_store_mode: Default::default(), @@ -5167,7 +5159,7 @@ async fn test_precedence_fixture_with_gpt5_profile() -> std::io::Result<()> { mcp_oauth_callback_port: None, mcp_oauth_callback_url: None, model_providers: fixture.model_provider_map.clone(), - project_doc_max_bytes: PROJECT_DOC_MAX_BYTES, + project_doc_max_bytes: AGENTS_MD_MAX_BYTES, project_doc_fallback_filenames: Vec::new(), tool_output_token_limit: None, agent_max_threads: DEFAULT_AGENT_MAX_THREADS, diff --git a/codex-rs/core/src/config/mod.rs b/codex-rs/core/src/config/mod.rs index 87e888e46e..c4217bdf2a 100644 --- a/codex-rs/core/src/config/mod.rs +++ b/codex-rs/core/src/config/mod.rs @@ -1,3 +1,4 @@ +use crate::agents_md::AgentsMdManager; use crate::config::edit::ConfigEdit; use crate::config::edit::ConfigEditsBuilder; use crate::config_loader::CloudRequirementsLoader; @@ -15,8 +16,6 @@ use crate::config_loader::load_config_layers_state; use crate::config_loader::project_trust_key; use crate::memories::memory_root; use crate::path_utils::normalize_for_native_workdir; -use crate::project_doc::DEFAULT_PROJECT_DOC_FILENAME; -use crate::project_doc::LOCAL_PROJECT_DOC_FILENAME; use crate::unified_exec::DEFAULT_MAX_BACKGROUND_TERMINAL_TIMEOUT_MS; use crate::unified_exec::MIN_EMPTY_YIELD_TIME_MS; use crate::windows_sandbox::WindowsSandboxLevelExt; @@ -120,7 +119,7 @@ pub use codex_git_utils::GhostSnapshotConfig; /// Maximum number of bytes of the documentation that will be embedded. Larger /// files are *silently truncated* to this size so we do not take up too much of /// the context window. -pub(crate) const PROJECT_DOC_MAX_BYTES: usize = 32 * 1024; // 32 KiB +pub(crate) const AGENTS_MD_MAX_BYTES: usize = 32 * 1024; // 32 KiB pub(crate) const DEFAULT_AGENT_MAX_THREADS: Option = Some(6); pub(crate) const DEFAULT_AGENT_MAX_DEPTH: i32 = 1; pub(crate) const DEFAULT_AGENT_JOB_MAX_RUNTIME_SECONDS: Option = None; @@ -272,9 +271,6 @@ pub struct Config { /// User-provided instructions from AGENTS.md. pub user_instructions: Option, - /// Path to the global AGENTS file loaded into `user_instructions`. - pub user_instructions_path: Option, - /// Base instructions override. pub base_instructions: Option, @@ -1451,10 +1447,8 @@ impl Config { network: network_requirements, } = config_layer_stack.requirements().clone(); - let (user_instructions, user_instructions_path) = - Self::load_instructions(Some(&codex_home)) - .map(|loaded| (Some(loaded.contents), Some(loaded.path))) - .unwrap_or((None, None)); + let user_instructions = AgentsMdManager::load_global_instructions(Some(&codex_home)) + .map(|loaded| loaded.contents); let mut startup_warnings = Vec::new(); // Destructure ConfigOverrides fully to ensure all overrides are applied. @@ -2046,7 +2040,6 @@ impl Config { enforce_residency: enforce_residency.value, notify: cfg.notify, user_instructions, - user_instructions_path, base_instructions, personality, developer_instructions, @@ -2071,7 +2064,7 @@ impl Config { mcp_oauth_callback_port: cfg.mcp_oauth_callback_port, mcp_oauth_callback_url: cfg.mcp_oauth_callback_url.clone(), model_providers, - project_doc_max_bytes: cfg.project_doc_max_bytes.unwrap_or(PROJECT_DOC_MAX_BYTES), + project_doc_max_bytes: cfg.project_doc_max_bytes.unwrap_or(AGENTS_MD_MAX_BYTES), project_doc_fallback_filenames: cfg .project_doc_fallback_filenames .unwrap_or_default() @@ -2222,23 +2215,6 @@ impl Config { .await } - fn load_instructions(codex_dir: Option<&AbsolutePathBuf>) -> Option { - let base = codex_dir?; - for candidate in [LOCAL_PROJECT_DOC_FILENAME, DEFAULT_PROJECT_DOC_FILENAME] { - let path = base.join(candidate); - if let Ok(contents) = std::fs::read_to_string(&path) { - let trimmed = contents.trim(); - if !trimmed.is_empty() { - return Some(LoadedUserInstructions { - contents: trimmed.to_string(), - path, - }); - } - } - } - None - } - /// If `path` is `Some`, attempts to read the file at the given path and /// returns its contents as a trimmed `String`. If the file is empty, or /// is `Some` but cannot be read, returns an `Err`. @@ -2310,11 +2286,6 @@ impl Config { } } -struct LoadedUserInstructions { - contents: String, - path: AbsolutePathBuf, -} - pub(crate) fn uses_deprecated_instructions_file(config_layer_stack: &ConfigLayerStack) -> bool { config_layer_stack .layers_high_to_low() diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index 17c4e63b7c..50777ad157 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -130,11 +130,10 @@ pub type ConversationManager = ThreadManager; pub type NewConversation = NewThread; #[deprecated(note = "use CodexThread")] pub type CodexConversation = CodexThread; -pub(crate) mod project_doc; -pub use project_doc::DEFAULT_PROJECT_DOC_FILENAME; -pub use project_doc::LOCAL_PROJECT_DOC_FILENAME; -pub use project_doc::discover_project_doc_paths; -pub use project_doc::read_project_docs; +pub(crate) mod agents_md; +pub use agents_md::AgentsMdManager; +pub use agents_md::DEFAULT_AGENTS_MD_FILENAME; +pub use agents_md::LOCAL_AGENTS_MD_FILENAME; mod rollout; pub(crate) mod safety; mod session_rollout_init_error; diff --git a/codex-rs/core/src/project_doc.rs b/codex-rs/core/src/project_doc.rs deleted file mode 100644 index c4f70a6f55..0000000000 --- a/codex-rs/core/src/project_doc.rs +++ /dev/null @@ -1,326 +0,0 @@ -//! Project-level documentation discovery. -//! -//! Project-level documentation is primarily stored in files named `AGENTS.md`. -//! Additional fallback filenames can be configured via `project_doc_fallback_filenames`. -//! We include the concatenation of all files found along the path from the -//! project root to the current working directory as follows: -//! -//! 1. Determine the project root by walking upwards from the current working -//! directory until a configured `project_root_markers` entry is found. -//! When `project_root_markers` is unset, the default marker list is used -//! (`.git`). If no marker is found, only the current working directory is -//! considered. An empty marker list disables parent traversal. -//! 2. Collect every `AGENTS.md` found from the project root down to the -//! current working directory (inclusive) and concatenate their contents in -//! that order. -//! 3. We do **not** walk past the project root. - -use crate::config::Config; -use crate::config_loader::ConfigLayerStackOrdering; -use crate::config_loader::default_project_root_markers; -use crate::config_loader::merge_toml_values; -use crate::config_loader::project_root_markers_from_config; -use codex_app_server_protocol::ConfigLayerSource; -use codex_exec_server::Environment; -use codex_exec_server::ExecutorFileSystem; -use codex_features::Feature; -use codex_utils_absolute_path::AbsolutePathBuf; -use dunce::canonicalize as normalize_path; -use std::io; -use toml::Value as TomlValue; -use tracing::error; - -pub(crate) const HIERARCHICAL_AGENTS_MESSAGE: &str = - include_str!("../hierarchical_agents_message.md"); - -/// Default filename scanned for project-level docs. -pub const DEFAULT_PROJECT_DOC_FILENAME: &str = "AGENTS.md"; -/// Preferred local override for project-level docs. -pub const LOCAL_PROJECT_DOC_FILENAME: &str = "AGENTS.override.md"; - -/// When both `Config::instructions` and the project doc are present, they will -/// be concatenated with the following separator. -const PROJECT_DOC_SEPARATOR: &str = "\n\n--- project-doc ---\n\n"; - -fn render_js_repl_instructions(config: &Config) -> Option { - if !config.features.enabled(Feature::JsRepl) { - return None; - } - - let mut section = String::from("## JavaScript REPL (Node)\n"); - section.push_str( - "- Use `js_repl` for Node-backed JavaScript with top-level await in a persistent kernel.\n", - ); - section.push_str("- `js_repl` is a freeform/custom tool. Direct `js_repl` calls must send raw JavaScript tool input (optionally with first-line `// codex-js-repl: timeout_ms=15000`). Do not wrap code in JSON (for example `{\"code\":\"...\"}`), quotes, or markdown code fences.\n"); - section.push_str( - "- Helpers: `codex.cwd`, `codex.homeDir`, `codex.tmpDir`, `codex.tool(name, args?)`, and `codex.emitImage(imageLike)`.\n", - ); - section.push_str("- `codex.tool` executes a normal tool call and resolves to the raw tool output object. Use it for shell and non-shell tools alike. Nested tool outputs stay inside JavaScript unless you emit them explicitly.\n"); - section.push_str("- `codex.emitImage(...)` adds one image to the outer `js_repl` function output each time you call it, so you can call it multiple times to emit multiple images. It accepts a data URL, a single `input_image` item, an object like `{ bytes, mimeType }`, or a raw tool response object with exactly one image and no text. It rejects mixed text-and-image content.\n"); - section.push_str("- `codex.tool(...)` and `codex.emitImage(...)` keep stable helper identities across cells. Saved references and persisted objects can reuse them in later cells, but async callbacks that fire after a cell finishes still fail because no exec is active.\n"); - section.push_str("- Request full-resolution image processing with `detail: \"original\"` only when the `view_image` tool schema includes a `detail` argument. The same availability applies to `codex.emitImage(...)`: if `view_image.detail` is present, you may also pass `detail: \"original\"` there. Use this when high-fidelity image perception or precise localization is needed, especially for CUA agents.\n"); - section.push_str("- Raw MCP image blocks can request the same behavior by returning `_meta: { \"codex/imageDetail\": \"original\" }` on the image content item.\n"); - section.push_str("- Example of sharing an in-memory Playwright screenshot: `await codex.emitImage({ bytes: await page.screenshot({ type: \"jpeg\", quality: 85 }), mimeType: \"image/jpeg\", detail: \"original\" })`.\n"); - section.push_str("- Example of sharing a local image tool result: `await codex.emitImage(codex.tool(\"view_image\", { path: \"/absolute/path\", detail: \"original\" }))`.\n"); - section.push_str("- When encoding an image to send with `codex.emitImage(...)` or `view_image`, prefer JPEG at about 85 quality when lossy compression is acceptable; use PNG when transparency or lossless detail matters. Smaller uploads are faster and less likely to hit size limits.\n"); - section.push_str("- Top-level bindings persist across cells. If a cell throws, prior bindings remain available and bindings that finished initializing before the throw often remain usable in later cells. For code you plan to reuse across cells, prefer declaring or assigning it in direct top-level statements before operations that might throw. If you hit `SyntaxError: Identifier 'x' has already been declared`, first reuse the existing binding, reassign a previously declared `let`, or pick a new descriptive name. Use `{ ... }` only for a short temporary block when you specifically need local scratch names; do not wrap an entire cell in block scope if you want those names reusable later. Reset the kernel with `js_repl_reset` only when you need a clean state.\n"); - section.push_str("- Top-level static import declarations (for example `import x from \"./file.js\"`) are currently unsupported in `js_repl`; use dynamic imports with `await import(\"pkg\")`, `await import(\"./file.js\")`, or `await import(\"/abs/path/file.mjs\")` instead. Imported local files must be ESM `.js`/`.mjs` files and run in the same REPL VM context. Bare package imports always resolve from REPL-global search roots (`CODEX_JS_REPL_NODE_MODULE_DIRS`, then cwd), not relative to the imported file location. Local files may statically import only other local relative/absolute/`file://` `.js`/`.mjs` files; package and builtin imports from local files must stay dynamic. `import.meta.resolve()` returns importable strings such as `file://...`, bare package names, and `node:...` specifiers. Local file modules reload between execs, while top-level bindings persist until `js_repl_reset`.\n"); - - if config.features.enabled(Feature::JsReplToolsOnly) { - section.push_str("- Do not call tools directly; use `js_repl` + `codex.tool(...)` for all tool calls, including shell commands.\n"); - section - .push_str("- MCP tools (if any) can also be called by name via `codex.tool(...)`.\n"); - } - - section.push_str("- Avoid direct access to `process.stdout` / `process.stderr` / `process.stdin`; it can corrupt the JSON line protocol. Use `console.log`, `codex.tool(...)`, and `codex.emitImage(...)`."); - - Some(section) -} - -/// Combines `Config::instructions` and `AGENTS.md` (if present) into a single -/// string of instructions. -pub(crate) async fn get_user_instructions( - config: &Config, - environment: Option<&Environment>, -) -> Option { - let fs = environment?.get_filesystem(); - get_user_instructions_with_fs(config, fs.as_ref()).await -} - -pub(crate) async fn get_user_instructions_with_fs( - config: &Config, - fs: &dyn ExecutorFileSystem, -) -> Option { - let project_docs = read_project_docs_with_fs(config, fs).await; - - let mut output = String::new(); - - if let Some(instructions) = config.user_instructions.clone() { - output.push_str(&instructions); - } - - match project_docs { - Ok(Some(docs)) => { - if !output.is_empty() { - output.push_str(PROJECT_DOC_SEPARATOR); - } - output.push_str(&docs); - } - Ok(None) => {} - Err(e) => { - error!("error trying to find project doc: {e:#}"); - } - }; - - if let Some(js_repl_section) = render_js_repl_instructions(config) { - if !output.is_empty() { - output.push_str("\n\n"); - } - output.push_str(&js_repl_section); - } - - if config.features.enabled(Feature::ChildAgentsMd) { - if !output.is_empty() { - output.push_str("\n\n"); - } - output.push_str(HIERARCHICAL_AGENTS_MESSAGE); - } - - if !output.is_empty() { - Some(output) - } else { - None - } -} - -/// Attempt to locate and load the project documentation. -/// -/// On success returns `Ok(Some(contents))` where `contents` is the -/// concatenation of all discovered docs. If no documentation file is found the -/// function returns `Ok(None)`. Unexpected I/O failures bubble up as `Err` so -/// callers can decide how to handle them. -pub async fn read_project_docs( - config: &Config, - environment: &Environment, -) -> io::Result> { - let fs = environment.get_filesystem(); - read_project_docs_with_fs(config, fs.as_ref()).await -} - -async fn read_project_docs_with_fs( - config: &Config, - fs: &dyn ExecutorFileSystem, -) -> io::Result> { - let max_total = config.project_doc_max_bytes; - - if max_total == 0 { - return Ok(None); - } - - let paths = discover_project_doc_paths(config, fs).await?; - if paths.is_empty() { - return Ok(None); - } - - let mut remaining: u64 = max_total as u64; - let mut parts: Vec = Vec::new(); - - for p in paths { - if remaining == 0 { - break; - } - - match fs.get_metadata(&p, /*sandbox*/ None).await { - Ok(metadata) if !metadata.is_file => continue, - Ok(_) => {} - Err(err) if err.kind() == io::ErrorKind::NotFound => continue, - Err(err) => return Err(err), - } - - let mut data = match fs.read_file(&p, /*sandbox*/ None).await { - Ok(data) => data, - Err(err) if err.kind() == io::ErrorKind::NotFound => continue, - Err(err) => return Err(err), - }; - let size = data.len() as u64; - if size > remaining { - data.truncate(remaining as usize); - } - - if size > remaining { - tracing::warn!( - "Project doc `{}` exceeds remaining budget ({} bytes) - truncating.", - p.display(), - remaining, - ); - } - - let text = String::from_utf8_lossy(&data).to_string(); - if !text.trim().is_empty() { - parts.push(text); - remaining = remaining.saturating_sub(data.len() as u64); - } - } - - if parts.is_empty() { - Ok(None) - } else { - Ok(Some(parts.join("\n\n"))) - } -} - -/// Discover the list of AGENTS.md files using the same search rules as -/// `read_project_docs`, but return the file paths instead of concatenated -/// contents. The list is ordered from project root to the current working -/// directory (inclusive). Symlinks are allowed. When `project_doc_max_bytes` -/// is zero, returns an empty list. -pub async fn discover_project_doc_paths( - config: &Config, - fs: &dyn ExecutorFileSystem, -) -> io::Result> { - if config.project_doc_max_bytes == 0 { - return Ok(Vec::new()); - } - - let mut dir = config.cwd.clone(); - if let Ok(canon) = normalize_path(&dir) { - dir = AbsolutePathBuf::try_from(canon)?; - } - - let mut merged = TomlValue::Table(toml::map::Map::new()); - for layer in config.config_layer_stack.get_layers( - ConfigLayerStackOrdering::LowestPrecedenceFirst, - /*include_disabled*/ false, - ) { - if matches!(layer.name, ConfigLayerSource::Project { .. }) { - continue; - } - merge_toml_values(&mut merged, &layer.config); - } - let project_root_markers = match project_root_markers_from_config(&merged) { - Ok(Some(markers)) => markers, - Ok(None) => default_project_root_markers(), - Err(err) => { - tracing::warn!("invalid project_root_markers: {err}"); - default_project_root_markers() - } - }; - let mut project_root = None; - if !project_root_markers.is_empty() { - for ancestor in dir.ancestors() { - for marker in &project_root_markers { - let marker_path = ancestor.join(marker); - let marker_exists = match fs.get_metadata(&marker_path, /*sandbox*/ None).await { - Ok(_) => true, - Err(err) if err.kind() == io::ErrorKind::NotFound => false, - Err(err) => return Err(err), - }; - if marker_exists { - project_root = Some(ancestor.clone()); - break; - } - } - if project_root.is_some() { - break; - } - } - } - - let search_dirs: Vec = if let Some(root) = project_root { - let mut dirs = Vec::new(); - let mut cursor = dir.clone(); - loop { - dirs.push(cursor.clone()); - if cursor == root { - break; - } - let Some(parent) = cursor.parent() else { - break; - }; - cursor = parent; - } - dirs.reverse(); - dirs - } else { - vec![dir] - }; - - let mut found: Vec = Vec::new(); - let candidate_filenames = candidate_filenames(config); - for d in search_dirs { - for name in &candidate_filenames { - let candidate = d.join(name); - match fs.get_metadata(&candidate, /*sandbox*/ None).await { - Ok(md) if md.is_file => { - found.push(candidate); - break; - } - Ok(_) => {} - Err(err) if err.kind() == io::ErrorKind::NotFound => continue, - Err(err) => return Err(err), - } - } - } - - Ok(found) -} -fn candidate_filenames<'a>(config: &'a Config) -> Vec<&'a str> { - let mut names: Vec<&'a str> = - Vec::with_capacity(2 + config.project_doc_fallback_filenames.len()); - names.push(LOCAL_PROJECT_DOC_FILENAME); - names.push(DEFAULT_PROJECT_DOC_FILENAME); - for candidate in &config.project_doc_fallback_filenames { - let candidate = candidate.as_str(); - if candidate.is_empty() { - continue; - } - if !names.contains(&candidate) { - names.push(candidate); - } - } - names -} - -#[cfg(test)] -#[path = "project_doc_tests.rs"] -mod tests; diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 890489bc2c..05f1bb5ccc 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -55,7 +55,7 @@ use crate::bottom_pane::StatusLinePreviewData; use crate::bottom_pane::StatusLineSetupView; use crate::bottom_pane::TerminalTitleItem; use crate::bottom_pane::TerminalTitleSetupView; -use crate::legacy_core::DEFAULT_PROJECT_DOC_FILENAME; +use crate::legacy_core::DEFAULT_AGENTS_MD_FILENAME; use crate::legacy_core::config::Config; use crate::legacy_core::config::Constrained; use crate::legacy_core::config::ConstraintResult; diff --git a/codex-rs/tui/src/chatwidget/slash_dispatch.rs b/codex-rs/tui/src/chatwidget/slash_dispatch.rs index ee4d22e95f..2c5c83b380 100644 --- a/codex-rs/tui/src/chatwidget/slash_dispatch.rs +++ b/codex-rs/tui/src/chatwidget/slash_dispatch.rs @@ -92,10 +92,10 @@ impl ChatWidget { self.app_event_tx.send(AppEvent::ForkCurrentSession); } SlashCommand::Init => { - let init_target = self.config.cwd.join(DEFAULT_PROJECT_DOC_FILENAME); + let init_target = self.config.cwd.join(DEFAULT_AGENTS_MD_FILENAME); if init_target.exists() { let message = format!( - "{DEFAULT_PROJECT_DOC_FILENAME} already exists here. Skipping /init to avoid overwriting it." + "{DEFAULT_AGENTS_MD_FILENAME} already exists here. Skipping /init to avoid overwriting it." ); self.add_info_message(message, /*hint*/ None); return; diff --git a/codex-rs/tui/src/chatwidget/tests/slash_commands.rs b/codex-rs/tui/src/chatwidget/tests/slash_commands.rs index 0c6b9be8e6..c7caacc2eb 100644 --- a/codex-rs/tui/src/chatwidget/tests/slash_commands.rs +++ b/codex-rs/tui/src/chatwidget/tests/slash_commands.rs @@ -73,7 +73,7 @@ async fn ctrl_d_with_modal_open_does_not_quit() { async fn slash_init_skips_when_project_doc_exists() { let (mut chat, mut rx, mut op_rx) = make_chatwidget_manual(/*model_override*/ None).await; let tempdir = tempdir().unwrap(); - let existing_path = tempdir.path().join(DEFAULT_PROJECT_DOC_FILENAME); + let existing_path = tempdir.path().join(DEFAULT_AGENTS_MD_FILENAME); std::fs::write(&existing_path, "existing instructions").unwrap(); chat.config.cwd = tempdir.path().to_path_buf().abs(); @@ -88,7 +88,7 @@ async fn slash_init_skips_when_project_doc_exists() { assert_eq!(cells.len(), 1, "expected one info message"); let rendered = lines_to_single_string(&cells[0]); assert!( - rendered.contains(DEFAULT_PROJECT_DOC_FILENAME), + rendered.contains(DEFAULT_AGENTS_MD_FILENAME), "info message should mention the existing file: {rendered:?}" ); assert!( diff --git a/codex-rs/tui/src/status/helpers.rs b/codex-rs/tui/src/status/helpers.rs index 2ea56e0b5c..27292e1364 100644 --- a/codex-rs/tui/src/status/helpers.rs +++ b/codex-rs/tui/src/status/helpers.rs @@ -185,8 +185,8 @@ fn title_case(s: &str) -> String { #[cfg(test)] mod tests { use super::*; - use crate::legacy_core::DEFAULT_PROJECT_DOC_FILENAME; - use crate::legacy_core::LOCAL_PROJECT_DOC_FILENAME; + use crate::legacy_core::DEFAULT_AGENTS_MD_FILENAME; + use crate::legacy_core::LOCAL_AGENTS_MD_FILENAME; use crate::legacy_core::config::ConfigBuilder; use codex_utils_absolute_path::test_support::PathBufExt; use pretty_assertions::assert_eq; @@ -227,7 +227,7 @@ mod tests { async fn compose_agents_summary_includes_global_agents_path() { let codex_home = TempDir::new().expect("temp codex home"); let cwd = TempDir::new().expect("temp cwd"); - let global_agents_path = codex_home.path().join(DEFAULT_PROJECT_DOC_FILENAME); + let global_agents_path = codex_home.path().join(DEFAULT_AGENTS_MD_FILENAME); let config = test_config(&codex_home, &cwd).await; assert_eq!( @@ -240,7 +240,7 @@ mod tests { async fn compose_agents_summary_names_global_agents_override() { let codex_home = TempDir::new().expect("temp codex home"); let cwd = TempDir::new().expect("temp cwd"); - let override_path = codex_home.path().join(LOCAL_PROJECT_DOC_FILENAME); + let override_path = codex_home.path().join(LOCAL_AGENTS_MD_FILENAME); let config = test_config(&codex_home, &cwd).await; assert_eq!( @@ -253,8 +253,8 @@ mod tests { async fn compose_agents_summary_orders_global_before_project_agents() { let codex_home = TempDir::new().expect("temp codex home"); let cwd = TempDir::new().expect("temp cwd"); - let global_agents_path = codex_home.path().join(DEFAULT_PROJECT_DOC_FILENAME); - let project_agents_path = cwd.path().join(DEFAULT_PROJECT_DOC_FILENAME); + let global_agents_path = codex_home.path().join(DEFAULT_AGENTS_MD_FILENAME); + let project_agents_path = cwd.path().join(DEFAULT_AGENTS_MD_FILENAME); let config = test_config(&codex_home, &cwd).await; let summary = compose_agents_summary( @@ -270,7 +270,7 @@ mod tests { Some(format_directory_display(&global_agents_path, /*max_width*/ None).as_str()) ); let project_path = paths.next().expect("project agents path"); - assert!(project_path.ends_with(DEFAULT_PROJECT_DOC_FILENAME)); + assert!(project_path.ends_with(DEFAULT_AGENTS_MD_FILENAME)); assert_eq!(paths.next(), None); } } From 206dd13c329e4174090b8d77b4da088ffa930144 Mon Sep 17 00:00:00 2001 From: pakrym-oai Date: Thu, 16 Apr 2026 11:16:44 -0700 Subject: [PATCH 31/59] Move more connector logic into connectors crate (#18158) Reduce the size of core --- codex-rs/Cargo.lock | 2 + codex-rs/chatgpt/Cargo.toml | 1 + codex-rs/chatgpt/src/connectors.rs | 97 ++--- codex-rs/connectors/src/accessible.rs | 76 ++++ codex-rs/connectors/src/filter.rs | 68 ++++ codex-rs/connectors/src/lib.rs | 5 + codex-rs/connectors/src/merge.rs | 119 ++++++ codex-rs/connectors/src/metadata.rs | 27 ++ codex-rs/core/src/codex.rs | 18 +- codex-rs/core/src/codex_tests.rs | 2 +- codex-rs/core/src/connectors.rs | 348 ++---------------- codex-rs/core/src/connectors_tests.rs | 44 ++- codex-rs/core/src/plugins/injection.rs | 3 +- codex-rs/core/src/plugins/mentions.rs | 3 +- codex-rs/tui/Cargo.toml | 1 + codex-rs/tui/src/app_event.rs | 2 +- codex-rs/tui/src/bottom_pane/chat_composer.rs | 7 +- codex-rs/tui/src/chatwidget.rs | 19 +- codex-rs/tui/src/chatwidget/skills.rs | 7 +- .../chatwidget/tests/popups_and_settings.rs | 49 +-- 20 files changed, 462 insertions(+), 436 deletions(-) create mode 100644 codex-rs/connectors/src/accessible.rs create mode 100644 codex-rs/connectors/src/filter.rs create mode 100644 codex-rs/connectors/src/merge.rs create mode 100644 codex-rs/connectors/src/metadata.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index a499a511ca..296e51725c 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1649,6 +1649,7 @@ version = "0.0.0" dependencies = [ "anyhow", "clap", + "codex-app-server-protocol", "codex-config", "codex-connectors", "codex-core", @@ -2936,6 +2937,7 @@ dependencies = [ "codex-cli", "codex-cloud-requirements", "codex-config", + "codex-connectors", "codex-exec-server", "codex-features", "codex-feedback", diff --git a/codex-rs/chatgpt/Cargo.toml b/codex-rs/chatgpt/Cargo.toml index 381b4fc873..354449934a 100644 --- a/codex-rs/chatgpt/Cargo.toml +++ b/codex-rs/chatgpt/Cargo.toml @@ -10,6 +10,7 @@ workspace = true [dependencies] anyhow = { workspace = true } clap = { workspace = true, features = ["derive"] } +codex-app-server-protocol = { workspace = true } codex-connectors = { workspace = true } codex-config = { workspace = true } codex-core = { workspace = true } diff --git a/codex-rs/chatgpt/src/connectors.rs b/codex-rs/chatgpt/src/connectors.rs index f37609be31..5f6efbc124 100644 --- a/codex-rs/chatgpt/src/connectors.rs +++ b/codex-rs/chatgpt/src/connectors.rs @@ -1,7 +1,3 @@ -use codex_core::config::Config; -use codex_login::AuthManager; -use codex_login::CodexAuth; -use codex_login::token_data::TokenData; use std::collections::HashSet; use std::time::Duration; @@ -9,21 +5,24 @@ use crate::chatgpt_client::chatgpt_get_request_with_timeout; use crate::chatgpt_token::get_chatgpt_token_data; use crate::chatgpt_token::init_chatgpt_token_from_auth; +use codex_app_server_protocol::AppInfo; use codex_connectors::AllConnectorsCacheKey; use codex_connectors::DirectoryListResponse; - -pub use codex_core::connectors::AppInfo; -pub use codex_core::connectors::connector_display_label; -use codex_core::connectors::filter_disallowed_connectors; +use codex_connectors::filter::filter_disallowed_connectors; +use codex_connectors::merge::merge_connectors; +use codex_connectors::merge::merge_plugin_connectors; +use codex_core::config::Config; pub use codex_core::connectors::list_accessible_connectors_from_mcp_tools; pub use codex_core::connectors::list_accessible_connectors_from_mcp_tools_with_options; pub use codex_core::connectors::list_accessible_connectors_from_mcp_tools_with_options_and_status; pub use codex_core::connectors::list_cached_accessible_connectors_from_mcp_tools; -use codex_core::connectors::merge_connectors; -use codex_core::connectors::merge_plugin_apps; pub use codex_core::connectors::with_app_enabled_state; use codex_core::plugins::AppConnectorId; use codex_core::plugins::PluginsManager; +use codex_login::AuthManager; +use codex_login::CodexAuth; +use codex_login::default_client::originator; +use codex_login::token_data::TokenData; const DIRECTORY_CONNECTORS_TIMEOUT: Duration = Duration::from_secs(60); @@ -74,8 +73,17 @@ pub async fn list_cached_all_connectors(config: &Config) -> Option> let token_data = get_chatgpt_token_data()?; let cache_key = all_connectors_cache_key(config, &token_data); let connectors = codex_connectors::cached_all_connectors(&cache_key)?; - let connectors = merge_plugin_apps(connectors, plugin_apps_for_config(config).await); - Some(filter_disallowed_connectors(connectors)) + let connectors = merge_plugin_connectors( + connectors, + plugin_apps_for_config(config) + .await + .into_iter() + .map(|connector_id| connector_id.0), + ); + Some(filter_disallowed_connectors( + connectors, + originator().value.as_str(), + )) } pub async fn list_all_connectors_with_options( @@ -105,8 +113,17 @@ pub async fn list_all_connectors_with_options( }, ) .await?; - let connectors = merge_plugin_apps(connectors, plugin_apps_for_config(config).await); - Ok(filter_disallowed_connectors(connectors)) + let connectors = merge_plugin_connectors( + connectors, + plugin_apps_for_config(config) + .await + .into_iter() + .map(|connector_id| connector_id.0), + ); + Ok(filter_disallowed_connectors( + connectors, + originator().value.as_str(), + )) } fn all_connectors_cache_key(config: &Config, token_data: &TokenData) -> AllConnectorsCacheKey { @@ -134,7 +151,13 @@ pub fn connectors_for_plugin_apps( .map(|connector_id| connector_id.0.as_str()) .collect::>(); - filter_disallowed_connectors(merge_plugin_apps(connectors, plugin_apps.to_vec())) + let connectors = merge_plugin_connectors( + connectors, + plugin_apps + .iter() + .map(|connector_id| connector_id.0.clone()), + ); + filter_disallowed_connectors(connectors, originator().value.as_str()) .into_iter() .filter(|connector| plugin_app_ids.contains(connector.id.as_str())) .collect() @@ -158,13 +181,13 @@ pub fn merge_connectors_with_accessible( accessible_connectors }; let merged = merge_connectors(connectors, accessible_connectors); - filter_disallowed_connectors(merged) + filter_disallowed_connectors(merged, originator().value.as_str()) } #[cfg(test)] mod tests { use super::*; - use codex_core::connectors::connector_install_url; + use codex_connectors::metadata::connector_install_url; use codex_core::plugins::AppConnectorId; use pretty_assertions::assert_eq; @@ -186,46 +209,6 @@ mod tests { } } - #[test] - fn allows_asdk_connectors() { - let filtered = filter_disallowed_connectors(vec![app("asdk_app_hidden"), app("alpha")]); - assert_eq!(filtered, vec![app("asdk_app_hidden"), app("alpha")]); - } - - #[test] - fn allows_whitelisted_asdk_connectors() { - let filtered = filter_disallowed_connectors(vec![ - app("asdk_app_69781557cc1481919cf5e9824fa2e792"), - app("beta"), - ]); - assert_eq!( - filtered, - vec![ - app("asdk_app_69781557cc1481919cf5e9824fa2e792"), - app("beta") - ] - ); - } - - #[test] - fn filters_openai_prefixed_connectors() { - let filtered = filter_disallowed_connectors(vec![ - app("connector_openai_foo"), - app("connector_openai_bar"), - app("gamma"), - ]); - assert_eq!(filtered, vec![app("gamma")]); - } - - #[test] - fn filters_disallowed_connector_ids() { - let filtered = filter_disallowed_connectors(vec![ - app("asdk_app_6938a94a61d881918ef32cb999ff937c"), - app("delta"), - ]); - assert_eq!(filtered, vec![app("delta")]); - } - fn merged_app(id: &str, is_accessible: bool) -> AppInfo { AppInfo { id: id.to_string(), diff --git a/codex-rs/connectors/src/accessible.rs b/codex-rs/connectors/src/accessible.rs new file mode 100644 index 0000000000..c44f8d8a38 --- /dev/null +++ b/codex-rs/connectors/src/accessible.rs @@ -0,0 +1,76 @@ +use std::collections::BTreeSet; +use std::collections::HashMap; + +use crate::metadata::connector_install_url; +use crate::normalize_connector_value; +use codex_app_server_protocol::AppInfo; + +pub struct AccessibleConnectorTool { + pub connector_id: String, + pub connector_name: Option, + pub connector_description: Option, + pub plugin_display_names: Vec, +} + +pub fn collect_accessible_connectors(tools: I) -> Vec +where + I: IntoIterator, +{ + let mut connectors: HashMap)> = HashMap::new(); + for tool in tools { + let connector_id = tool.connector_id; + let connector_name = normalize_connector_value(tool.connector_name.as_deref()) + .unwrap_or_else(|| connector_id.clone()); + let connector_description = + normalize_connector_value(tool.connector_description.as_deref()); + if let Some((existing, existing_plugin_display_names)) = connectors.get_mut(&connector_id) { + if existing.name == connector_id && connector_name != connector_id { + existing.name = connector_name; + } + if existing.description.is_none() && connector_description.is_some() { + existing.description = connector_description; + } + existing_plugin_display_names.extend(tool.plugin_display_names); + } else { + connectors.insert( + connector_id.clone(), + ( + AppInfo { + id: connector_id.clone(), + name: connector_name, + description: connector_description, + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + branding: None, + app_metadata: None, + labels: None, + install_url: None, + is_accessible: true, + is_enabled: true, + plugin_display_names: Vec::new(), + }, + tool.plugin_display_names + .into_iter() + .collect::>(), + ), + ); + } + } + let mut accessible: Vec = connectors + .into_values() + .map(|(mut connector, plugin_display_names)| { + connector.plugin_display_names = plugin_display_names.into_iter().collect(); + connector.install_url = Some(connector_install_url(&connector.name, &connector.id)); + connector + }) + .collect(); + accessible.sort_by(|left, right| { + right + .is_accessible + .cmp(&left.is_accessible) + .then_with(|| left.name.cmp(&right.name)) + .then_with(|| left.id.cmp(&right.id)) + }); + accessible +} diff --git a/codex-rs/connectors/src/filter.rs b/codex-rs/connectors/src/filter.rs new file mode 100644 index 0000000000..82c334f82d --- /dev/null +++ b/codex-rs/connectors/src/filter.rs @@ -0,0 +1,68 @@ +use std::collections::HashSet; + +use codex_app_server_protocol::AppInfo; + +pub fn filter_tool_suggest_discoverable_connectors( + directory_connectors: Vec, + accessible_connectors: &[AppInfo], + discoverable_connector_ids: &HashSet, + originator_value: &str, +) -> Vec { + let accessible_connector_ids: HashSet<&str> = accessible_connectors + .iter() + .filter(|connector| connector.is_accessible) + .map(|connector| connector.id.as_str()) + .collect(); + + let mut connectors = filter_disallowed_connectors(directory_connectors, originator_value) + .into_iter() + .filter(|connector| !accessible_connector_ids.contains(connector.id.as_str())) + .filter(|connector| discoverable_connector_ids.contains(connector.id.as_str())) + .collect::>(); + connectors.sort_by(|left, right| { + left.name + .cmp(&right.name) + .then_with(|| left.id.cmp(&right.id)) + }); + connectors +} + +const DISALLOWED_CONNECTOR_IDS: &[&str] = &[ + "asdk_app_6938a94a61d881918ef32cb999ff937c", + "connector_2b0a9009c9c64bf9933a3dae3f2b1254", + "connector_3f8d1a79f27c4c7ba1a897ab13bf37dc", + "connector_68de829bf7648191acd70a907364c67c", + "connector_68e004f14af881919eb50893d3d9f523", + "connector_69272cb413a081919685ec3c88d1744e", +]; +const FIRST_PARTY_CHAT_DISALLOWED_CONNECTOR_IDS: &[&str] = + &["connector_0f9c9d4592e54d0a9a12b3f44a1e2010"]; +const DISALLOWED_CONNECTOR_PREFIX: &str = "connector_openai_"; + +pub fn filter_disallowed_connectors( + connectors: Vec, + originator_value: &str, +) -> Vec { + let first_party_chat_originator = is_first_party_chat_originator(originator_value); + connectors + .into_iter() + .filter(|connector| { + is_connector_id_allowed(connector.id.as_str(), first_party_chat_originator) + }) + .collect() +} + +fn is_first_party_chat_originator(originator_value: &str) -> bool { + originator_value == "codex_atlas" || originator_value == "codex_chatgpt_desktop" +} + +fn is_connector_id_allowed(connector_id: &str, first_party_chat_originator: bool) -> bool { + let disallowed_connector_ids = if first_party_chat_originator { + FIRST_PARTY_CHAT_DISALLOWED_CONNECTOR_IDS + } else { + DISALLOWED_CONNECTOR_IDS + }; + + !connector_id.starts_with(DISALLOWED_CONNECTOR_PREFIX) + && !disallowed_connector_ids.contains(&connector_id) +} diff --git a/codex-rs/connectors/src/lib.rs b/codex-rs/connectors/src/lib.rs index a74ece5a35..d7c5fb405c 100644 --- a/codex-rs/connectors/src/lib.rs +++ b/codex-rs/connectors/src/lib.rs @@ -10,6 +10,11 @@ use codex_app_server_protocol::AppInfo; use codex_app_server_protocol::AppMetadata; use serde::Deserialize; +pub mod accessible; +pub mod filter; +pub mod merge; +pub mod metadata; + pub const CONNECTORS_CACHE_TTL: Duration = Duration::from_secs(3600); #[derive(Clone, Debug, PartialEq, Eq)] diff --git a/codex-rs/connectors/src/merge.rs b/codex-rs/connectors/src/merge.rs new file mode 100644 index 0000000000..b41ee63add --- /dev/null +++ b/codex-rs/connectors/src/merge.rs @@ -0,0 +1,119 @@ +use std::collections::HashMap; +use std::collections::HashSet; + +use crate::metadata::connector_install_url; +use crate::metadata::sort_connectors_by_accessibility_and_name; +use codex_app_server_protocol::AppInfo; + +pub fn merge_connectors( + connectors: Vec, + accessible_connectors: Vec, +) -> Vec { + let mut merged: HashMap = connectors + .into_iter() + .map(|mut connector| { + connector.is_accessible = false; + (connector.id.clone(), connector) + }) + .collect(); + + for mut connector in accessible_connectors { + connector.is_accessible = true; + let connector_id = connector.id.clone(); + if let Some(existing) = merged.get_mut(&connector_id) { + existing.is_accessible = true; + if existing.name == existing.id && connector.name != connector.id { + existing.name = connector.name; + } + if existing.description.is_none() && connector.description.is_some() { + existing.description = connector.description; + } + if existing.logo_url.is_none() && connector.logo_url.is_some() { + existing.logo_url = connector.logo_url; + } + if existing.logo_url_dark.is_none() && connector.logo_url_dark.is_some() { + existing.logo_url_dark = connector.logo_url_dark; + } + if existing.distribution_channel.is_none() && connector.distribution_channel.is_some() { + existing.distribution_channel = connector.distribution_channel; + } + existing + .plugin_display_names + .extend(connector.plugin_display_names); + } else { + merged.insert(connector_id, connector); + } + } + + let mut merged = merged.into_values().collect::>(); + for connector in &mut merged { + if connector.install_url.is_none() { + connector.install_url = Some(connector_install_url(&connector.name, &connector.id)); + } + connector.plugin_display_names.sort_unstable(); + connector.plugin_display_names.dedup(); + } + sort_connectors_by_accessibility_and_name(&mut merged); + merged +} + +pub fn merge_plugin_connectors(connectors: Vec, plugin_app_ids: I) -> Vec +where + I: IntoIterator, +{ + let mut merged = connectors; + let mut connector_ids = merged + .iter() + .map(|connector| connector.id.clone()) + .collect::>(); + + for connector_id in plugin_app_ids { + if connector_ids.insert(connector_id.clone()) { + merged.push(plugin_connector_to_app_info(connector_id)); + } + } + + sort_connectors_by_accessibility_and_name(&mut merged); + merged +} + +pub fn merge_plugin_connectors_with_accessible( + plugin_app_ids: I, + accessible_connectors: Vec, +) -> Vec +where + I: IntoIterator, +{ + let accessible_connector_ids: HashSet<&str> = accessible_connectors + .iter() + .map(|connector| connector.id.as_str()) + .collect(); + let plugin_connectors = plugin_app_ids + .into_iter() + .filter(|connector_id| accessible_connector_ids.contains(connector_id.as_str())) + .map(plugin_connector_to_app_info) + .collect::>(); + merge_connectors(plugin_connectors, accessible_connectors) +} + +pub fn plugin_connector_to_app_info(connector_id: String) -> AppInfo { + // Leave the placeholder name as the connector id so merge_connectors() can + // replace it with canonical app metadata from directory fetches or + // connector_name values from codex_apps tool discovery. + let name = connector_id.clone(); + AppInfo { + id: connector_id.clone(), + name: name.clone(), + description: None, + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + branding: None, + app_metadata: None, + labels: None, + install_url: Some(connector_install_url(&name, &connector_id)), + is_accessible: false, + is_enabled: true, + plugin_display_names: Vec::new(), + } +} diff --git a/codex-rs/connectors/src/metadata.rs b/codex-rs/connectors/src/metadata.rs new file mode 100644 index 0000000000..64becb766d --- /dev/null +++ b/codex-rs/connectors/src/metadata.rs @@ -0,0 +1,27 @@ +use codex_app_server_protocol::AppInfo; + +pub fn connector_display_label(connector: &AppInfo) -> String { + connector.name.clone() +} + +pub fn connector_mention_slug(connector: &AppInfo) -> String { + crate::connector_name_slug(&connector_display_label(connector)) +} + +pub fn connector_install_url(name: &str, connector_id: &str) -> String { + crate::connector_install_url(name, connector_id) +} + +pub fn sanitize_name(name: &str) -> String { + crate::connector_name_slug(name).replace("-", "_") +} + +pub(crate) fn sort_connectors_by_accessibility_and_name(connectors: &mut [AppInfo]) { + connectors.sort_by(|left, right| { + right + .is_accessible + .cmp(&left.is_accessible) + .then_with(|| left.name.cmp(&right.name)) + .then_with(|| left.id.cmp(&right.id)) + }); +} diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 50a4c25abd..5274d88664 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -6267,8 +6267,11 @@ pub(crate) async fn run_turn( HashMap::new() }; let available_connectors = if turn_context.apps_enabled() { - let connectors = connectors::merge_plugin_apps_with_accessible( - loaded_plugins.effective_apps(), + let connectors = codex_connectors::merge::merge_plugin_connectors_with_accessible( + loaded_plugins + .effective_apps() + .into_iter() + .map(|connector_id| connector_id.0), connectors::accessible_connectors_from_mcp_tools(&mcp_tools), ); connectors::with_app_enabled_state(connectors, &turn_context.config) @@ -6914,7 +6917,7 @@ fn collect_explicit_app_ids_from_skill_items( let connector_slug_counts = build_connector_slug_counts(connectors); for connector in connectors { - let slug = connectors::connector_mention_slug(connector); + let slug = codex_connectors::metadata::connector_mention_slug(connector); let connector_count = connector_slug_counts.get(&slug).copied().unwrap_or(0); let skill_count = skill_name_counts_lower.get(&slug).copied().unwrap_or(0); if connector_count == 1 && skill_count == 0 && mention_names_lower.contains(&slug) { @@ -6989,7 +6992,7 @@ fn connector_inserted_in_messages( return true; } - let mention_slug = connectors::connector_mention_slug(connector); + let mention_slug = codex_connectors::metadata::connector_mention_slug(connector); let connector_count = connector_slug_counts .get(&mention_slug) .copied() @@ -7217,8 +7220,11 @@ pub(crate) async fn built_tools( connectors::with_app_enabled_state(connectors.clone(), &turn_context.config) }); let connectors = if apps_enabled { - let connectors = connectors::merge_plugin_apps_with_accessible( - loaded_plugins.effective_apps(), + let connectors = codex_connectors::merge::merge_plugin_connectors_with_accessible( + loaded_plugins + .effective_apps() + .into_iter() + .map(|connector_id| connector_id.0), accessible_connectors.clone().unwrap_or_default(), ); Some(connectors::with_app_enabled_state( diff --git a/codex-rs/core/src/codex_tests.rs b/codex-rs/core/src/codex_tests.rs index 6f4276b077..81ebd6c9f8 100644 --- a/codex-rs/core/src/codex_tests.rs +++ b/codex-rs/core/src/codex_tests.rs @@ -413,7 +413,7 @@ fn make_mcp_tool( ) -> ToolInfo { let tool_namespace = if server_name == CODEX_APPS_MCP_SERVER_NAME { connector_name - .map(crate::connectors::sanitize_name) + .map(codex_connectors::metadata::sanitize_name) .map(|connector_name| format!("mcp__{server_name}__{connector_name}")) .unwrap_or_else(|| server_name.to_string()) } else { diff --git a/codex-rs/core/src/connectors.rs b/codex-rs/core/src/connectors.rs index 77603adcf1..103ce251fc 100644 --- a/codex-rs/core/src/connectors.rs +++ b/codex-rs/core/src/connectors.rs @@ -1,4 +1,3 @@ -use std::collections::BTreeSet; use std::collections::HashMap; use std::collections::HashSet; use std::sync::Arc; @@ -26,7 +25,6 @@ use crate::codex::INITIAL_SUBMIT_ID; use crate::config::Config; use crate::config_loader::AppsRequirementsToml; use crate::mcp::McpManager; -use crate::plugins::AppConnectorId; use crate::plugins::PluginsManager; use crate::plugins::list_tool_suggest_discoverable_plugins; use codex_config::types::AppToolApproval; @@ -36,7 +34,6 @@ use codex_features::Feature; use codex_login::AuthManager; use codex_login::CodexAuth; use codex_login::default_client::create_client; -use codex_login::default_client::is_first_party_chat_originator; use codex_login::default_client::originator; use codex_mcp::CODEX_APPS_MCP_SERVER_NAME; use codex_mcp::McpConnectionManager; @@ -46,7 +43,6 @@ use codex_mcp::codex_apps_tools_cache_key; use codex_mcp::compute_auth_statuses; use codex_mcp::with_codex_apps_mcp; -pub use codex_connectors::CONNECTORS_CACHE_TTL; const CONNECTORS_READY_TIMEOUT_ON_EMPTY_TOOLS: Duration = Duration::from_secs(30); const DIRECTORY_CONNECTORS_TIMEOUT: Duration = Duration::from_secs(60); @@ -122,13 +118,15 @@ pub(crate) async fn list_tool_suggest_discoverable_tools_with_auth( let directory_connectors = list_directory_connectors_for_tool_suggest_with_auth(config, auth).await?; let connector_ids = tool_suggest_connector_ids(config).await; - let discoverable_connectors = filter_tool_suggest_discoverable_connectors( - directory_connectors, - accessible_connectors, - &connector_ids, - ) - .into_iter() - .map(DiscoverableTool::from); + let discoverable_connectors = + codex_connectors::filter::filter_tool_suggest_discoverable_connectors( + directory_connectors, + accessible_connectors, + &connector_ids, + originator().value.as_str(), + ) + .into_iter() + .map(DiscoverableTool::from); let discoverable_plugins = list_tool_suggest_discoverable_plugins(config) .await? .into_iter() @@ -151,7 +149,12 @@ pub async fn list_cached_accessible_connectors_from_mcp_tools( return Some(Vec::new()); } let cache_key = accessible_connectors_cache_key(config, auth.as_ref()); - read_cached_accessible_connectors(&cache_key).map(filter_disallowed_connectors) + read_cached_accessible_connectors(&cache_key).map(|connectors| { + codex_connectors::filter::filter_disallowed_connectors( + connectors, + originator().value.as_str(), + ) + }) } pub(crate) fn refresh_accessible_connectors_cache_from_mcp_tools( @@ -164,8 +167,10 @@ pub(crate) fn refresh_accessible_connectors_cache_from_mcp_tools( } let cache_key = accessible_connectors_cache_key(config, auth); - let accessible_connectors = - filter_disallowed_connectors(accessible_connectors_from_mcp_tools(mcp_tools)); + let accessible_connectors = codex_connectors::filter::filter_disallowed_connectors( + accessible_connectors_from_mcp_tools(mcp_tools), + originator().value.as_str(), + ); write_cached_accessible_connectors(cache_key, &accessible_connectors); } @@ -202,7 +207,10 @@ pub async fn list_accessible_connectors_from_mcp_tools_with_options_and_status( let tool_plugin_provenance = mcp_manager.tool_plugin_provenance(config).await; if !force_refetch && let Some(cached_connectors) = read_cached_accessible_connectors(&cache_key) { - let cached_connectors = filter_disallowed_connectors(cached_connectors); + let cached_connectors = codex_connectors::filter::filter_disallowed_connectors( + cached_connectors, + originator().value.as_str(), + ); let cached_connectors = with_app_plugin_sources(cached_connectors, &tool_plugin_provenance); return Ok(AccessibleConnectorsStatus { connectors: cached_connectors, @@ -293,8 +301,10 @@ pub async fn list_accessible_connectors_from_mcp_tools_with_options_and_status( cancel_token.cancel(); } - let accessible_connectors = - filter_disallowed_connectors(accessible_connectors_from_mcp_tools(&tools)); + let accessible_connectors = codex_connectors::filter::filter_disallowed_connectors( + accessible_connectors_from_mcp_tools(&tools), + originator().value.as_str(), + ); if codex_apps_ready || !accessible_connectors.is_empty() { write_cached_accessible_connectors(cache_key, &accessible_connectors); } @@ -357,35 +367,11 @@ fn write_cached_accessible_connectors( .unwrap_or_else(std::sync::PoisonError::into_inner); *cache_guard = Some(CachedAccessibleConnectors { key: cache_key, - expires_at: Instant::now() + CONNECTORS_CACHE_TTL, + expires_at: Instant::now() + codex_connectors::CONNECTORS_CACHE_TTL, connectors: connectors.to_vec(), }); } -fn filter_tool_suggest_discoverable_connectors( - directory_connectors: Vec, - accessible_connectors: &[AppInfo], - discoverable_connector_ids: &HashSet, -) -> Vec { - let accessible_connector_ids: HashSet<&str> = accessible_connectors - .iter() - .filter(|connector| connector.is_accessible) - .map(|connector| connector.id.as_str()) - .collect(); - - let mut connectors = filter_disallowed_connectors(directory_connectors) - .into_iter() - .filter(|connector| !accessible_connector_ids.contains(connector.id.as_str())) - .filter(|connector| discoverable_connector_ids.contains(connector.id.as_str())) - .collect::>(); - connectors.sort_by(|left, right| { - left.name - .cmp(&right.name) - .then_with(|| left.id.cmp(&right.id)) - }); - connectors -} - async fn tool_suggest_connector_ids(config: &Config) -> HashSet { let mut connector_ids = PluginsManager::new(config.codex_home.to_path_buf()) .plugins_for_config(config) @@ -493,14 +479,6 @@ async fn chatgpt_get_request_with_token( } } -pub fn connector_display_label(connector: &AppInfo) -> String { - format_connector_label(&connector.name, &connector.id) -} - -pub fn connector_mention_slug(connector: &AppInfo) -> String { - sanitize_slug(&connector_display_label(connector)) -} - pub(crate) fn accessible_connectors_from_mcp_tools( mcp_tools: &HashMap, ) -> Vec { @@ -511,114 +489,14 @@ pub(crate) fn accessible_connectors_from_mcp_tools( return None; } let connector_id = tool.connector_id.as_deref()?; - Some(( - connector_id.to_string(), - normalize_connector_value(tool.connector_name.as_deref()), - normalize_connector_value(tool.connector_description.as_deref()), - tool.plugin_display_names.clone(), - )) - }); - collect_accessible_connectors(tools) -} - -pub fn merge_connectors( - connectors: Vec, - accessible_connectors: Vec, -) -> Vec { - let mut merged: HashMap = connectors - .into_iter() - .map(|mut connector| { - connector.is_accessible = false; - (connector.id.clone(), connector) + Some(codex_connectors::accessible::AccessibleConnectorTool { + connector_id: connector_id.to_string(), + connector_name: tool.connector_name.clone(), + connector_description: tool.connector_description.clone(), + plugin_display_names: tool.plugin_display_names.clone(), }) - .collect(); - - for mut connector in accessible_connectors { - connector.is_accessible = true; - let connector_id = connector.id.clone(); - if let Some(existing) = merged.get_mut(&connector_id) { - existing.is_accessible = true; - if existing.name == existing.id && connector.name != connector.id { - existing.name = connector.name; - } - if existing.description.is_none() && connector.description.is_some() { - existing.description = connector.description; - } - if existing.logo_url.is_none() && connector.logo_url.is_some() { - existing.logo_url = connector.logo_url; - } - if existing.logo_url_dark.is_none() && connector.logo_url_dark.is_some() { - existing.logo_url_dark = connector.logo_url_dark; - } - if existing.distribution_channel.is_none() && connector.distribution_channel.is_some() { - existing.distribution_channel = connector.distribution_channel; - } - existing - .plugin_display_names - .extend(connector.plugin_display_names); - } else { - merged.insert(connector_id, connector); - } - } - - let mut merged = merged.into_values().collect::>(); - for connector in &mut merged { - if connector.install_url.is_none() { - connector.install_url = Some(connector_install_url(&connector.name, &connector.id)); - } - connector.plugin_display_names.sort_unstable(); - connector.plugin_display_names.dedup(); - } - merged.sort_by(|left, right| { - right - .is_accessible - .cmp(&left.is_accessible) - .then_with(|| left.name.cmp(&right.name)) - .then_with(|| left.id.cmp(&right.id)) }); - merged -} - -pub fn merge_plugin_apps( - connectors: Vec, - plugin_apps: Vec, -) -> Vec { - let mut merged = connectors; - let mut connector_ids = merged - .iter() - .map(|connector| connector.id.clone()) - .collect::>(); - - for connector_id in plugin_apps { - if connector_ids.insert(connector_id.0.clone()) { - merged.push(plugin_app_to_app_info(connector_id)); - } - } - - merged.sort_by(|left, right| { - right - .is_accessible - .cmp(&left.is_accessible) - .then_with(|| left.name.cmp(&right.name)) - .then_with(|| left.id.cmp(&right.id)) - }); - merged -} - -pub fn merge_plugin_apps_with_accessible( - plugin_apps: Vec, - accessible_connectors: Vec, -) -> Vec { - let accessible_connector_ids: HashSet<&str> = accessible_connectors - .iter() - .map(|connector| connector.id.as_str()) - .collect(); - let plugin_connectors = plugin_apps - .into_iter() - .filter(|connector_id| accessible_connector_ids.contains(connector_id.0.as_str())) - .map(plugin_app_to_app_info) - .collect::>(); - merge_connectors(plugin_connectors, accessible_connectors) + codex_connectors::accessible::collect_accessible_connectors(tools) } pub fn with_app_enabled_state(mut connectors: Vec, config: &Config) -> Vec { @@ -691,45 +569,6 @@ pub(crate) fn codex_app_tool_is_enabled(config: &Config, tool_info: &ToolInfo) - .enabled } -const DISALLOWED_CONNECTOR_IDS: &[&str] = &[ - "asdk_app_6938a94a61d881918ef32cb999ff937c", - "connector_2b0a9009c9c64bf9933a3dae3f2b1254", - "connector_3f8d1a79f27c4c7ba1a897ab13bf37dc", - "connector_68de829bf7648191acd70a907364c67c", - "connector_68e004f14af881919eb50893d3d9f523", - "connector_69272cb413a081919685ec3c88d1744e", -]; -const FIRST_PARTY_CHAT_DISALLOWED_CONNECTOR_IDS: &[&str] = - &["connector_0f9c9d4592e54d0a9a12b3f44a1e2010"]; -const DISALLOWED_CONNECTOR_PREFIX: &str = "connector_openai_"; - -pub fn filter_disallowed_connectors(connectors: Vec) -> Vec { - filter_disallowed_connectors_for_originator(connectors, originator().value.as_str()) -} - -fn filter_disallowed_connectors_for_originator( - connectors: Vec, - originator_value: &str, -) -> Vec { - connectors - .into_iter() - .filter(|connector| { - is_connector_id_allowed_for_originator(connector.id.as_str(), originator_value) - }) - .collect() -} - -fn is_connector_id_allowed_for_originator(connector_id: &str, originator_value: &str) -> bool { - let disallowed_connector_ids = if is_first_party_chat_originator(originator_value) { - FIRST_PARTY_CHAT_DISALLOWED_CONNECTOR_IDS - } else { - DISALLOWED_CONNECTOR_IDS - }; - - !connector_id.starts_with(DISALLOWED_CONNECTOR_PREFIX) - && !disallowed_connector_ids.contains(&connector_id) -} - fn read_apps_config(config: &Config) -> Option { let apps_config = read_user_apps_config(config); let had_apps_config = apps_config.is_some(); @@ -851,125 +690,6 @@ fn app_tool_policy_from_apps_config( AppToolPolicy { enabled, approval } } -fn collect_accessible_connectors(tools: I) -> Vec -where - I: IntoIterator, Option, Vec)>, -{ - let mut connectors: HashMap)> = HashMap::new(); - for (connector_id, connector_name, connector_description, plugin_display_names) in tools { - let connector_name = connector_name.unwrap_or_else(|| connector_id.clone()); - if let Some((existing, existing_plugin_display_names)) = connectors.get_mut(&connector_id) { - if existing.name == connector_id && connector_name != connector_id { - existing.name = connector_name; - } - if existing.description.is_none() && connector_description.is_some() { - existing.description = connector_description; - } - existing_plugin_display_names.extend(plugin_display_names); - } else { - connectors.insert( - connector_id.clone(), - ( - AppInfo { - id: connector_id.clone(), - name: connector_name, - description: connector_description, - logo_url: None, - logo_url_dark: None, - distribution_channel: None, - branding: None, - app_metadata: None, - labels: None, - install_url: None, - is_accessible: true, - is_enabled: true, - plugin_display_names: Vec::new(), - }, - plugin_display_names - .into_iter() - .collect::>(), - ), - ); - } - } - let mut accessible: Vec = connectors - .into_values() - .map(|(mut connector, plugin_display_names)| { - connector.plugin_display_names = plugin_display_names.into_iter().collect(); - connector.install_url = Some(connector_install_url(&connector.name, &connector.id)); - connector - }) - .collect(); - accessible.sort_by(|left, right| { - right - .is_accessible - .cmp(&left.is_accessible) - .then_with(|| left.name.cmp(&right.name)) - .then_with(|| left.id.cmp(&right.id)) - }); - accessible -} - -fn plugin_app_to_app_info(connector_id: AppConnectorId) -> AppInfo { - // Leave the placeholder name as the connector id so merge_connectors() can - // replace it with canonical app metadata from directory fetches or - // connector_name values from codex_apps tool discovery. - let connector_id = connector_id.0; - let name = connector_id.clone(); - AppInfo { - id: connector_id.clone(), - name: name.clone(), - description: None, - logo_url: None, - logo_url_dark: None, - distribution_channel: None, - branding: None, - app_metadata: None, - labels: None, - install_url: Some(connector_install_url(&name, &connector_id)), - is_accessible: false, - is_enabled: true, - plugin_display_names: Vec::new(), - } -} - -fn normalize_connector_value(value: Option<&str>) -> Option { - value - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_string) -} - -pub fn connector_install_url(name: &str, connector_id: &str) -> String { - let slug = sanitize_slug(name); - format!("https://chatgpt.com/apps/{slug}/{connector_id}") -} - -pub fn sanitize_name(name: &str) -> String { - sanitize_slug(name).replace("-", "_") -} - -fn sanitize_slug(name: &str) -> String { - let mut normalized = String::with_capacity(name.len()); - for character in name.chars() { - if character.is_ascii_alphanumeric() { - normalized.push(character.to_ascii_lowercase()); - } else { - normalized.push('-'); - } - } - let normalized = normalized.trim_matches('-'); - if normalized.is_empty() { - "app".to_string() - } else { - normalized.to_string() - } -} - -fn format_connector_label(name: &str, _id: &str) -> String { - name.to_string() -} - #[cfg(test)] #[path = "connectors_tests.rs"] mod tests; diff --git a/codex-rs/core/src/connectors_tests.rs b/codex-rs/core/src/connectors_tests.rs index c9f77d0469..e462c7939e 100644 --- a/codex-rs/core/src/connectors_tests.rs +++ b/codex-rs/core/src/connectors_tests.rs @@ -11,6 +11,13 @@ use codex_config::types::AppConfig; use codex_config::types::AppToolConfig; use codex_config::types::AppToolsConfig; use codex_config::types::AppsDefaultConfig; +use codex_connectors::filter::filter_disallowed_connectors; +use codex_connectors::filter::filter_tool_suggest_discoverable_connectors; +use codex_connectors::merge::merge_connectors; +use codex_connectors::merge::plugin_connector_to_app_info; +use codex_connectors::metadata::connector_install_url; +use codex_connectors::metadata::connector_mention_slug; +use codex_connectors::metadata::sanitize_name; use codex_features::Feature; use codex_mcp::CODEX_APPS_MCP_SERVER_NAME; use codex_mcp::ToolInfo; @@ -138,7 +145,7 @@ fn with_accessible_connectors_cache_cleared(f: impl FnOnce() -> R) -> R { #[test] fn merge_connectors_replaces_plugin_placeholder_name_with_accessible_name() { - let plugin = plugin_app_to_app_info(AppConnectorId("calendar".to_string())); + let plugin = plugin_connector_to_app_info("calendar".to_string()); let accessible = google_calendar_accessible_connector(&[]); let merged = merge_connectors(vec![plugin], vec![accessible]); @@ -281,7 +288,7 @@ async fn refresh_accessible_connectors_cache_from_mcp_tools_writes_latest_instal #[test] fn merge_connectors_unions_and_dedupes_plugin_display_names() { - let mut plugin = plugin_app_to_app_info(AppConnectorId("calendar".to_string())); + let mut plugin = plugin_connector_to_app_info("calendar".to_string()); plugin.plugin_display_names = plugin_names(&["sample", "alpha", "sample"]); let accessible = google_calendar_accessible_connector(&["beta", "alpha"]); @@ -975,33 +982,40 @@ fn app_tool_policy_matches_prefix_stripped_tool_name_for_tool_config() { #[test] fn filter_disallowed_connectors_allows_non_disallowed_connectors() { - let filtered = filter_disallowed_connectors(vec![app("asdk_app_hidden"), app("alpha")]); + let filtered = + filter_disallowed_connectors(vec![app("asdk_app_hidden"), app("alpha")], "codex_cli"); assert_eq!(filtered, vec![app("asdk_app_hidden"), app("alpha")]); } #[test] fn filter_disallowed_connectors_filters_openai_prefix() { - let filtered = filter_disallowed_connectors(vec![ - app("connector_openai_foo"), - app("connector_openai_bar"), - app("gamma"), - ]); + let filtered = filter_disallowed_connectors( + vec![ + app("connector_openai_foo"), + app("connector_openai_bar"), + app("gamma"), + ], + "codex_cli", + ); assert_eq!(filtered, vec![app("gamma")]); } #[test] fn filter_disallowed_connectors_filters_disallowed_connector_ids() { - let filtered = filter_disallowed_connectors(vec![ - app("asdk_app_6938a94a61d881918ef32cb999ff937c"), - app("connector_3f8d1a79f27c4c7ba1a897ab13bf37dc"), - app("delta"), - ]); + let filtered = filter_disallowed_connectors( + vec![ + app("asdk_app_6938a94a61d881918ef32cb999ff937c"), + app("connector_3f8d1a79f27c4c7ba1a897ab13bf37dc"), + app("delta"), + ], + "codex_cli", + ); assert_eq!(filtered, vec![app("delta")]); } #[test] fn first_party_chat_originator_filters_target_and_openai_prefixed_connectors() { - let filtered = filter_disallowed_connectors_for_originator( + let filtered = filter_disallowed_connectors( vec![ app("connector_openai_foo"), app("asdk_app_6938a94a61d881918ef32cb999ff937c"), @@ -1064,6 +1078,7 @@ fn filter_tool_suggest_discoverable_connectors_keeps_only_plugin_backed_uninstal "connector_2128aebfecb84f64a069897515042a44".to_string(), "connector_68df038e0ba48191908c8434991bbac2".to_string(), ]), + "codex_cli", ); assert_eq!( @@ -1103,6 +1118,7 @@ fn filter_tool_suggest_discoverable_connectors_excludes_accessible_apps_even_whe "connector_2128aebfecb84f64a069897515042a44".to_string(), "connector_68df038e0ba48191908c8434991bbac2".to_string(), ]), + "codex_cli", ); assert_eq!(filtered, Vec::::new()); diff --git a/codex-rs/core/src/plugins/injection.rs b/codex-rs/core/src/plugins/injection.rs index 3e5bb6ecb5..2f6ec26260 100644 --- a/codex-rs/core/src/plugins/injection.rs +++ b/codex-rs/core/src/plugins/injection.rs @@ -1,6 +1,7 @@ use std::collections::BTreeSet; use std::collections::HashMap; +use codex_connectors::metadata::connector_display_label; use codex_protocol::models::DeveloperInstructions; use codex_protocol::models::ResponseItem; @@ -46,7 +47,7 @@ pub(crate) fn build_plugin_injections( .iter() .any(|plugin_name| plugin_name == &plugin.display_name) }) - .map(connectors::connector_display_label) + .map(connector_display_label) .collect::>() .into_iter() .collect::>(); diff --git a/codex-rs/core/src/plugins/mentions.rs b/codex-rs/core/src/plugins/mentions.rs index 5e11b9b91e..a5e94345cb 100644 --- a/codex-rs/core/src/plugins/mentions.rs +++ b/codex-rs/core/src/plugins/mentions.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use std::collections::HashSet; +use codex_connectors::metadata::connector_mention_slug; use codex_protocol::user_input::UserInput; use crate::connectors; @@ -108,7 +109,7 @@ pub(crate) fn build_connector_slug_counts( ) -> HashMap { let mut counts: HashMap = HashMap::new(); for connector in connectors { - let slug = connectors::connector_mention_slug(connector); + let slug = connector_mention_slug(connector); *counts.entry(slug).or_insert(0) += 1; } counts diff --git a/codex-rs/tui/Cargo.toml b/codex-rs/tui/Cargo.toml index 0f29fdeae6..65f35b665a 100644 --- a/codex-rs/tui/Cargo.toml +++ b/codex-rs/tui/Cargo.toml @@ -33,6 +33,7 @@ codex-install-context = { workspace = true } codex-chatgpt = { workspace = true } codex-cloud-requirements = { workspace = true } codex-config = { workspace = true } +codex-connectors = { workspace = true } codex-exec-server = { workspace = true } codex-features = { workspace = true } codex-feedback = { workspace = true } diff --git a/codex-rs/tui/src/app_event.rs b/codex-rs/tui/src/app_event.rs index a3b31aa988..934baac231 100644 --- a/codex-rs/tui/src/app_event.rs +++ b/codex-rs/tui/src/app_event.rs @@ -10,13 +10,13 @@ use std::path::PathBuf; +use codex_app_server_protocol::AppInfo; use codex_app_server_protocol::McpServerStatus; use codex_app_server_protocol::PluginInstallResponse; use codex_app_server_protocol::PluginListResponse; use codex_app_server_protocol::PluginReadParams; use codex_app_server_protocol::PluginReadResponse; use codex_app_server_protocol::PluginUninstallResponse; -use codex_chatgpt::connectors::AppInfo; use codex_file_search::FileMatch; use codex_protocol::ThreadId; use codex_protocol::openai_models::ModelPreset; diff --git a/codex-rs/tui/src/bottom_pane/chat_composer.rs b/codex-rs/tui/src/bottom_pane/chat_composer.rs index b1c3129ff6..5b39a0f01d 100644 --- a/codex-rs/tui/src/bottom_pane/chat_composer.rs +++ b/codex-rs/tui/src/bottom_pane/chat_composer.rs @@ -209,8 +209,7 @@ use crate::legacy_core::plugins::PluginCapabilitySummary; use crate::legacy_core::skills::model::SkillMetadata; use crate::tui::FrameRequester; use crate::ui_consts::LIVE_PREFIX_COLS; -use codex_chatgpt::connectors; -use codex_chatgpt::connectors::AppInfo; +use codex_app_server_protocol::AppInfo; use codex_file_search::FileMatch; use std::cell::RefCell; use std::collections::HashMap; @@ -3417,9 +3416,9 @@ impl ChatComposer { if !connector.is_accessible || !connector.is_enabled { continue; } - let display_name = connectors::connector_display_label(connector); + let display_name = codex_connectors::metadata::connector_display_label(connector); let description = Some(Self::connector_brief_description(connector)); - let slug = crate::legacy_core::connectors::connector_mention_slug(connector); + let slug = codex_connectors::metadata::connector_mention_slug(connector); let search_terms = vec![display_name.clone(), connector.id.clone(), slug.clone()]; let connector_id = connector.id.as_str(); mentions.push(MentionItem { diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 05f1bb5ccc..f2697d9072 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -79,6 +79,7 @@ use crate::terminal_title::clear_terminal_title; use crate::terminal_title::set_terminal_title; use crate::text_formatting::proper_join; use crate::version::CODEX_CLI_VERSION; +use codex_app_server_protocol::AppInfo; use codex_app_server_protocol::AppSummary; use codex_app_server_protocol::CodexErrorInfo as AppServerCodexErrorInfo; use codex_app_server_protocol::CollabAgentState as AppServerCollabAgentState; @@ -5525,7 +5526,7 @@ impl ChatWidget { let app_mentions = find_app_mentions(&mentions, apps, &skill_names_lower); for app in app_mentions { - let slug = crate::legacy_core::connectors::connector_mention_slug(&app); + let slug = codex_connectors::metadata::connector_mention_slug(&app); if bound_names.contains(&slug) || !selected_app_ids.insert(app.id.clone()) { continue; } @@ -9701,7 +9702,7 @@ impl ChatWidget { self.config.features.enabled(Feature::Apps) && self.has_chatgpt_account } - fn connectors_for_mentions(&self) -> Option<&[connectors::AppInfo]> { + fn connectors_for_mentions(&self) -> Option<&[AppInfo]> { if !self.connectors_enabled() { return None; } @@ -9888,7 +9889,7 @@ impl ChatWidget { } } - fn open_connectors_popup(&mut self, connectors: &[connectors::AppInfo]) { + fn open_connectors_popup(&mut self, connectors: &[AppInfo]) { self.bottom_pane.show_selection_view( self.connectors_popup_params(connectors, /*selected_connector_id*/ None), ); @@ -9914,7 +9915,7 @@ impl ChatWidget { fn connectors_popup_params( &self, - connectors: &[connectors::AppInfo], + connectors: &[AppInfo], selected_connector_id: Option<&str>, ) -> SelectionViewParams { let total = connectors.len(); @@ -9937,7 +9938,7 @@ impl ChatWidget { }); let mut items: Vec = Vec::with_capacity(connectors.len()); for connector in connectors { - let connector_label = connectors::connector_display_label(connector); + let connector_label = codex_connectors::metadata::connector_display_label(connector); let connector_title = connector_label.clone(); let link_description = Self::connector_description(connector); let description = Self::connector_brief_description(connector); @@ -10011,7 +10012,7 @@ impl ChatWidget { } } - fn refresh_connectors_popup_if_open(&mut self, connectors: &[connectors::AppInfo]) { + fn refresh_connectors_popup_if_open(&mut self, connectors: &[AppInfo]) { let selected_connector_id = if let (Some(selected_index), ConnectorsCacheState::Ready(snapshot)) = ( self.bottom_pane @@ -10039,7 +10040,7 @@ impl ChatWidget { ]) } - fn connector_brief_description(connector: &connectors::AppInfo) -> String { + fn connector_brief_description(connector: &AppInfo) -> String { let status_label = Self::connector_status_label(connector); match Self::connector_description(connector) { Some(description) => format!("{status_label} · {description}"), @@ -10047,7 +10048,7 @@ impl ChatWidget { } } - fn connector_status_label(connector: &connectors::AppInfo) -> &'static str { + fn connector_status_label(connector: &AppInfo) -> &'static str { if connector.is_accessible { if connector.is_enabled { "Installed" @@ -10059,7 +10060,7 @@ impl ChatWidget { } } - fn connector_description(connector: &connectors::AppInfo) -> Option { + fn connector_description(connector: &AppInfo) -> Option { connector .description .as_deref() diff --git a/codex-rs/tui/src/chatwidget/skills.rs b/codex-rs/tui/src/chatwidget/skills.rs index 0bcfb3b3bd..9bbde1aaec 100644 --- a/codex-rs/tui/src/chatwidget/skills.rs +++ b/codex-rs/tui/src/chatwidget/skills.rs @@ -9,14 +9,13 @@ use crate::bottom_pane::SkillsToggleItem; use crate::bottom_pane::SkillsToggleView; use crate::bottom_pane::popup_consts::standard_popup_hint_line; use crate::legacy_core::TOOL_MENTION_SIGIL; -use crate::legacy_core::connectors::connector_mention_slug; use crate::legacy_core::skills::model::SkillDependencies; use crate::legacy_core::skills::model::SkillInterface; use crate::legacy_core::skills::model::SkillMetadata; use crate::legacy_core::skills::model::SkillToolDependency; use crate::skills_helpers::skill_description; use crate::skills_helpers::skill_display_name; -use codex_chatgpt::connectors::AppInfo; +use codex_app_server_protocol::AppInfo; use codex_protocol::parse_command::ParsedCommand; use codex_protocol::protocol::ListSkillsResponseEvent; use codex_protocol::protocol::SkillMetadata as ProtocolSkillMetadata; @@ -296,12 +295,12 @@ pub(crate) fn find_app_mentions( let mut slug_counts: HashMap = HashMap::new(); for app in apps.iter().filter(|app| app.is_enabled) { - let slug = connector_mention_slug(app); + let slug = codex_connectors::metadata::connector_mention_slug(app); *slug_counts.entry(slug).or_insert(0) += 1; } for app in apps.iter().filter(|app| app.is_enabled) { - let slug = connector_mention_slug(app); + let slug = codex_connectors::metadata::connector_mention_slug(app); let slug_count = slug_counts.get(&slug).copied().unwrap_or(0); if mentions.names.contains(&slug) && !explicit_names.contains(&slug) diff --git a/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs b/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs index 375c37314b..10e3451d03 100644 --- a/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs +++ b/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs @@ -1,4 +1,5 @@ use super::*; +use codex_app_server_protocol::AppInfo; use pretty_assertions::assert_eq; #[tokio::test] @@ -511,7 +512,7 @@ async fn apps_popup_stays_loading_until_final_snapshot_updates() { chat.on_connectors_loaded( Ok(ConnectorsSnapshot { - connectors: vec![codex_chatgpt::connectors::AppInfo { + connectors: vec![AppInfo { id: notion_id.to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -545,7 +546,7 @@ async fn apps_popup_stays_loading_until_final_snapshot_updates() { chat.on_connectors_loaded( Ok(ConnectorsSnapshot { connectors: vec![ - codex_chatgpt::connectors::AppInfo { + AppInfo { id: notion_id.to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -560,7 +561,7 @@ async fn apps_popup_stays_loading_until_final_snapshot_updates() { is_enabled: true, plugin_display_names: Vec::new(), }, - codex_chatgpt::connectors::AppInfo { + AppInfo { id: linear_id.to_string(), name: "Linear".to_string(), description: Some("Project tracking".to_string()), @@ -604,7 +605,7 @@ async fn apps_refresh_failure_keeps_existing_full_snapshot() { let linear_id = "unit_test_apps_refresh_failure_connector_2"; let full_connectors = vec![ - codex_chatgpt::connectors::AppInfo { + AppInfo { id: notion_id.to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -619,7 +620,7 @@ async fn apps_refresh_failure_keeps_existing_full_snapshot() { is_enabled: true, plugin_display_names: Vec::new(), }, - codex_chatgpt::connectors::AppInfo { + AppInfo { id: linear_id.to_string(), name: "Linear".to_string(), description: Some("Project tracking".to_string()), @@ -644,7 +645,7 @@ async fn apps_refresh_failure_keeps_existing_full_snapshot() { chat.on_connectors_loaded( Ok(ConnectorsSnapshot { - connectors: vec![codex_chatgpt::connectors::AppInfo { + connectors: vec![AppInfo { id: notion_id.to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -693,7 +694,7 @@ async fn apps_popup_preserves_selected_app_across_refresh() { chat.on_connectors_loaded( Ok(ConnectorsSnapshot { connectors: vec![ - codex_chatgpt::connectors::AppInfo { + AppInfo { id: "notion".to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -708,7 +709,7 @@ async fn apps_popup_preserves_selected_app_across_refresh() { is_enabled: true, plugin_display_names: Vec::new(), }, - codex_chatgpt::connectors::AppInfo { + AppInfo { id: "slack".to_string(), name: "Slack".to_string(), description: Some("Team chat".to_string()), @@ -739,7 +740,7 @@ async fn apps_popup_preserves_selected_app_across_refresh() { chat.on_connectors_loaded( Ok(ConnectorsSnapshot { connectors: vec![ - codex_chatgpt::connectors::AppInfo { + AppInfo { id: "airtable".to_string(), name: "Airtable".to_string(), description: Some("Spreadsheets".to_string()), @@ -754,7 +755,7 @@ async fn apps_popup_preserves_selected_app_across_refresh() { is_enabled: true, plugin_display_names: Vec::new(), }, - codex_chatgpt::connectors::AppInfo { + AppInfo { id: "notion".to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -769,7 +770,7 @@ async fn apps_popup_preserves_selected_app_across_refresh() { is_enabled: true, plugin_display_names: Vec::new(), }, - codex_chatgpt::connectors::AppInfo { + AppInfo { id: "slack".to_string(), name: "Slack".to_string(), description: Some("Team chat".to_string()), @@ -812,7 +813,7 @@ async fn apps_refresh_failure_with_cached_snapshot_triggers_pending_force_refetc chat.connectors_prefetch_in_flight = true; chat.connectors_force_refetch_pending = true; - let full_connectors = vec![codex_chatgpt::connectors::AppInfo { + let full_connectors = vec![AppInfo { id: "unit_test_apps_refresh_failure_pending_connector".to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -855,7 +856,7 @@ async fn apps_popup_keeps_existing_full_snapshot_while_partial_refresh_loads() { chat.bottom_pane.set_connectors_enabled(/*enabled*/ true); let full_connectors = vec![ - codex_chatgpt::connectors::AppInfo { + AppInfo { id: "unit_test_connector_1".to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -870,7 +871,7 @@ async fn apps_popup_keeps_existing_full_snapshot_while_partial_refresh_loads() { is_enabled: true, plugin_display_names: Vec::new(), }, - codex_chatgpt::connectors::AppInfo { + AppInfo { id: "unit_test_connector_2".to_string(), name: "Linear".to_string(), description: Some("Project tracking".to_string()), @@ -897,7 +898,7 @@ async fn apps_popup_keeps_existing_full_snapshot_while_partial_refresh_loads() { chat.on_connectors_loaded( Ok(ConnectorsSnapshot { connectors: vec![ - codex_chatgpt::connectors::AppInfo { + AppInfo { id: "unit_test_connector_1".to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -912,7 +913,7 @@ async fn apps_popup_keeps_existing_full_snapshot_while_partial_refresh_loads() { is_enabled: true, plugin_display_names: Vec::new(), }, - codex_chatgpt::connectors::AppInfo { + AppInfo { id: "connector_openai_hidden".to_string(), name: "Hidden OpenAI".to_string(), description: Some("Should be filtered".to_string()), @@ -960,7 +961,7 @@ async fn apps_refresh_failure_without_full_snapshot_falls_back_to_installed_apps chat.on_connectors_loaded( Ok(ConnectorsSnapshot { - connectors: vec![codex_chatgpt::connectors::AppInfo { + connectors: vec![AppInfo { id: "unit_test_apps_refresh_failure_fallback_connector".to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -1019,7 +1020,7 @@ async fn apps_popup_shows_disabled_status_for_installed_but_disabled_apps() { chat.on_connectors_loaded( Ok(ConnectorsSnapshot { - connectors: vec![codex_chatgpt::connectors::AppInfo { + connectors: vec![AppInfo { id: "connector_1".to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -1073,7 +1074,7 @@ async fn apps_initial_load_applies_enabled_state_from_config() { chat.on_connectors_loaded( Ok(ConnectorsSnapshot { - connectors: vec![codex_chatgpt::connectors::AppInfo { + connectors: vec![AppInfo { id: "connector_1".to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -1139,7 +1140,7 @@ async fn apps_initial_load_applies_enabled_state_from_requirements_with_user_ove chat.on_connectors_loaded( Ok(ConnectorsSnapshot { - connectors: vec![codex_chatgpt::connectors::AppInfo { + connectors: vec![AppInfo { id: "connector_1".to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -1203,7 +1204,7 @@ async fn apps_initial_load_applies_enabled_state_from_requirements_without_user_ chat.on_connectors_loaded( Ok(ConnectorsSnapshot { - connectors: vec![codex_chatgpt::connectors::AppInfo { + connectors: vec![AppInfo { id: "connector_1".to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -1252,7 +1253,7 @@ async fn apps_refresh_preserves_toggled_enabled_state() { chat.on_connectors_loaded( Ok(ConnectorsSnapshot { - connectors: vec![codex_chatgpt::connectors::AppInfo { + connectors: vec![AppInfo { id: "connector_1".to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -1274,7 +1275,7 @@ async fn apps_refresh_preserves_toggled_enabled_state() { chat.on_connectors_loaded( Ok(ConnectorsSnapshot { - connectors: vec![codex_chatgpt::connectors::AppInfo { + connectors: vec![AppInfo { id: "connector_1".to_string(), name: "Notion".to_string(), description: Some("Workspace docs".to_string()), @@ -1323,7 +1324,7 @@ async fn apps_popup_for_not_installed_app_uses_install_only_selected_description chat.on_connectors_loaded( Ok(ConnectorsSnapshot { - connectors: vec![codex_chatgpt::connectors::AppInfo { + connectors: vec![AppInfo { id: "connector_2".to_string(), name: "Linear".to_string(), description: Some("Project tracking".to_string()), From 71174574adb09a90ebd83e2acfe284a39aaca2cf Mon Sep 17 00:00:00 2001 From: Matthew Zeng Date: Thu, 16 Apr 2026 11:18:07 -0700 Subject: [PATCH 32/59] Add server-level approval defaults for custom MCP servers (#17843) ## Summary - Add `default_tools_approval_mode` support for custom MCP server configs, matching the existing `codex_apps` behavior - Apply approval precedence as per-tool override, then server default, then `auto` - Update config serialization, CLI display, schema generation, docs, and tests ## Testing - `cargo check -p codex-config` - `cargo check -p codex-core` - `just write-config-schema` - `just fmt` - `cargo test -p codex-config` - Targeted `codex-core` tests for config parsing, config writes, and MCP approval precedence - `just fix -p codex-config -p codex-core` --- codex-rs/cli/src/mcp_cmd.rs | 10 +++++ codex-rs/codex-mcp/src/mcp/mod.rs | 1 + codex-rs/codex-mcp/src/mcp/mod_tests.rs | 2 + .../codex-mcp/src/mcp/skill_dependencies.rs | 2 + .../src/mcp/skill_dependencies_tests.rs | 2 + .../src/mcp_connection_manager_tests.rs | 2 + codex-rs/config/src/mcp_edit.rs | 7 +++ codex-rs/config/src/mcp_edit_tests.rs | 2 + codex-rs/config/src/mcp_types.rs | 8 ++++ codex-rs/config/src/mcp_types_tests.rs | 33 ++++++++++++++ codex-rs/core/config.schema.json | 8 ++++ codex-rs/core/src/config/config_tests.rs | 33 +++++++++++--- codex-rs/core/src/config/edit.rs | 7 +++ codex-rs/core/src/config/edit_tests.rs | 8 ++++ codex-rs/core/src/config/service_tests.rs | 43 +++++++++++++++++++ codex-rs/core/src/mcp_skill_dependencies.rs | 2 + codex-rs/core/src/mcp_tool_call.rs | 11 +++-- codex-rs/core/src/mcp_tool_call_tests.rs | 37 ++++++++++++++++ codex-rs/core/src/plugins/manager_tests.rs | 4 ++ codex-rs/core/tests/suite/code_mode.rs | 1 + codex-rs/core/tests/suite/js_repl.rs | 1 + codex-rs/core/tests/suite/rmcp_client.rs | 1 + codex-rs/core/tests/suite/search_tool.rs | 1 + codex-rs/core/tests/suite/sqlite_state.rs | 1 + codex-rs/core/tests/suite/tools.rs | 1 + codex-rs/core/tests/suite/truncation.rs | 3 ++ docs/config.md | 12 ++++-- 27 files changed, 231 insertions(+), 12 deletions(-) diff --git a/codex-rs/cli/src/mcp_cmd.rs b/codex-rs/cli/src/mcp_cmd.rs index 4ea084b83f..d413f72ddc 100644 --- a/codex-rs/cli/src/mcp_cmd.rs +++ b/codex-rs/cli/src/mcp_cmd.rs @@ -6,6 +6,7 @@ use anyhow::Result; use anyhow::anyhow; use anyhow::bail; use clap::ArgGroup; +use codex_config::types::AppToolApproval; use codex_config::types::McpServerConfig; use codex_config::types::McpServerTransportConfig; use codex_core::McpManager; @@ -304,6 +305,7 @@ async fn run_add(config_overrides: &CliConfigOverrides, add_args: AddArgs) -> Re disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -879,6 +881,14 @@ async fn run_get(config_overrides: &CliConfigOverrides, get_args: GetArgs) -> Re if let Some(timeout) = server.tool_timeout_sec { println!(" tool_timeout_sec: {}", timeout.as_secs_f64()); } + if let Some(approval_mode) = server.default_tools_approval_mode { + let approval_mode = match approval_mode { + AppToolApproval::Auto => "auto", + AppToolApproval::Prompt => "prompt", + AppToolApproval::Approve => "approve", + }; + println!(" default_tools_approval_mode: {approval_mode}"); + } println!(" remove: codex mcp remove {}", get_args.name); Ok(()) diff --git a/codex-rs/codex-mcp/src/mcp/mod.rs b/codex-rs/codex-mcp/src/mcp/mod.rs index db90bfe429..1dc9db0789 100644 --- a/codex-rs/codex-mcp/src/mcp/mod.rs +++ b/codex-rs/codex-mcp/src/mcp/mod.rs @@ -276,6 +276,7 @@ fn codex_apps_mcp_server_config(config: &McpConfig, auth: Option<&CodexAuth>) -> disabled_reason: None, startup_timeout_sec: Some(Duration::from_secs(30)), tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, diff --git a/codex-rs/codex-mcp/src/mcp/mod_tests.rs b/codex-rs/codex-mcp/src/mcp/mod_tests.rs index c1d39f2a7c..8db52a9d83 100644 --- a/codex-rs/codex-mcp/src/mcp/mod_tests.rs +++ b/codex-rs/codex-mcp/src/mcp/mod_tests.rs @@ -200,6 +200,7 @@ async fn effective_mcp_servers_preserve_user_servers_and_add_codex_apps() { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -223,6 +224,7 @@ async fn effective_mcp_servers_preserve_user_servers_and_add_codex_apps() { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, diff --git a/codex-rs/codex-mcp/src/mcp/skill_dependencies.rs b/codex-rs/codex-mcp/src/mcp/skill_dependencies.rs index ab72e25958..f785fe4bde 100644 --- a/codex-rs/codex-mcp/src/mcp/skill_dependencies.rs +++ b/codex-rs/codex-mcp/src/mcp/skill_dependencies.rs @@ -126,6 +126,7 @@ fn mcp_dependency_to_server_config( disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -154,6 +155,7 @@ fn mcp_dependency_to_server_config( disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, diff --git a/codex-rs/codex-mcp/src/mcp/skill_dependencies_tests.rs b/codex-rs/codex-mcp/src/mcp/skill_dependencies_tests.rs index b562bf9e2c..2d8390d15e 100644 --- a/codex-rs/codex-mcp/src/mcp/skill_dependencies_tests.rs +++ b/codex-rs/codex-mcp/src/mcp/skill_dependencies_tests.rs @@ -46,6 +46,7 @@ fn collect_missing_respects_canonical_installed_key() { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -98,6 +99,7 @@ fn collect_missing_dedupes_by_canonical_key_but_preserves_original_name() { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, diff --git a/codex-rs/codex-mcp/src/mcp_connection_manager_tests.rs b/codex-rs/codex-mcp/src/mcp_connection_manager_tests.rs index 6b65d455a8..cf2889ccde 100644 --- a/codex-rs/codex-mcp/src/mcp_connection_manager_tests.rs +++ b/codex-rs/codex-mcp/src/mcp_connection_manager_tests.rs @@ -799,6 +799,7 @@ fn mcp_init_error_display_prompts_for_github_pat() { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -850,6 +851,7 @@ fn mcp_init_error_display_reports_generic_errors() { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, diff --git a/codex-rs/config/src/mcp_edit.rs b/codex-rs/config/src/mcp_edit.rs index d315d23f8e..c4bb38c543 100644 --- a/codex-rs/config/src/mcp_edit.rs +++ b/codex-rs/config/src/mcp_edit.rs @@ -189,6 +189,13 @@ fn serialize_mcp_server(config: &McpServerConfig) -> TomlItem { if let Some(timeout) = config.tool_timeout_sec { entry["tool_timeout_sec"] = value(timeout.as_secs_f64()); } + if let Some(approval_mode) = config.default_tools_approval_mode { + entry["default_tools_approval_mode"] = value(match approval_mode { + AppToolApproval::Auto => "auto", + AppToolApproval::Prompt => "prompt", + AppToolApproval::Approve => "approve", + }); + } if let Some(enabled_tools) = &config.enabled_tools && !enabled_tools.is_empty() { diff --git a/codex-rs/config/src/mcp_edit_tests.rs b/codex-rs/config/src/mcp_edit_tests.rs index 5f60c2612a..cfcd73c3e5 100644 --- a/codex-rs/config/src/mcp_edit_tests.rs +++ b/codex-rs/config/src/mcp_edit_tests.rs @@ -29,6 +29,7 @@ async fn replace_mcp_servers_serializes_per_tool_approval_overrides() -> anyhow: disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: Some(AppToolApproval::Auto), enabled_tools: None, disabled_tools: None, scopes: None, @@ -62,6 +63,7 @@ async fn replace_mcp_servers_serializes_per_tool_approval_overrides() -> anyhow: r#"[mcp_servers.docs] command = "docs-server" supports_parallel_tool_calls = true +default_tools_approval_mode = "auto" [mcp_servers.docs.tools] diff --git a/codex-rs/config/src/mcp_types.rs b/codex-rs/config/src/mcp_types.rs index 8aece35c0d..75b68c3f94 100644 --- a/codex-rs/config/src/mcp_types.rs +++ b/codex-rs/config/src/mcp_types.rs @@ -93,6 +93,10 @@ pub struct McpServerConfig { #[serde(default, with = "option_duration_secs")] pub tool_timeout_sec: Option, + /// Approval mode for tools in this server unless a tool override exists. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub default_tools_approval_mode: Option, + /// Explicit allow-list of tools exposed from this server. When set, only these tools will be registered. #[serde(default, skip_serializing_if = "Option::is_none")] pub enabled_tools: Option>, @@ -158,6 +162,8 @@ pub struct RawMcpServerConfig { #[serde(default)] pub supports_parallel_tool_calls: Option, #[serde(default)] + pub default_tools_approval_mode: Option, + #[serde(default)] pub enabled_tools: Option>, #[serde(default)] pub disabled_tools: Option>, @@ -194,6 +200,7 @@ impl TryFrom for McpServerConfig { enabled, required, supports_parallel_tool_calls, + default_tools_approval_mode, enabled_tools, disabled_tools, scopes, @@ -260,6 +267,7 @@ impl TryFrom for McpServerConfig { required: required.unwrap_or_default(), supports_parallel_tool_calls: supports_parallel_tool_calls.unwrap_or_default(), disabled_reason: None, + default_tools_approval_mode, enabled_tools, disabled_tools, scopes, diff --git a/codex-rs/config/src/mcp_types_tests.rs b/codex-rs/config/src/mcp_types_tests.rs index 500ea56f62..dff4a6bbf5 100644 --- a/codex-rs/config/src/mcp_types_tests.rs +++ b/codex-rs/config/src/mcp_types_tests.rs @@ -258,6 +258,38 @@ fn deserialize_server_config_with_parallel_tool_calls() { assert!(cfg.supports_parallel_tool_calls); } +#[test] +fn deserialize_server_config_with_default_tool_approval_mode() { + let cfg: McpServerConfig = toml::from_str( + r#" + command = "echo" + default_tools_approval_mode = "approve" + + [tools.search] + approval_mode = "prompt" + "#, + ) + .expect("should deserialize default tool approval mode"); + + assert_eq!( + cfg.default_tools_approval_mode, + Some(AppToolApproval::Approve) + ); + assert_eq!( + cfg.tools.get("search"), + Some(&McpServerToolConfig { + approval_mode: Some(AppToolApproval::Prompt), + }) + ); + + let serialized = toml::to_string(&cfg).expect("should serialize MCP config"); + assert!(serialized.contains("default_tools_approval_mode = \"approve\"")); + + let round_tripped: McpServerConfig = + toml::from_str(&serialized).expect("should deserialize serialized MCP config"); + assert_eq!(round_tripped, cfg); +} + #[test] fn serialize_round_trips_server_config_with_parallel_tool_calls() { let cfg: McpServerConfig = toml::from_str( @@ -304,6 +336,7 @@ fn deserialize_ignores_unknown_server_fields() { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.json index 9c7eb853a9..966555856b 100644 --- a/codex-rs/core/config.schema.json +++ b/codex-rs/core/config.schema.json @@ -1463,6 +1463,14 @@ "default": null, "type": "string" }, + "default_tools_approval_mode": { + "allOf": [ + { + "$ref": "#/definitions/AppToolApproval" + } + ], + "default": null + }, "disabled_tools": { "default": null, "items": { diff --git a/codex-rs/core/src/config/config_tests.rs b/codex-rs/core/src/config/config_tests.rs index 0001439816..34002b129a 100644 --- a/codex-rs/core/src/config/config_tests.rs +++ b/codex-rs/core/src/config/config_tests.rs @@ -90,6 +90,7 @@ fn stdio_mcp(command: &str) -> McpServerConfig { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -113,6 +114,7 @@ fn http_mcp(url: &str) -> McpServerConfig { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -2130,6 +2132,7 @@ async fn replace_mcp_servers_round_trips_entries() -> anyhow::Result<()> { disabled_reason: None, startup_timeout_sec: Some(Duration::from_secs(3)), tool_timeout_sec: Some(Duration::from_secs(5)), + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -2243,23 +2246,28 @@ fn mcp_servers_toml_parses_per_tool_approval_overrides() { [mcp_servers.docs] command = "docs-server" name = "Docs" +default_tools_approval_mode = "prompt" [mcp_servers.docs.tools.search] approval_mode = "approve" "#, ) .expect("TOML deserialization should succeed"); - let tool = config + let server = config .mcp_servers .get("docs") - .and_then(|server| server.tools.get("search")) - .expect("docs/search tool config exists"); + .expect("docs server config exists"); assert_eq!( - tool, - &McpServerToolConfig { + server.default_tools_approval_mode, + Some(AppToolApproval::Prompt) + ); + + assert_eq!( + server.tools.get("search"), + Some(&McpServerToolConfig { approval_mode: Some(AppToolApproval::Approve), - } + }) ); } @@ -2380,6 +2388,7 @@ async fn replace_mcp_servers_serializes_env_sorted() -> anyhow::Result<()> { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -2455,6 +2464,7 @@ async fn replace_mcp_servers_serializes_env_vars() -> anyhow::Result<()> { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -2510,6 +2520,7 @@ async fn replace_mcp_servers_serializes_cwd() -> anyhow::Result<()> { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -2563,6 +2574,7 @@ async fn replace_mcp_servers_streamable_http_serializes_bearer_token() -> anyhow disabled_reason: None, startup_timeout_sec: Some(Duration::from_secs(2)), tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -2632,6 +2644,7 @@ async fn replace_mcp_servers_streamable_http_serializes_custom_headers() -> anyh disabled_reason: None, startup_timeout_sec: Some(Duration::from_secs(2)), tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -2713,6 +2726,7 @@ async fn replace_mcp_servers_streamable_http_removes_optional_sections() -> anyh disabled_reason: None, startup_timeout_sec: Some(Duration::from_secs(2)), tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -2747,6 +2761,7 @@ async fn replace_mcp_servers_streamable_http_removes_optional_sections() -> anyh disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -2816,6 +2831,7 @@ async fn replace_mcp_servers_streamable_http_isolates_headers_between_servers() disabled_reason: None, startup_timeout_sec: Some(Duration::from_secs(2)), tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -2840,6 +2856,7 @@ async fn replace_mcp_servers_streamable_http_isolates_headers_between_servers() disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -2927,6 +2944,7 @@ async fn replace_mcp_servers_serializes_disabled_flag() -> anyhow::Result<()> { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -2976,6 +2994,7 @@ async fn replace_mcp_servers_serializes_required_flag() -> anyhow::Result<()> { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -3025,6 +3044,7 @@ async fn replace_mcp_servers_serializes_tool_filters() -> anyhow::Result<()> { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: Some(vec!["allowed".to_string()]), disabled_tools: Some(vec!["blocked".to_string()]), scopes: None, @@ -3078,6 +3098,7 @@ async fn replace_mcp_servers_streamable_http_serializes_oauth_resource() -> anyh disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, diff --git a/codex-rs/core/src/config/edit.rs b/codex-rs/core/src/config/edit.rs index 6f4231f908..5adc0f998c 100644 --- a/codex-rs/core/src/config/edit.rs +++ b/codex-rs/core/src/config/edit.rs @@ -237,6 +237,13 @@ mod document_helpers { if let Some(timeout) = config.tool_timeout_sec { entry["tool_timeout_sec"] = value(timeout.as_secs_f64()); } + if let Some(approval_mode) = config.default_tools_approval_mode { + entry["default_tools_approval_mode"] = value(match approval_mode { + AppToolApproval::Auto => "auto", + AppToolApproval::Prompt => "prompt", + AppToolApproval::Approve => "approve", + }); + } if let Some(enabled_tools) = &config.enabled_tools && !enabled_tools.is_empty() { diff --git a/codex-rs/core/src/config/edit_tests.rs b/codex-rs/core/src/config/edit_tests.rs index 46680d5434..4f340d89b3 100644 --- a/codex-rs/core/src/config/edit_tests.rs +++ b/codex-rs/core/src/config/edit_tests.rs @@ -582,6 +582,7 @@ fn blocking_replace_mcp_servers_round_trips() { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: Some(vec!["one".to_string(), "two".to_string()]), disabled_tools: None, scopes: None, @@ -610,6 +611,7 @@ fn blocking_replace_mcp_servers_round_trips() { disabled_reason: None, startup_timeout_sec: Some(std::time::Duration::from_secs(5)), tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: Some(vec!["forbidden".to_string()]), scopes: None, @@ -675,6 +677,7 @@ fn blocking_replace_mcp_servers_serializes_tool_approval_overrides() { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: Some(AppToolApproval::Prompt), enabled_tools: None, disabled_tools: None, scopes: None, @@ -699,6 +702,7 @@ fn blocking_replace_mcp_servers_serializes_tool_approval_overrides() { let expected = "\ [mcp_servers.docs] command = \"docs-server\" +default_tools_approval_mode = \"prompt\" [mcp_servers.docs.tools.search] approval_mode = \"approve\" @@ -737,6 +741,7 @@ foo = { command = "cmd" } disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -790,6 +795,7 @@ foo = { command = "cmd" } # keep me disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -842,6 +848,7 @@ foo = { command = "cmd", args = ["--flag"] } # keep me disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -895,6 +902,7 @@ foo = { command = "cmd" } disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, diff --git a/codex-rs/core/src/config/service_tests.rs b/codex-rs/core/src/config/service_tests.rs index ae62326605..faaffb34df 100644 --- a/codex-rs/core/src/config/service_tests.rs +++ b/codex-rs/core/src/config/service_tests.rs @@ -164,6 +164,49 @@ async fn write_value_supports_nested_app_paths() -> Result<()> { Ok(()) } +#[tokio::test] +async fn write_value_supports_custom_mcp_server_default_tool_approval_mode() -> Result<()> { + let tmp = tempdir().expect("tempdir"); + std::fs::write( + tmp.path().join(CONFIG_TOML_FILE), + "[mcp_servers.docs]\ncommand = \"docs-server\"\n", + )?; + + let service = ConfigService::without_managed_config_for_tests(tmp.path().to_path_buf()); + service + .write_value(ConfigValueWriteParams { + file_path: Some(tmp.path().join(CONFIG_TOML_FILE).display().to_string()), + key_path: "mcp_servers.docs.default_tools_approval_mode".to_string(), + value: serde_json::json!("approve"), + merge_strategy: MergeStrategy::Replace, + expected_version: None, + }) + .await + .expect("write mcp server default_tools_approval_mode succeeds"); + + let contents = std::fs::read_to_string(tmp.path().join(CONFIG_TOML_FILE))?; + assert!(contents.contains("default_tools_approval_mode = \"approve\"")); + + let read = service + .read(ConfigReadParams { + include_layers: false, + cwd: None, + }) + .await + .expect("config read succeeds"); + + assert_eq!( + read.config + .additional + .get("mcp_servers") + .and_then(|servers| servers.get("docs")) + .and_then(|docs| docs.get("default_tools_approval_mode")), + Some(&serde_json::json!("approve")) + ); + + Ok(()) +} + #[tokio::test] async fn read_includes_origins_and_layers() { let tmp = tempdir().expect("tempdir"); diff --git a/codex-rs/core/src/mcp_skill_dependencies.rs b/codex-rs/core/src/mcp_skill_dependencies.rs index c87b02730c..c12fe28cb2 100644 --- a/codex-rs/core/src/mcp_skill_dependencies.rs +++ b/codex-rs/core/src/mcp_skill_dependencies.rs @@ -371,6 +371,7 @@ fn mcp_dependency_to_server_config( disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -399,6 +400,7 @@ fn mcp_dependency_to_server_config( disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, diff --git a/codex-rs/core/src/mcp_tool_call.rs b/codex-rs/core/src/mcp_tool_call.rs index 064585f447..b707d51890 100644 --- a/codex-rs/core/src/mcp_tool_call.rs +++ b/codex-rs/core/src/mcp_tool_call.rs @@ -676,9 +676,14 @@ fn custom_mcp_tool_approval_mode( .and_then(|value| { HashMap::::deserialize(value).ok() }) - .and_then(|servers| servers.get(server).cloned()) - .and_then(|server| server.tools.get(tool_name).cloned()) - .and_then(|tool| tool.approval_mode) + .and_then(|servers| { + let server_config = servers.get(server)?; + server_config + .tools + .get(tool_name) + .and_then(|tool| tool.approval_mode) + .or(server_config.default_tools_approval_mode) + }) .unwrap_or_default() } diff --git a/codex-rs/core/src/mcp_tool_call_tests.rs b/codex-rs/core/src/mcp_tool_call_tests.rs index 53e838908d..a85233c773 100644 --- a/codex-rs/core/src/mcp_tool_call_tests.rs +++ b/codex-rs/core/src/mcp_tool_call_tests.rs @@ -1146,6 +1146,43 @@ async fn persist_custom_mcp_tool_approval_writes_tool_override() { assert!(contents.contains("[mcp_servers.docs.tools.search]")); } +#[tokio::test] +async fn custom_mcp_tool_approval_mode_uses_server_default_with_tool_override() { + let tmp = tempdir().expect("tempdir"); + std::fs::write( + tmp.path().join(CONFIG_TOML_FILE), + r#" +[mcp_servers.docs] +command = "docs-server" +default_tools_approval_mode = "approve" + +[mcp_servers.docs.tools.search] +approval_mode = "prompt" +"#, + ) + .expect("seed config"); + let config = ConfigBuilder::default() + .codex_home(tmp.path().to_path_buf()) + .build() + .await + .expect("load config"); + let (_session, mut turn_context) = make_session_and_context().await; + turn_context.config = Arc::new(config); + + assert_eq!( + custom_mcp_tool_approval_mode(&turn_context, "docs", "read"), + AppToolApproval::Approve + ); + assert_eq!( + custom_mcp_tool_approval_mode(&turn_context, "docs", "search"), + AppToolApproval::Prompt + ); + assert_eq!( + custom_mcp_tool_approval_mode(&turn_context, "unknown", "search"), + AppToolApproval::Auto + ); +} + #[tokio::test] async fn maybe_persist_mcp_tool_approval_reloads_session_config() { let (session, turn_context) = make_session_and_context().await; diff --git a/codex-rs/core/src/plugins/manager_tests.rs b/codex-rs/core/src/plugins/manager_tests.rs index c82e1d174c..2b2f486586 100644 --- a/codex-rs/core/src/plugins/manager_tests.rs +++ b/codex-rs/core/src/plugins/manager_tests.rs @@ -183,6 +183,7 @@ async fn load_plugins_loads_default_skills_and_mcp_servers() { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -520,6 +521,7 @@ async fn load_plugins_uses_manifest_configured_component_paths() { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -630,6 +632,7 @@ async fn load_plugins_ignores_manifest_component_paths_without_dot_slash() { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -788,6 +791,7 @@ fn capability_index_filters_inactive_and_zero_capability_plugins() { disabled_reason: None, startup_timeout_sec: None, tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, diff --git a/codex-rs/core/tests/suite/code_mode.rs b/codex-rs/core/tests/suite/code_mode.rs index a428b32ba2..474d221d27 100644 --- a/codex-rs/core/tests/suite/code_mode.rs +++ b/codex-rs/core/tests/suite/code_mode.rs @@ -239,6 +239,7 @@ async fn run_code_mode_turn_with_rmcp_config( disabled_reason: None, startup_timeout_sec: Some(Duration::from_secs(10)), tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, diff --git a/codex-rs/core/tests/suite/js_repl.rs b/codex-rs/core/tests/suite/js_repl.rs index 9c5eaef191..450aa08f0b 100644 --- a/codex-rs/core/tests/suite/js_repl.rs +++ b/codex-rs/core/tests/suite/js_repl.rs @@ -629,6 +629,7 @@ async fn js_repl_can_invoke_mcp_tools_by_display_name() -> Result<()> { disabled_reason: None, startup_timeout_sec: Some(Duration::from_secs(10)), tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, diff --git a/codex-rs/core/tests/suite/rmcp_client.rs b/codex-rs/core/tests/suite/rmcp_client.rs index 607dc7bf9c..8c7e7133a3 100644 --- a/codex-rs/core/tests/suite/rmcp_client.rs +++ b/codex-rs/core/tests/suite/rmcp_client.rs @@ -151,6 +151,7 @@ fn insert_mcp_server( disabled_reason: None, startup_timeout_sec: Some(Duration::from_secs(10)), tool_timeout_sec: options.tool_timeout_sec, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, diff --git a/codex-rs/core/tests/suite/search_tool.rs b/codex-rs/core/tests/suite/search_tool.rs index eff7bfb427..41f8e623c9 100644 --- a/codex-rs/core/tests/suite/search_tool.rs +++ b/codex-rs/core/tests/suite/search_tool.rs @@ -703,6 +703,7 @@ async fn tool_search_indexes_only_enabled_non_app_mcp_tools() -> Result<()> { disabled_reason: None, startup_timeout_sec: Some(Duration::from_secs(10)), tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: Some(vec!["echo".to_string(), "image".to_string()]), disabled_tools: Some(vec!["image".to_string()]), scopes: None, diff --git a/codex-rs/core/tests/suite/sqlite_state.rs b/codex-rs/core/tests/suite/sqlite_state.rs index acdc1e9d8c..271f7bfc90 100644 --- a/codex-rs/core/tests/suite/sqlite_state.rs +++ b/codex-rs/core/tests/suite/sqlite_state.rs @@ -378,6 +378,7 @@ async fn mcp_call_marks_thread_memory_mode_polluted_when_configured() -> Result< disabled_reason: None, startup_timeout_sec: Some(Duration::from_secs(10)), tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, diff --git a/codex-rs/core/tests/suite/tools.rs b/codex-rs/core/tests/suite/tools.rs index d0bb138188..bc286ce5ea 100644 --- a/codex-rs/core/tests/suite/tools.rs +++ b/codex-rs/core/tests/suite/tools.rs @@ -162,6 +162,7 @@ async fn historical_unavailable_mcp_call_is_exposed_as_placeholder_tool() -> Res disabled_reason: None, startup_timeout_sec: Some(Duration::from_secs(10)), tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, diff --git a/codex-rs/core/tests/suite/truncation.rs b/codex-rs/core/tests/suite/truncation.rs index 33c74633f3..81c22e1b6c 100644 --- a/codex-rs/core/tests/suite/truncation.rs +++ b/codex-rs/core/tests/suite/truncation.rs @@ -382,6 +382,7 @@ async fn mcp_tool_call_output_exceeds_limit_truncated_for_model() -> Result<()> disabled_reason: None, startup_timeout_sec: Some(std::time::Duration::from_secs(10)), tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -480,6 +481,7 @@ async fn mcp_image_output_preserves_image_and_no_text_summary() -> Result<()> { disabled_reason: None, startup_timeout_sec: Some(Duration::from_secs(10)), tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, @@ -761,6 +763,7 @@ async fn mcp_tool_call_output_not_truncated_with_custom_limit() -> Result<()> { disabled_reason: None, startup_timeout_sec: Some(std::time::Duration::from_secs(10)), tool_timeout_sec: None, + default_tools_approval_mode: None, enabled_tools: None, disabled_tools: None, scopes: None, diff --git a/docs/config.md b/docs/config.md index c314ce2283..d583122e01 100644 --- a/docs/config.md +++ b/docs/config.md @@ -28,12 +28,18 @@ resources, review those read/write race conditions before enabling this setting. ## MCP tool approvals -Codex stores per-tool approval overrides for custom MCP servers under -`mcp_servers` in `~/.codex/config.toml`: +Codex stores approval defaults and per-tool overrides for custom MCP servers +under `mcp_servers` in `~/.codex/config.toml`. Set +`default_tools_approval_mode` on the server to apply a default to every tool, +and use per-tool `approval_mode` entries for exceptions: ```toml +[mcp_servers.docs] +command = "docs-server" +default_tools_approval_mode = "approve" + [mcp_servers.docs.tools.search] -approval_mode = "approve" +approval_mode = "prompt" ``` ## Apps (Connectors) From dfff8a7d0352631efe5e9424da4f148b1d8223f5 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 16 Apr 2026 12:29:25 -0700 Subject: [PATCH 33/59] fix: drop lock earlier; was held across send_event().await unnecessarily (#18178) This was flagged by the Codex Security tool: the `state` lock was held longer than necessary, which included being held across an `async` call, increasing the potential for deadlock. While this was flagged by the Codex Security tool, I will look into enabling https://rust-lang.github.io/rust-clippy/stable/index.html#await_holding_lock in a follow-up PR (though unfortunately, that Clippy rule claims it reports false positives when `drop()` is used to drop a guard instead of using the end of block scope to drop). Though I can't seem to find a Clippy rule that checks for opportunities to drop a guard as soon as it is no longer referenced, in general. --- codex-rs/core/src/codex.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 5274d88664..42f165bbf3 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -6696,13 +6696,16 @@ pub(crate) async fn run_turn( break; } Err(CodexErr::InvalidImageRequest()) => { - let mut state = sess.state.lock().await; - error_or_panic( - "Invalid image detected; sanitizing tool output to prevent poisoning", - ); - if state.history.replace_last_turn_images("Invalid image") { - continue; + { + let mut state = sess.state.lock().await; + error_or_panic( + "Invalid image detected; sanitizing tool output to prevent poisoning", + ); + if state.history.replace_last_turn_images("Invalid image") { + continue; + } } + let event = EventMsg::Error(ErrorEvent { message: "Invalid image in your last message. Please remove it and try again." .to_string(), From 62847e7554759b8a5b28e90863e94a5b2211d7db Mon Sep 17 00:00:00 2001 From: starr-openai Date: Thu, 16 Apr 2026 12:34:04 -0700 Subject: [PATCH 34/59] Make thread unsubscribe test deterministic (#18000) ## Summary - replace the unsubscribe-during-turn test's sleep/polling flow with a gated streaming SSE response - add request-count notification support to the streaming SSE test server so the test can wait for the in-flight Responses request deterministically ## Scope - codex-rs/app-server/tests/suite/v2/thread_unsubscribe.rs - codex-rs/core/tests/common/streaming_sse.rs ## Validation - Not run locally; this is a narrow extraction from the prior CI-green branch. --------- Co-authored-by: Codex --- .../tests/suite/v2/thread_unsubscribe.rs | 207 ++++++++++-------- codex-rs/core/tests/common/streaming_sse.rs | 16 ++ 2 files changed, 131 insertions(+), 92 deletions(-) diff --git a/codex-rs/app-server/tests/suite/v2/thread_unsubscribe.rs b/codex-rs/app-server/tests/suite/v2/thread_unsubscribe.rs index 7a3ae61b72..e91a8654cb 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_unsubscribe.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_unsubscribe.rs @@ -1,14 +1,15 @@ -use anyhow::Context; use anyhow::Result; use app_test_support::McpProcess; -use app_test_support::create_final_assistant_message_sse_response; use app_test_support::create_mock_responses_server_repeating_assistant; -use app_test_support::create_mock_responses_server_sequence_unchecked; -use app_test_support::create_shell_command_sse_response; use app_test_support::to_response; +use codex_app_server_protocol::DynamicToolCallOutputContentItem; +use codex_app_server_protocol::DynamicToolCallParams; +use codex_app_server_protocol::DynamicToolCallResponse; +use codex_app_server_protocol::DynamicToolSpec; use codex_app_server_protocol::ItemStartedNotification; use codex_app_server_protocol::JSONRPCResponse; use codex_app_server_protocol::RequestId; +use codex_app_server_protocol::ServerRequest; use codex_app_server_protocol::ThreadItem; use codex_app_server_protocol::ThreadLoadedListParams; use codex_app_server_protocol::ThreadLoadedListResponse; @@ -26,61 +27,14 @@ use codex_app_server_protocol::TurnStartParams; use codex_app_server_protocol::TurnStartResponse; use codex_app_server_protocol::UserInput as V2UserInput; use core_test_support::responses; +use core_test_support::streaming_sse::StreamingSseChunk; +use core_test_support::streaming_sse::start_streaming_sse_server; use pretty_assertions::assert_eq; +use serde_json::json; use tempfile::TempDir; use tokio::time::timeout; const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); -// Windows CI can spend most of the default read timeout launching PowerShell -// before the command finishes and the follow-up model request is sent. -const TURN_COMPLETION_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); - -async fn wait_for_responses_request_count_to_stabilize( - server: &wiremock::MockServer, - expected_count: usize, - settle_duration: std::time::Duration, - timeout_duration: std::time::Duration, -) -> Result<()> { - timeout(timeout_duration, async { - let mut stable_since: Option = None; - loop { - let requests = server - .received_requests() - .await - .context("failed to fetch received requests")?; - let responses_request_count = requests - .iter() - .filter(|request| { - request.method == "POST" && request.url.path().ends_with("/responses") - }) - .count(); - - if responses_request_count > expected_count { - anyhow::bail!( - "expected exactly {expected_count} /responses requests, got {responses_request_count}" - ); - } - - if responses_request_count == expected_count { - match stable_since { - Some(stable_since) if stable_since.elapsed() >= settle_duration => { - return Ok::<(), anyhow::Error>(()); - } - None => stable_since = Some(tokio::time::Instant::now()), - Some(_) => {} - } - } else { - stable_since = None; - } - - tokio::time::sleep(std::time::Duration::from_millis(10)).await; - } - }) - .await??; - - Ok(()) -} - #[tokio::test] async fn thread_unsubscribe_keeps_thread_loaded_until_idle_timeout() -> Result<()> { let server = create_mock_responses_server_repeating_assistant("Done").await; @@ -132,14 +86,10 @@ async fn thread_unsubscribe_keeps_thread_loaded_until_idle_timeout() -> Result<( #[tokio::test] async fn thread_unsubscribe_during_turn_keeps_turn_running() -> Result<()> { - #[cfg(target_os = "windows")] - let shell_command = vec![ - "Start-Sleep".to_string(), - "-Seconds".to_string(), - "1".to_string(), - ]; - #[cfg(not(target_os = "windows"))] - let shell_command = vec!["sleep".to_string(), "1".to_string()]; + let call_id = "deterministic-wait-call"; + let tool_name = "deterministic_wait"; + let tool_args = json!({}); + let tool_call_arguments = serde_json::to_string(&tool_args)?; let tmp = TempDir::new()?; let codex_home = tmp.path().join("codex_home"); @@ -147,28 +97,61 @@ async fn thread_unsubscribe_during_turn_keeps_turn_running() -> Result<()> { let working_directory = tmp.path().join("workdir"); std::fs::create_dir(&working_directory)?; - let server = create_mock_responses_server_sequence_unchecked(vec![ - create_shell_command_sse_response( - shell_command.clone(), - Some(&working_directory), - Some(10_000), - "call_sleep", - )?, - create_final_assistant_message_sse_response("Done")?, + let (server, mut completions) = start_streaming_sse_server(vec![ + vec![StreamingSseChunk { + gate: None, + body: responses::sse(vec![ + responses::ev_response_created("resp-1"), + responses::ev_function_call(call_id, tool_name, &tool_call_arguments), + responses::ev_completed("resp-1"), + ]), + }], + vec![StreamingSseChunk { + gate: None, + body: responses::sse(vec![ + responses::ev_response_created("resp-2"), + responses::ev_assistant_message("msg-1", "Done"), + responses::ev_completed("resp-2"), + ]), + }], ]) .await; - create_config_toml(&codex_home, &server.uri())?; + let first_response_completed = completions.remove(0); + let final_response_completed = completions.remove(0); + create_config_toml(&codex_home, server.uri())?; let mut mcp = McpProcess::new(&codex_home).await?; timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; - let thread_id = start_thread(&mut mcp).await?; + let thread_req = mcp + .send_thread_start_request(ThreadStartParams { + model: Some("mock-model".to_string()), + dynamic_tools: Some(vec![DynamicToolSpec { + name: tool_name.to_string(), + description: "Deterministic wait tool".to_string(), + input_schema: json!({ + "type": "object", + "properties": {}, + "additionalProperties": false, + }), + defer_loading: false, + }]), + ..Default::default() + }) + .await?; + let thread_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(thread_req)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response::(thread_resp)?; + let thread_id = thread.id; let turn_req = mcp .send_turn_start_request(TurnStartParams { thread_id: thread_id.clone(), input: vec![V2UserInput::Text { - text: "run sleep".to_string(), + text: "run deterministic tool".to_string(), text_elements: Vec::new(), }], cwd: Some(working_directory), @@ -184,9 +167,37 @@ async fn thread_unsubscribe_during_turn_keeps_turn_running() -> Result<()> { timeout( DEFAULT_READ_TIMEOUT, - wait_for_command_execution_item_started(&mut mcp), + server.wait_for_request_count(/*count*/ 1), + ) + .await?; + timeout(DEFAULT_READ_TIMEOUT, first_response_completed).await??; + + let started = timeout( + DEFAULT_READ_TIMEOUT, + wait_for_dynamic_tool_started(&mut mcp, call_id), ) .await??; + assert_eq!(started.thread_id, thread_id); + + let request = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_request_message(), + ) + .await??; + let (request_id, params) = match request { + ServerRequest::DynamicToolCall { request_id, params } => (request_id, params), + other => panic!("expected DynamicToolCall request, got {other:?}"), + }; + assert_eq!( + params, + DynamicToolCallParams { + thread_id: thread_id.clone(), + turn_id: started.turn_id, + call_id: call_id.to_string(), + tool: tool_name.to_string(), + arguments: tool_args, + } + ); let unsubscribe_id = mcp .send_thread_unsubscribe_request(ThreadUnsubscribeParams { @@ -201,22 +212,29 @@ async fn thread_unsubscribe_during_turn_keeps_turn_running() -> Result<()> { let unsubscribe = to_response::(unsubscribe_resp)?; assert_eq!(unsubscribe.status, ThreadUnsubscribeStatus::Unsubscribed); - assert!( - timeout( - std::time::Duration::from_millis(250), - mcp.read_stream_until_notification_message("thread/closed"), - ) - .await - .is_err() + let closed_while_tool_call_blocked = timeout( + std::time::Duration::from_millis(250), + mcp.read_stream_until_notification_message("thread/closed"), ); + let closed_while_tool_call_blocked = closed_while_tool_call_blocked.await; + assert!(closed_while_tool_call_blocked.is_err()); - wait_for_responses_request_count_to_stabilize( - &server, - /*expected_count*/ 2, - std::time::Duration::from_millis(200), - TURN_COMPLETION_TIMEOUT, + let response = DynamicToolCallResponse { + content_items: vec![DynamicToolCallOutputContentItem::InputText { + text: "dynamic-ok".to_string(), + }], + success: true, + }; + mcp.send_response(request_id, serde_json::to_value(response)?) + .await?; + + timeout( + DEFAULT_READ_TIMEOUT, + server.wait_for_request_count(/*count*/ 2), ) .await?; + timeout(DEFAULT_READ_TIMEOUT, final_response_completed).await??; + server.shutdown().await; Ok(()) } @@ -355,15 +373,20 @@ async fn thread_unsubscribe_reports_not_subscribed_before_idle_unload() -> Resul Ok(()) } -async fn wait_for_command_execution_item_started(mcp: &mut McpProcess) -> Result<()> { +async fn wait_for_dynamic_tool_started( + mcp: &mut McpProcess, + call_id: &str, +) -> Result { loop { - let started_notif = mcp + let notification = mcp .read_stream_until_notification_message("item/started") .await?; - let started_params = started_notif.params.context("item/started params")?; - let started: ItemStartedNotification = serde_json::from_value(started_params)?; - if let ThreadItem::CommandExecution { .. } = started.item { - return Ok(()); + let Some(params) = notification.params else { + continue; + }; + let started: ItemStartedNotification = serde_json::from_value(params)?; + if matches!(&started.item, ThreadItem::DynamicToolCall { id, .. } if id == call_id) { + return Ok(started); } } } diff --git a/codex-rs/core/tests/common/streaming_sse.rs b/codex-rs/core/tests/common/streaming_sse.rs index 82edcd39d7..a86256ed5a 100644 --- a/codex-rs/core/tests/common/streaming_sse.rs +++ b/codex-rs/core/tests/common/streaming_sse.rs @@ -7,6 +7,7 @@ use tokio::io::AsyncReadExt; use tokio::io::AsyncWriteExt; use tokio::net::TcpListener; use tokio::sync::Mutex as TokioMutex; +use tokio::sync::Notify; use tokio::sync::oneshot; /// Streaming SSE chunk payload gated by a per-chunk signal. @@ -20,6 +21,7 @@ pub struct StreamingSseChunk { pub struct StreamingSseServer { uri: String, requests: Arc>>>, + request_notify: Arc, shutdown: oneshot::Sender<()>, task: tokio::task::JoinHandle<()>, } @@ -33,6 +35,15 @@ impl StreamingSseServer { self.requests.lock().await.clone() } + pub async fn wait_for_request_count(&self, count: usize) { + loop { + if self.requests.lock().await.len() >= count { + return; + } + self.request_notify.notified().await; + } + } + pub async fn shutdown(self) { let _ = self.shutdown.send(()); let _ = self.task.await; @@ -67,7 +78,9 @@ pub async fn start_streaming_sse_server( completions: VecDeque::from(completion_senders), })); let requests = Arc::new(TokioMutex::new(Vec::new())); + let request_notify = Arc::new(Notify::new()); let requests_for_task = Arc::clone(&requests); + let request_notify_for_task = Arc::clone(&request_notify); let (shutdown_tx, mut shutdown_rx) = oneshot::channel(); let task = tokio::spawn(async move { @@ -78,6 +91,7 @@ pub async fn start_streaming_sse_server( let (mut stream, _) = accept_res.expect("accept streaming SSE connection"); let state = Arc::clone(&state); let requests = Arc::clone(&requests_for_task); + let request_notify = Arc::clone(&request_notify_for_task); tokio::spawn(async move { let (request, body_prefix) = read_http_request(&mut stream).await; let Some((method, path)) = parse_request_line(&request) else { @@ -113,6 +127,7 @@ pub async fn start_streaming_sse_server( } }; requests.lock().await.push(body); + request_notify.notify_one(); let Some((chunks, completion)) = take_next_stream(&state).await else { let _ = write_http_response(&mut stream, /*status*/ 500, "no responses queued", "text/plain").await; return; @@ -149,6 +164,7 @@ pub async fn start_streaming_sse_server( StreamingSseServer { uri, requests, + request_notify, shutdown: shutdown_tx, task, }, From 8720b7bdce451c2f805894501317579b7c667c39 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Thu, 16 Apr 2026 12:43:16 -0700 Subject: [PATCH 35/59] Add codex_hook_run analytics event (#17996) # Why Add product analytics for hook handler executions so we can understand which hooks are running, where they came from, and whether they completed, failed, stopped, or blocked work. # What - add the new `codex_hook_run` analytics event and payload plumbing in `codex-rs/analytics` - emit hook-run analytics from the shared hook completion path in `codex-rs/core` - classify hook source from the loaded hook path as `system`, `user`, `project`, or `unknown` ``` { "event_type": "codex_hook_run", "event_params": { "thread_id": "string", "turn_id": "string", "model_slug": "string", "hook_name": "string, // any HookEventName "hook_source": "system | user | project | unknown", "status": "completed | failed | stopped | blocked" } } ``` --------- Co-authored-by: Codex --- .../analytics/src/analytics_client_tests.rs | 141 ++++++++++ codex-rs/analytics/src/client.rs | 8 + codex-rs/analytics/src/events.rs | 66 +++++ codex-rs/analytics/src/facts.rs | 15 + codex-rs/analytics/src/lib.rs | 1 + codex-rs/analytics/src/reducer.rs | 14 + .../schema/json/ServerNotification.json | 21 ++ .../codex_app_server_protocol.schemas.json | 21 ++ .../codex_app_server_protocol.v2.schemas.json | 21 ++ .../json/v2/HookCompletedNotification.json | 21 ++ .../json/v2/HookStartedNotification.json | 21 ++ .../schema/typescript/v2/HookRunSummary.ts | 3 +- .../schema/typescript/v2/HookSource.ts | 5 + .../schema/typescript/v2/index.ts | 1 + .../app-server-protocol/src/protocol/v2.rs | 21 ++ codex-rs/config/src/lib.rs | 2 + codex-rs/core/src/codex.rs | 7 +- codex-rs/core/src/hook_runtime.rs | 105 ++++++- codex-rs/hooks/src/engine/discovery.rs | 264 +++++++++++------- codex-rs/hooks/src/engine/dispatcher.rs | 4 + codex-rs/hooks/src/engine/mod.rs | 2 + codex-rs/hooks/src/events/post_tool_use.rs | 1 + codex-rs/hooks/src/events/pre_tool_use.rs | 1 + codex-rs/hooks/src/events/session_start.rs | 1 + codex-rs/hooks/src/events/stop.rs | 1 + .../hooks/src/events/user_prompt_submit.rs | 1 + codex-rs/protocol/src/protocol.rs | 16 ++ codex-rs/tui/src/app.rs | 2 + codex-rs/tui/src/chatwidget.rs | 1 + codex-rs/tui/src/chatwidget/tests/helpers.rs | 2 + .../src/chatwidget/tests/status_and_layout.rs | 5 + codex-rs/tui/src/history_cell/hook_cell.rs | 1 + 32 files changed, 682 insertions(+), 114 deletions(-) create mode 100644 codex-rs/app-server-protocol/schema/typescript/v2/HookSource.ts diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs index eb46674156..4b13b2819e 100644 --- a/codex-rs/analytics/src/analytics_client_tests.rs +++ b/codex-rs/analytics/src/analytics_client_tests.rs @@ -4,6 +4,7 @@ use crate::events::CodexAppMentionedEventRequest; use crate::events::CodexAppServerClientMetadata; use crate::events::CodexAppUsedEventRequest; use crate::events::CodexCompactionEventRequest; +use crate::events::CodexHookRunEventRequest; use crate::events::CodexPluginEventRequest; use crate::events::CodexPluginUsedEventRequest; use crate::events::CodexRuntimeMetadata; @@ -12,6 +13,7 @@ use crate::events::ThreadInitializedEvent; use crate::events::ThreadInitializedEventParams; use crate::events::TrackEventRequest; use crate::events::codex_app_metadata; +use crate::events::codex_hook_run_metadata; use crate::events::codex_plugin_metadata; use crate::events::codex_plugin_used_metadata; use crate::events::subagent_thread_started_event_request; @@ -28,6 +30,8 @@ use crate::facts::CompactionStatus; use crate::facts::CompactionStrategy; use crate::facts::CompactionTrigger; use crate::facts::CustomAnalyticsFact; +use crate::facts::HookRunFact; +use crate::facts::HookRunInput; use crate::facts::InputError; use crate::facts::InvocationType; use crate::facts::PluginState; @@ -81,6 +85,9 @@ use codex_plugin::PluginTelemetryMetadata; use codex_protocol::config_types::ApprovalsReviewer; use codex_protocol::config_types::ModeKind; use codex_protocol::protocol::AskForApproval; +use codex_protocol::protocol::HookEventName; +use codex_protocol::protocol::HookRunStatus; +use codex_protocol::protocol::HookSource; use codex_protocol::protocol::SandboxPolicy; use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::SubAgentSource; @@ -1282,6 +1289,109 @@ fn plugin_management_event_serializes_expected_shape() { ); } +#[test] +fn hook_run_event_serializes_expected_shape() { + let tracking = TrackEventsContext { + model_slug: "gpt-5".to_string(), + thread_id: "thread-3".to_string(), + turn_id: "turn-3".to_string(), + }; + let event = TrackEventRequest::HookRun(CodexHookRunEventRequest { + event_type: "codex_hook_run", + event_params: codex_hook_run_metadata( + &tracking, + HookRunFact { + event_name: HookEventName::PreToolUse, + hook_source: HookSource::User, + status: HookRunStatus::Completed, + }, + ), + }); + + let payload = serde_json::to_value(&event).expect("serialize hook run event"); + + assert_eq!( + payload, + json!({ + "event_type": "codex_hook_run", + "event_params": { + "thread_id": "thread-3", + "turn_id": "turn-3", + "model_slug": "gpt-5", + "hook_name": "PreToolUse", + "hook_source": "user", + "status": "completed" + } + }) + ); +} + +#[test] +fn hook_run_metadata_maps_sources_and_statuses() { + let tracking = TrackEventsContext { + model_slug: "gpt-5".to_string(), + thread_id: "thread-1".to_string(), + turn_id: "turn-1".to_string(), + }; + + let system = serde_json::to_value(codex_hook_run_metadata( + &tracking, + HookRunFact { + event_name: HookEventName::SessionStart, + hook_source: HookSource::System, + status: HookRunStatus::Completed, + }, + )) + .expect("serialize system hook"); + let project = serde_json::to_value(codex_hook_run_metadata( + &tracking, + HookRunFact { + event_name: HookEventName::Stop, + hook_source: HookSource::Project, + status: HookRunStatus::Blocked, + }, + )) + .expect("serialize project hook"); + let unknown = serde_json::to_value(codex_hook_run_metadata( + &tracking, + HookRunFact { + event_name: HookEventName::UserPromptSubmit, + hook_source: HookSource::Unknown, + status: HookRunStatus::Failed, + }, + )) + .expect("serialize unknown hook"); + + assert_eq!(system["hook_source"], "system"); + assert_eq!(system["status"], "completed"); + assert_eq!(project["hook_source"], "project"); + assert_eq!(project["status"], "blocked"); + assert_eq!(unknown["hook_source"], "unknown"); + assert_eq!(unknown["status"], "failed"); +} + +#[test] +fn hook_run_metadata_maps_stopped_status() { + let tracking = TrackEventsContext { + model_slug: "gpt-5".to_string(), + thread_id: "thread-1".to_string(), + turn_id: "turn-1".to_string(), + }; + + let stopped = serde_json::to_value(codex_hook_run_metadata( + &tracking, + HookRunFact { + event_name: HookEventName::Stop, + hook_source: HookSource::User, + status: HookRunStatus::Stopped, + }, + )) + .expect("serialize stopped hook"); + + assert_eq!(stopped["hook_source"], "user"); + assert_eq!(stopped["status"], "stopped"); +} + #[test] fn plugin_used_dedupe_is_keyed_by_turn_and_plugin() { let (sender, _receiver) = mpsc::channel(1); @@ -1359,6 +1469,37 @@ async fn reducer_ingests_skill_invoked_fact() { ); } +#[tokio::test] +async fn reducer_ingests_hook_run_fact() { + let mut reducer = AnalyticsReducer::default(); + let mut events = Vec::new(); + + reducer + .ingest( + AnalyticsFact::Custom(CustomAnalyticsFact::HookRun(HookRunInput { + tracking: TrackEventsContext { + model_slug: "gpt-5".to_string(), + thread_id: "thread-1".to_string(), + turn_id: "turn-1".to_string(), + }, + hook: HookRunFact { + event_name: HookEventName::PostToolUse, + hook_source: HookSource::Unknown, + status: HookRunStatus::Failed, + }, + })), + &mut events, + ) + .await; + + let payload = serde_json::to_value(&events).expect("serialize events"); + assert_eq!(payload.as_array().expect("events array").len(), 1); + assert_eq!(payload[0]["event_type"], "codex_hook_run"); + assert_eq!(payload[0]["event_params"]["hook_name"], "PostToolUse"); + assert_eq!(payload[0]["event_params"]["hook_source"], "unknown"); + assert_eq!(payload[0]["event_params"]["status"], "failed"); +} + #[tokio::test] async fn reducer_ingests_app_and_plugin_facts() { let mut reducer = AnalyticsReducer::default(); diff --git a/codex-rs/analytics/src/client.rs b/codex-rs/analytics/src/client.rs index 5ba60e3ef4..1a4b5defe9 100644 --- a/codex-rs/analytics/src/client.rs +++ b/codex-rs/analytics/src/client.rs @@ -9,6 +9,8 @@ use crate::facts::AppInvocation; use crate::facts::AppMentionedInput; use crate::facts::AppUsedInput; use crate::facts::CustomAnalyticsFact; +use crate::facts::HookRunFact; +use crate::facts::HookRunInput; use crate::facts::PluginState; use crate::facts::PluginStateChangedInput; use crate::facts::SkillInvocation; @@ -191,6 +193,12 @@ impl AnalyticsEventsClient { ))); } + pub fn track_hook_run(&self, tracking: TrackEventsContext, hook: HookRunFact) { + self.record_fact(AnalyticsFact::Custom(CustomAnalyticsFact::HookRun( + HookRunInput { tracking, hook }, + ))); + } + pub fn track_plugin_used(&self, tracking: TrackEventsContext, plugin: PluginTelemetryMetadata) { if !self.queue.should_enqueue_plugin_used(&tracking, &plugin) { return; diff --git a/codex-rs/analytics/src/events.rs b/codex-rs/analytics/src/events.rs index eb312da140..d3aa5263da 100644 --- a/codex-rs/analytics/src/events.rs +++ b/codex-rs/analytics/src/events.rs @@ -1,5 +1,6 @@ use crate::facts::AppInvocation; use crate::facts::CodexCompactionEvent; +use crate::facts::HookRunFact; use crate::facts::InvocationType; use crate::facts::PluginState; use crate::facts::SubAgentThreadStartedInput; @@ -15,6 +16,9 @@ use codex_plugin::PluginTelemetryMetadata; use codex_protocol::approvals::NetworkApprovalProtocol; use codex_protocol::models::PermissionProfile; use codex_protocol::models::SandboxPermissions; +use codex_protocol::protocol::HookEventName; +use codex_protocol::protocol::HookRunStatus; +use codex_protocol::protocol::HookSource; use codex_protocol::protocol::SubAgentSource; use serde::Serialize; @@ -39,6 +43,7 @@ pub(crate) enum TrackEventRequest { GuardianReview(Box), AppMentioned(CodexAppMentionedEventRequest), AppUsed(CodexAppUsedEventRequest), + HookRun(CodexHookRunEventRequest), Compaction(Box), TurnEvent(Box), TurnSteer(CodexTurnSteerEventRequest), @@ -300,6 +305,22 @@ pub(crate) struct CodexAppUsedEventRequest { pub(crate) event_params: CodexAppMetadata, } +#[derive(Serialize)] +pub(crate) struct CodexHookRunMetadata { + pub(crate) thread_id: Option, + pub(crate) turn_id: Option, + pub(crate) model_slug: Option, + pub(crate) hook_name: Option, + pub(crate) hook_source: Option<&'static str>, + pub(crate) status: Option, +} + +#[derive(Serialize)] +pub(crate) struct CodexHookRunEventRequest { + pub(crate) event_type: &'static str, + pub(crate) event_params: CodexHookRunMetadata, +} + #[derive(Serialize)] pub(crate) struct CodexCompactionEventParams { pub(crate) thread_id: String, @@ -529,6 +550,43 @@ pub(crate) fn codex_plugin_used_metadata( } } +pub(crate) fn codex_hook_run_metadata( + tracking: &TrackEventsContext, + hook: HookRunFact, +) -> CodexHookRunMetadata { + CodexHookRunMetadata { + thread_id: Some(tracking.thread_id.clone()), + turn_id: Some(tracking.turn_id.clone()), + model_slug: Some(tracking.model_slug.clone()), + hook_name: Some(analytics_hook_event_name(hook.event_name).to_owned()), + hook_source: Some(analytics_hook_source(hook.hook_source)), + status: Some(analytics_hook_status(hook.status)), + } +} + +fn analytics_hook_event_name(event_name: HookEventName) -> &'static str { + match event_name { + HookEventName::PreToolUse => "PreToolUse", + HookEventName::PostToolUse => "PostToolUse", + HookEventName::SessionStart => "SessionStart", + HookEventName::UserPromptSubmit => "UserPromptSubmit", + HookEventName::Stop => "Stop", + } +} + +fn analytics_hook_source(source: HookSource) -> &'static str { + match source { + HookSource::System => "system", + HookSource::User => "user", + HookSource::Project => "project", + HookSource::Mdm => "mdm", + HookSource::SessionFlags => "session_flags", + HookSource::LegacyManagedConfigFile => "legacy_managed_config_file", + HookSource::LegacyManagedConfigMdm => "legacy_managed_config_mdm", + HookSource::Unknown => "unknown", + } +} + pub(crate) fn current_runtime_metadata() -> CodexRuntimeMetadata { let os_info = os_info::get(); CodexRuntimeMetadata { @@ -586,3 +644,11 @@ pub(crate) fn subagent_parent_thread_id(subagent_source: &SubAgentSource) -> Opt _ => None, } } + +fn analytics_hook_status(status: HookRunStatus) -> HookRunStatus { + match status { + // Running is unexpected here and normalized defensively. + HookRunStatus::Running => HookRunStatus::Failed, + other => other, + } +} diff --git a/codex-rs/analytics/src/facts.rs b/codex-rs/analytics/src/facts.rs index 5590fbfa6d..1d371acb1c 100644 --- a/codex-rs/analytics/src/facts.rs +++ b/codex-rs/analytics/src/facts.rs @@ -15,6 +15,9 @@ use codex_protocol::config_types::ReasoningSummary; use codex_protocol::config_types::ServiceTier; use codex_protocol::openai_models::ReasoningEffort; use codex_protocol::protocol::AskForApproval; +use codex_protocol::protocol::HookEventName; +use codex_protocol::protocol::HookRunStatus; +use codex_protocol::protocol::HookSource; use codex_protocol::protocol::SandboxPolicy; use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::SkillScope; @@ -298,6 +301,7 @@ pub(crate) enum CustomAnalyticsFact { SkillInvoked(SkillInvokedInput), AppMentioned(AppMentionedInput), AppUsed(AppUsedInput), + HookRun(HookRunInput), PluginUsed(PluginUsedInput), PluginStateChanged(PluginStateChangedInput), } @@ -317,6 +321,17 @@ pub(crate) struct AppUsedInput { pub app: AppInvocation, } +pub(crate) struct HookRunInput { + pub tracking: TrackEventsContext, + pub hook: HookRunFact, +} + +pub struct HookRunFact { + pub event_name: HookEventName, + pub hook_source: HookSource, + pub status: HookRunStatus, +} + pub(crate) struct PluginUsedInput { pub tracking: TrackEventsContext, pub plugin: PluginTelemetryMetadata, diff --git a/codex-rs/analytics/src/lib.rs b/codex-rs/analytics/src/lib.rs index 1a1a123152..03f485a1c1 100644 --- a/codex-rs/analytics/src/lib.rs +++ b/codex-rs/analytics/src/lib.rs @@ -29,6 +29,7 @@ pub use facts::CompactionReason; pub use facts::CompactionStatus; pub use facts::CompactionStrategy; pub use facts::CompactionTrigger; +pub use facts::HookRunFact; pub use facts::InputError; pub use facts::InvocationType; pub use facts::SkillInvocation; diff --git a/codex-rs/analytics/src/reducer.rs b/codex-rs/analytics/src/reducer.rs index a5e5165bd2..a6ce3fc831 100644 --- a/codex-rs/analytics/src/reducer.rs +++ b/codex-rs/analytics/src/reducer.rs @@ -3,6 +3,7 @@ use crate::events::CodexAppMentionedEventRequest; use crate::events::CodexAppServerClientMetadata; use crate::events::CodexAppUsedEventRequest; use crate::events::CodexCompactionEventRequest; +use crate::events::CodexHookRunEventRequest; use crate::events::CodexPluginEventRequest; use crate::events::CodexPluginUsedEventRequest; use crate::events::CodexRuntimeMetadata; @@ -20,6 +21,7 @@ use crate::events::ThreadInitializedEventParams; use crate::events::TrackEventRequest; use crate::events::codex_app_metadata; use crate::events::codex_compaction_event_params; +use crate::events::codex_hook_run_metadata; use crate::events::codex_plugin_metadata; use crate::events::codex_plugin_used_metadata; use crate::events::plugin_state_event_type; @@ -32,6 +34,7 @@ use crate::facts::AppMentionedInput; use crate::facts::AppUsedInput; use crate::facts::CodexCompactionEvent; use crate::facts::CustomAnalyticsFact; +use crate::facts::HookRunInput; use crate::facts::PluginState; use crate::facts::PluginStateChangedInput; use crate::facts::PluginUsedInput; @@ -217,6 +220,9 @@ impl AnalyticsReducer { CustomAnalyticsFact::AppUsed(input) => { self.ingest_app_used(input, out); } + CustomAnalyticsFact::HookRun(input) => { + self.ingest_hook_run(input, out); + } CustomAnalyticsFact::PluginUsed(input) => { self.ingest_plugin_used(input, out); } @@ -442,6 +448,14 @@ impl AnalyticsReducer { })); } + fn ingest_hook_run(&mut self, input: HookRunInput, out: &mut Vec) { + let HookRunInput { tracking, hook } = input; + out.push(TrackEventRequest::HookRun(CodexHookRunEventRequest { + event_type: "codex_hook_run", + event_params: codex_hook_run_metadata(&tracking, hook), + })); + } + fn ingest_plugin_used(&mut self, input: PluginUsedInput, out: &mut Vec) { let PluginUsedInput { tracking, plugin } = input; out.push(TrackEventRequest::PluginUsed(CodexPluginUsedEventRequest { diff --git a/codex-rs/app-server-protocol/schema/json/ServerNotification.json b/codex-rs/app-server-protocol/schema/json/ServerNotification.json index b84fbe9a02..d02a28e370 100644 --- a/codex-rs/app-server-protocol/schema/json/ServerNotification.json +++ b/codex-rs/app-server-protocol/schema/json/ServerNotification.json @@ -1517,6 +1517,14 @@ "scope": { "$ref": "#/definitions/HookScope" }, + "source": { + "allOf": [ + { + "$ref": "#/definitions/HookSource" + } + ], + "default": "unknown" + }, "sourcePath": { "$ref": "#/definitions/AbsolutePathBuf" }, @@ -1555,6 +1563,19 @@ ], "type": "string" }, + "HookSource": { + "enum": [ + "system", + "user", + "project", + "mdm", + "sessionFlags", + "legacyManagedConfigFile", + "legacyManagedConfigMdm", + "unknown" + ], + "type": "string" + }, "HookStartedNotification": { "properties": { "run": { diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index 943b8480f9..e87fa17928 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -8597,6 +8597,14 @@ "scope": { "$ref": "#/definitions/v2/HookScope" }, + "source": { + "allOf": [ + { + "$ref": "#/definitions/v2/HookSource" + } + ], + "default": "unknown" + }, "sourcePath": { "$ref": "#/definitions/v2/AbsolutePathBuf" }, @@ -8635,6 +8643,19 @@ ], "type": "string" }, + "HookSource": { + "enum": [ + "system", + "user", + "project", + "mdm", + "sessionFlags", + "legacyManagedConfigFile", + "legacyManagedConfigMdm", + "unknown" + ], + "type": "string" + }, "HookStartedNotification": { "$schema": "http://json-schema.org/draft-07/schema#", "properties": { diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index 1717dfb393..8254ad0127 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -5325,6 +5325,14 @@ "scope": { "$ref": "#/definitions/HookScope" }, + "source": { + "allOf": [ + { + "$ref": "#/definitions/HookSource" + } + ], + "default": "unknown" + }, "sourcePath": { "$ref": "#/definitions/AbsolutePathBuf" }, @@ -5363,6 +5371,19 @@ ], "type": "string" }, + "HookSource": { + "enum": [ + "system", + "user", + "project", + "mdm", + "sessionFlags", + "legacyManagedConfigFile", + "legacyManagedConfigMdm", + "unknown" + ], + "type": "string" + }, "HookStartedNotification": { "$schema": "http://json-schema.org/draft-07/schema#", "properties": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/HookCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/HookCompletedNotification.json index 2faa347f8e..4f444961a9 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/HookCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/HookCompletedNotification.json @@ -106,6 +106,14 @@ "scope": { "$ref": "#/definitions/HookScope" }, + "source": { + "allOf": [ + { + "$ref": "#/definitions/HookSource" + } + ], + "default": "unknown" + }, "sourcePath": { "$ref": "#/definitions/AbsolutePathBuf" }, @@ -143,6 +151,19 @@ "turn" ], "type": "string" + }, + "HookSource": { + "enum": [ + "system", + "user", + "project", + "mdm", + "sessionFlags", + "legacyManagedConfigFile", + "legacyManagedConfigMdm", + "unknown" + ], + "type": "string" } }, "properties": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/HookStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/HookStartedNotification.json index 496175d01b..6d439c95ca 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/HookStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/HookStartedNotification.json @@ -106,6 +106,14 @@ "scope": { "$ref": "#/definitions/HookScope" }, + "source": { + "allOf": [ + { + "$ref": "#/definitions/HookSource" + } + ], + "default": "unknown" + }, "sourcePath": { "$ref": "#/definitions/AbsolutePathBuf" }, @@ -143,6 +151,19 @@ "turn" ], "type": "string" + }, + "HookSource": { + "enum": [ + "system", + "user", + "project", + "mdm", + "sessionFlags", + "legacyManagedConfigFile", + "legacyManagedConfigMdm", + "unknown" + ], + "type": "string" } }, "properties": { diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/HookRunSummary.ts b/codex-rs/app-server-protocol/schema/typescript/v2/HookRunSummary.ts index f6d4b75378..75ab780b93 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/HookRunSummary.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/HookRunSummary.ts @@ -8,5 +8,6 @@ import type { HookHandlerType } from "./HookHandlerType"; import type { HookOutputEntry } from "./HookOutputEntry"; import type { HookRunStatus } from "./HookRunStatus"; import type { HookScope } from "./HookScope"; +import type { HookSource } from "./HookSource"; -export type HookRunSummary = { id: string, eventName: HookEventName, handlerType: HookHandlerType, executionMode: HookExecutionMode, scope: HookScope, sourcePath: AbsolutePathBuf, displayOrder: bigint, status: HookRunStatus, statusMessage: string | null, startedAt: bigint, completedAt: bigint | null, durationMs: bigint | null, entries: Array, }; +export type HookRunSummary = { id: string, eventName: HookEventName, handlerType: HookHandlerType, executionMode: HookExecutionMode, scope: HookScope, sourcePath: AbsolutePathBuf, source: HookSource, displayOrder: bigint, status: HookRunStatus, statusMessage: string | null, startedAt: bigint, completedAt: bigint | null, durationMs: bigint | null, entries: Array, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/HookSource.ts b/codex-rs/app-server-protocol/schema/typescript/v2/HookSource.ts new file mode 100644 index 0000000000..7edf61f918 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/HookSource.ts @@ -0,0 +1,5 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type HookSource = "system" | "user" | "project" | "mdm" | "sessionFlags" | "legacyManagedConfigFile" | "legacyManagedConfigMdm" | "unknown"; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/index.ts b/codex-rs/app-server-protocol/schema/typescript/v2/index.ts index d432530b42..69cec75e77 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/index.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/index.ts @@ -139,6 +139,7 @@ export type { HookPromptFragment } from "./HookPromptFragment"; export type { HookRunStatus } from "./HookRunStatus"; export type { HookRunSummary } from "./HookRunSummary"; export type { HookScope } from "./HookScope"; +export type { HookSource } from "./HookSource"; export type { HookStartedNotification } from "./HookStartedNotification"; export type { ItemCompletedNotification } from "./ItemCompletedNotification"; export type { ItemGuardianApprovalReviewCompletedNotification } from "./ItemGuardianApprovalReviewCompletedNotification"; diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index 155c67e7bf..5173549326 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -65,6 +65,7 @@ use codex_protocol::protocol::HookOutputEntryKind as CoreHookOutputEntryKind; use codex_protocol::protocol::HookRunStatus as CoreHookRunStatus; use codex_protocol::protocol::HookRunSummary as CoreHookRunSummary; use codex_protocol::protocol::HookScope as CoreHookScope; +use codex_protocol::protocol::HookSource as CoreHookSource; use codex_protocol::protocol::ModelRerouteReason as CoreModelRerouteReason; use codex_protocol::protocol::NetworkAccess as CoreNetworkAccess; use codex_protocol::protocol::NonSteerableTurnKind as CoreNonSteerableTurnKind; @@ -402,6 +403,23 @@ v2_enum_from_core!( } ); +v2_enum_from_core!( + pub enum HookSource from CoreHookSource { + System, + User, + Project, + Mdm, + SessionFlags, + LegacyManagedConfigFile, + LegacyManagedConfigMdm, + Unknown, + } +); + +fn default_hook_source() -> HookSource { + HookSource::Unknown +} + v2_enum_from_core!( pub enum HookRunStatus from CoreHookRunStatus { Running, Completed, Failed, Blocked, Stopped @@ -449,6 +467,8 @@ pub struct HookRunSummary { pub execution_mode: HookExecutionMode, pub scope: HookScope, pub source_path: AbsolutePathBuf, + #[serde(default = "default_hook_source")] + pub source: HookSource, pub display_order: i64, pub status: HookRunStatus, pub status_message: Option, @@ -467,6 +487,7 @@ impl From for HookRunSummary { execution_mode: value.execution_mode.into(), scope: value.scope.into(), source_path: value.source_path, + source: value.source.into(), display_order: value.display_order, status: value.status.into(), status_message: value.status_message, diff --git a/codex-rs/config/src/lib.rs b/codex-rs/config/src/lib.rs index ffed561984..66a6fc22fc 100644 --- a/codex-rs/config/src/lib.rs +++ b/codex-rs/config/src/lib.rs @@ -86,3 +86,5 @@ pub use state::ConfigLayerEntry; pub use state::ConfigLayerStack; pub use state::ConfigLayerStackOrdering; pub use state::LoaderOverrides; + +pub use codex_app_server_protocol::ConfigLayerSource; diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 42f165bbf3..844d585be5 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -24,6 +24,7 @@ use crate::compact_remote::run_inline_remote_auto_compact_task; use crate::config::ManagedFeatures; use crate::connectors; use crate::exec_policy::ExecPolicyManager; +use crate::hook_runtime::emit_hook_completed_events; use crate::installation_id::resolve_installation_id; use crate::mcp_tool_exposure::build_mcp_tool_exposure; use crate::parse_turn_item; @@ -6601,10 +6602,8 @@ pub(crate) async fn run_turn( .await; } let stop_outcome = sess.hooks().run_stop(stop_request).await; - for completed in stop_outcome.hook_events { - sess.send_event(&turn_context, EventMsg::HookCompleted(completed)) - .await; - } + emit_hook_completed_events(&sess, &turn_context, stop_outcome.hook_events) + .await; if stop_outcome.should_block { if let Some(hook_prompt_message) = build_hook_prompt_message(&stop_outcome.continuation_fragments) diff --git a/codex-rs/core/src/hook_runtime.rs b/codex-rs/core/src/hook_runtime.rs index d2d672fcd4..58cac101b0 100644 --- a/codex-rs/core/src/hook_runtime.rs +++ b/codex-rs/core/src/hook_runtime.rs @@ -1,6 +1,8 @@ use std::future::Future; use std::sync::Arc; +use codex_analytics::HookRunFact; +use codex_analytics::build_track_events_context; use codex_hooks::PostToolUseOutcome; use codex_hooks::PostToolUseRequest; use codex_hooks::PreToolUseOutcome; @@ -316,17 +318,52 @@ async fn emit_hook_started_events( } } -async fn emit_hook_completed_events( +pub(crate) async fn emit_hook_completed_events( sess: &Arc, turn_context: &Arc, completed_events: Vec, ) { for completed in completed_events { + track_hook_completed_analytics(sess, turn_context, &completed); sess.send_event(turn_context, EventMsg::HookCompleted(completed)) .await; } } +fn track_hook_completed_analytics( + sess: &Arc, + turn_context: &Arc, + completed: &HookCompletedEvent, +) { + let (tracking, hook) = + hook_run_analytics_payload(sess.conversation_id.to_string(), turn_context, completed); + sess.services + .analytics_events_client + .track_hook_run(tracking, hook); +} + +fn hook_run_analytics_payload( + thread_id: String, + turn_context: &TurnContext, + completed: &HookCompletedEvent, +) -> (codex_analytics::TrackEventsContext, HookRunFact) { + ( + build_track_events_context( + turn_context.model_info.slug.clone(), + thread_id, + completed + .turn_id + .clone() + .unwrap_or_else(|| turn_context.sub_id.clone()), + ), + HookRunFact { + event_name: completed.run.event_name, + hook_source: completed.run.source, + status: completed.run.status, + }, + ) +} + fn hook_permission_mode(turn_context: &TurnContext) -> String { match turn_context.approval_policy.value() { AskForApproval::Never => "bypassPermissions", @@ -341,9 +378,21 @@ fn hook_permission_mode(turn_context: &TurnContext) -> String { #[cfg(test)] mod tests { use codex_protocol::models::ContentItem; + use codex_protocol::protocol::HookEventName; + use codex_protocol::protocol::HookExecutionMode; + use codex_protocol::protocol::HookHandlerType; + use codex_protocol::protocol::HookRunStatus; + use codex_protocol::protocol::HookScope; + use codex_protocol::protocol::HookSource; use pretty_assertions::assert_eq; use super::additional_context_messages; + use super::hook_run_analytics_payload; + use crate::codex::make_session_and_context; + use codex_protocol::protocol::HookCompletedEvent; + use codex_protocol::protocol::HookRunSummary; + use codex_utils_absolute_path::test_support::PathBufExt; + use codex_utils_absolute_path::test_support::test_path_buf; #[test] fn additional_context_messages_stay_separate_and_ordered() { @@ -378,4 +427,58 @@ mod tests { ], ); } + + #[tokio::test] + async fn hook_run_analytics_payload_uses_completed_turn_id() { + let (_session, turn_context) = make_session_and_context().await; + let completed = HookCompletedEvent { + turn_id: Some("turn-from-hook".to_string()), + run: sample_hook_run(HookRunStatus::Blocked, HookSource::Project), + }; + + let (tracking, hook) = + hook_run_analytics_payload("thread-123".to_string(), &turn_context, &completed); + + assert_eq!(tracking.thread_id, "thread-123"); + assert_eq!(tracking.turn_id, "turn-from-hook"); + assert_eq!(tracking.model_slug, turn_context.model_info.slug); + assert_eq!(hook.event_name, HookEventName::Stop); + assert_eq!(hook.hook_source, HookSource::Project); + assert_eq!(hook.status, HookRunStatus::Blocked); + } + + #[tokio::test] + async fn hook_run_analytics_payload_falls_back_to_turn_context_id() { + let (_session, turn_context) = make_session_and_context().await; + let completed = HookCompletedEvent { + turn_id: None, + run: sample_hook_run(HookRunStatus::Failed, HookSource::Unknown), + }; + + let (tracking, hook) = + hook_run_analytics_payload("thread-123".to_string(), &turn_context, &completed); + + assert_eq!(tracking.turn_id, turn_context.sub_id); + assert_eq!(hook.hook_source, HookSource::Unknown); + assert_eq!(hook.status, HookRunStatus::Failed); + } + + fn sample_hook_run(status: HookRunStatus, source: HookSource) -> HookRunSummary { + HookRunSummary { + id: "stop:0:/tmp/hooks.json".to_string(), + event_name: HookEventName::Stop, + handler_type: HookHandlerType::Command, + execution_mode: HookExecutionMode::Sync, + scope: HookScope::Turn, + source_path: test_path_buf("/tmp/hooks.json").abs(), + source, + display_order: 0, + status, + status_message: None, + started_at: 10, + completed_at: Some(37), + duration_ms: Some(27), + entries: Vec::new(), + } + } } diff --git a/codex-rs/hooks/src/engine/discovery.rs b/codex-rs/hooks/src/engine/discovery.rs index 5505d8e914..682328e640 100644 --- a/codex-rs/hooks/src/engine/discovery.rs +++ b/codex-rs/hooks/src/engine/discovery.rs @@ -9,6 +9,8 @@ use super::config::HooksFile; use super::config::MatcherGroup; use crate::events::common::matcher_pattern_for_event; use crate::events::common::validate_matcher_pattern; +use codex_config::ConfigLayerSource; +use codex_protocol::protocol::HookSource; pub(crate) struct DiscoveryResult { pub handlers: Vec, @@ -93,6 +95,7 @@ pub(crate) fn discover_handlers(config_layer_stack: Option<&ConfigLayerStack>) - &mut warnings, &mut display_order, &source_path, + hook_source_for_config_layer_source(&layer.name), event_name, groups, ); @@ -102,95 +105,94 @@ pub(crate) fn discover_handlers(config_layer_stack: Option<&ConfigLayerStack>) - DiscoveryResult { handlers, warnings } } -fn append_group_handlers( - handlers: &mut Vec, - warnings: &mut Vec, - display_order: &mut i64, - source_path: &AbsolutePathBuf, - event_name: codex_protocol::protocol::HookEventName, - matcher: Option<&str>, - group_handlers: Vec, -) { - if let Some(matcher) = matcher - && let Err(err) = validate_matcher_pattern(matcher) - { - warnings.push(format!( - "invalid matcher {matcher:?} in {}: {err}", - source_path.display() - )); - return; - } - - for handler in group_handlers { - match handler { - HookHandlerConfig::Command { - command, - timeout_sec, - r#async, - status_message, - } => { - if r#async { - warnings.push(format!( - "skipping async hook in {}: async hooks are not supported yet", - source_path.display() - )); - continue; - } - if command.trim().is_empty() { - warnings.push(format!( - "skipping empty hook command in {}", - source_path.display() - )); - continue; - } - let timeout_sec = timeout_sec.unwrap_or(600).max(1); - handlers.push(ConfiguredHandler { - event_name, - matcher: matcher.map(ToOwned::to_owned), - command, - timeout_sec, - status_message, - source_path: source_path.clone(), - display_order: *display_order, - }); - *display_order += 1; - } - HookHandlerConfig::Prompt {} => warnings.push(format!( - "skipping prompt hook in {}: prompt hooks are not supported yet", - source_path.display() - )), - HookHandlerConfig::Agent {} => warnings.push(format!( - "skipping agent hook in {}: agent hooks are not supported yet", - source_path.display() - )), - } - } -} - fn append_matcher_groups( handlers: &mut Vec, warnings: &mut Vec, display_order: &mut i64, source_path: &AbsolutePathBuf, + source: HookSource, event_name: codex_protocol::protocol::HookEventName, groups: Vec, ) { for group in groups { - append_group_handlers( - handlers, - warnings, - display_order, - source_path, - event_name, - matcher_pattern_for_event(event_name, group.matcher.as_deref()), - group.hooks, - ); + let matcher = matcher_pattern_for_event(event_name, group.matcher.as_deref()); + if let Some(matcher) = matcher + && let Err(err) = validate_matcher_pattern(matcher) + { + warnings.push(format!( + "invalid matcher {matcher:?} in {}: {err}", + source_path.display() + )); + continue; + } + + for handler in group.hooks { + match handler { + HookHandlerConfig::Command { + command, + timeout_sec, + r#async, + status_message, + } => { + if r#async { + warnings.push(format!( + "skipping async hook in {}: async hooks are not supported yet", + source_path.display() + )); + continue; + } + if command.trim().is_empty() { + warnings.push(format!( + "skipping empty hook command in {}", + source_path.display() + )); + continue; + } + let timeout_sec = timeout_sec.unwrap_or(600).max(1); + handlers.push(ConfiguredHandler { + event_name, + matcher: matcher.map(ToOwned::to_owned), + command, + timeout_sec, + status_message, + source_path: source_path.clone(), + source, + display_order: *display_order, + }); + *display_order += 1; + } + HookHandlerConfig::Prompt {} => warnings.push(format!( + "skipping prompt hook in {}: prompt hooks are not supported yet", + source_path.display() + )), + HookHandlerConfig::Agent {} => warnings.push(format!( + "skipping agent hook in {}: agent hooks are not supported yet", + source_path.display() + )), + } + } + } +} + +fn hook_source_for_config_layer_source(source: &ConfigLayerSource) -> HookSource { + match source { + ConfigLayerSource::System { .. } => HookSource::System, + ConfigLayerSource::User { .. } => HookSource::User, + ConfigLayerSource::Project { .. } => HookSource::Project, + ConfigLayerSource::Mdm { .. } => HookSource::Mdm, + ConfigLayerSource::SessionFlags => HookSource::SessionFlags, + ConfigLayerSource::LegacyManagedConfigTomlFromFile { .. } => { + HookSource::LegacyManagedConfigFile + } + ConfigLayerSource::LegacyManagedConfigTomlFromMdm => HookSource::LegacyManagedConfigMdm, } } #[cfg(test)] mod tests { + use codex_config::ConfigLayerSource; use codex_protocol::protocol::HookEventName; + use codex_protocol::protocol::HookSource; use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_absolute_path::test_support::PathBufExt; use codex_utils_absolute_path::test_support::test_path_buf; @@ -198,32 +200,43 @@ mod tests { use super::ConfiguredHandler; use super::HookHandlerConfig; - use super::append_group_handlers; - use crate::events::common::matcher_pattern_for_event; + use super::MatcherGroup; + use super::append_matcher_groups; fn source_path() -> AbsolutePathBuf { test_path_buf("/tmp/hooks.json").abs() } + fn hook_source() -> HookSource { + HookSource::User + } + + fn command_group(matcher: Option<&str>) -> MatcherGroup { + MatcherGroup { + matcher: matcher.map(str::to_string), + hooks: vec![HookHandlerConfig::Command { + command: "echo hello".to_string(), + timeout_sec: None, + r#async: false, + status_message: None, + }], + } + } + #[test] fn user_prompt_submit_ignores_invalid_matcher_during_discovery() { let mut handlers = Vec::new(); let mut warnings = Vec::new(); let mut display_order = 0; - append_group_handlers( + append_matcher_groups( &mut handlers, &mut warnings, &mut display_order, &source_path(), + hook_source(), HookEventName::UserPromptSubmit, - matcher_pattern_for_event(HookEventName::UserPromptSubmit, Some("[")), - vec![HookHandlerConfig::Command { - command: "echo hello".to_string(), - timeout_sec: None, - r#async: false, - status_message: None, - }], + vec![command_group(Some("["))], ); assert_eq!(warnings, Vec::::new()); @@ -236,6 +249,7 @@ mod tests { timeout_sec: 600, status_message: None, source_path: source_path(), + source: hook_source(), display_order: 0, }] ); @@ -247,19 +261,14 @@ mod tests { let mut warnings = Vec::new(); let mut display_order = 0; - append_group_handlers( + append_matcher_groups( &mut handlers, &mut warnings, &mut display_order, &source_path(), + hook_source(), HookEventName::PreToolUse, - matcher_pattern_for_event(HookEventName::PreToolUse, Some("^Bash$")), - vec![HookHandlerConfig::Command { - command: "echo hello".to_string(), - timeout_sec: None, - r#async: false, - status_message: None, - }], + vec![command_group(Some("^Bash$"))], ); assert_eq!(warnings, Vec::::new()); @@ -272,6 +281,7 @@ mod tests { timeout_sec: 600, status_message: None, source_path: source_path(), + source: hook_source(), display_order: 0, }] ); @@ -283,19 +293,14 @@ mod tests { let mut warnings = Vec::new(); let mut display_order = 0; - append_group_handlers( + append_matcher_groups( &mut handlers, &mut warnings, &mut display_order, &source_path(), + hook_source(), HookEventName::PreToolUse, - matcher_pattern_for_event(HookEventName::PreToolUse, Some("*")), - vec![HookHandlerConfig::Command { - command: "echo hello".to_string(), - timeout_sec: None, - r#async: false, - status_message: None, - }], + vec![command_group(Some("*"))], ); assert_eq!(warnings, Vec::::new()); @@ -309,19 +314,14 @@ mod tests { let mut warnings = Vec::new(); let mut display_order = 0; - append_group_handlers( + append_matcher_groups( &mut handlers, &mut warnings, &mut display_order, &source_path(), + hook_source(), HookEventName::PostToolUse, - matcher_pattern_for_event(HookEventName::PostToolUse, Some("Edit|Write")), - vec![HookHandlerConfig::Command { - command: "echo hello".to_string(), - timeout_sec: None, - r#async: false, - status_message: None, - }], + vec![command_group(Some("Edit|Write"))], ); assert_eq!(warnings, Vec::::new()); @@ -329,4 +329,52 @@ mod tests { assert_eq!(handlers[0].event_name, HookEventName::PostToolUse); assert_eq!(handlers[0].matcher.as_deref(), Some("Edit|Write")); } + + #[test] + fn hook_source_for_config_layer_source_discards_source_details() { + let config_file = test_path_buf("/tmp/.codex/config.toml").abs(); + let dot_codex_folder = test_path_buf("/tmp/worktree/.codex").abs(); + + assert_eq!( + super::hook_source_for_config_layer_source(&ConfigLayerSource::System { + file: config_file.clone(), + }), + HookSource::System, + ); + assert_eq!( + super::hook_source_for_config_layer_source(&ConfigLayerSource::User { + file: config_file.clone(), + }), + HookSource::User, + ); + assert_eq!( + super::hook_source_for_config_layer_source(&ConfigLayerSource::Project { + dot_codex_folder + }), + HookSource::Project, + ); + assert_eq!( + super::hook_source_for_config_layer_source(&ConfigLayerSource::Mdm { + domain: "com.openai.codex".to_string(), + key: "config".to_string(), + }), + HookSource::Mdm, + ); + assert_eq!( + super::hook_source_for_config_layer_source(&ConfigLayerSource::SessionFlags), + HookSource::SessionFlags, + ); + assert_eq!( + super::hook_source_for_config_layer_source( + &ConfigLayerSource::LegacyManagedConfigTomlFromFile { file: config_file }, + ), + HookSource::LegacyManagedConfigFile, + ); + assert_eq!( + super::hook_source_for_config_layer_source( + &ConfigLayerSource::LegacyManagedConfigTomlFromMdm, + ), + HookSource::LegacyManagedConfigMdm, + ); + } } diff --git a/codex-rs/hooks/src/engine/dispatcher.rs b/codex-rs/hooks/src/engine/dispatcher.rs index e3dedf54bb..fbb2a1843f 100644 --- a/codex-rs/hooks/src/engine/dispatcher.rs +++ b/codex-rs/hooks/src/engine/dispatcher.rs @@ -50,6 +50,7 @@ pub(crate) fn running_summary(handler: &ConfiguredHandler) -> HookRunSummary { execution_mode: HookExecutionMode::Sync, scope: scope_for_event(handler.event_name), source_path: handler.source_path.clone(), + source: handler.source, display_order: handler.display_order, status: HookRunStatus::Running, status_message: handler.status_message.clone(), @@ -95,6 +96,7 @@ pub(crate) fn completed_summary( execution_mode: HookExecutionMode::Sync, scope: scope_for_event(handler.event_name), source_path: handler.source_path.clone(), + source: handler.source, display_order: handler.display_order, status, status_message: handler.status_message.clone(), @@ -118,6 +120,7 @@ fn scope_for_event(event_name: HookEventName) -> HookScope { #[cfg(test)] mod tests { use codex_protocol::protocol::HookEventName; + use codex_protocol::protocol::HookSource; use codex_utils_absolute_path::test_support::PathBufExt; use codex_utils_absolute_path::test_support::test_path_buf; @@ -137,6 +140,7 @@ mod tests { timeout_sec: 5, status_message: None, source_path: test_path_buf("/tmp/hooks.json").abs(), + source: HookSource::User, display_order, } } diff --git a/codex-rs/hooks/src/engine/mod.rs b/codex-rs/hooks/src/engine/mod.rs index f3992e8ef3..9bc23705d4 100644 --- a/codex-rs/hooks/src/engine/mod.rs +++ b/codex-rs/hooks/src/engine/mod.rs @@ -7,6 +7,7 @@ pub(crate) mod schema_loader; use codex_config::ConfigLayerStack; use codex_protocol::protocol::HookRunSummary; +use codex_protocol::protocol::HookSource; use codex_utils_absolute_path::AbsolutePathBuf; use crate::events::post_tool_use::PostToolUseOutcome; @@ -34,6 +35,7 @@ pub(crate) struct ConfiguredHandler { pub timeout_sec: u64, pub status_message: Option, pub source_path: AbsolutePathBuf, + pub source: HookSource, pub display_order: i64, } diff --git a/codex-rs/hooks/src/events/post_tool_use.rs b/codex-rs/hooks/src/events/post_tool_use.rs index 056956705d..f7f40d2794 100644 --- a/codex-rs/hooks/src/events/post_tool_use.rs +++ b/codex-rs/hooks/src/events/post_tool_use.rs @@ -525,6 +525,7 @@ mod tests { timeout_sec: 5, status_message: Some("running post tool use hook".to_string()), source_path: test_path_buf("/tmp/hooks.json").abs(), + source: codex_protocol::protocol::HookSource::User, display_order: 0, } } diff --git a/codex-rs/hooks/src/events/pre_tool_use.rs b/codex-rs/hooks/src/events/pre_tool_use.rs index e188207c8a..f90abc2816 100644 --- a/codex-rs/hooks/src/events/pre_tool_use.rs +++ b/codex-rs/hooks/src/events/pre_tool_use.rs @@ -514,6 +514,7 @@ mod tests { timeout_sec: 5, status_message: None, source_path: test_path_buf("/tmp/hooks.json").abs(), + source: codex_protocol::protocol::HookSource::User, display_order: 0, } } diff --git a/codex-rs/hooks/src/events/session_start.rs b/codex-rs/hooks/src/events/session_start.rs index ea396a2231..3c51426fa4 100644 --- a/codex-rs/hooks/src/events/session_start.rs +++ b/codex-rs/hooks/src/events/session_start.rs @@ -361,6 +361,7 @@ mod tests { timeout_sec: 600, status_message: None, source_path: test_path_buf("/tmp/hooks.json").abs(), + source: codex_protocol::protocol::HookSource::User, display_order: 0, } } diff --git a/codex-rs/hooks/src/events/stop.rs b/codex-rs/hooks/src/events/stop.rs index 478cdbdd8b..5fe55568a4 100644 --- a/codex-rs/hooks/src/events/stop.rs +++ b/codex-rs/hooks/src/events/stop.rs @@ -528,6 +528,7 @@ mod tests { timeout_sec: 600, status_message: None, source_path: test_path_buf("/tmp/hooks.json").abs(), + source: codex_protocol::protocol::HookSource::User, display_order: 0, } } diff --git a/codex-rs/hooks/src/events/user_prompt_submit.rs b/codex-rs/hooks/src/events/user_prompt_submit.rs index 875d56bd38..9aaff9a7d4 100644 --- a/codex-rs/hooks/src/events/user_prompt_submit.rs +++ b/codex-rs/hooks/src/events/user_prompt_submit.rs @@ -419,6 +419,7 @@ mod tests { timeout_sec: 5, status_message: None, source_path: test_path_buf("/tmp/hooks.json").abs(), + source: codex_protocol::protocol::HookSource::User, display_order: 0, } } diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index 8aea3544d4..d9c63370db 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -1635,6 +1635,20 @@ pub enum HookScope { Turn, } +#[derive(Debug, Clone, Copy, Default, Deserialize, Serialize, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "snake_case")] +pub enum HookSource { + System, + User, + Project, + Mdm, + SessionFlags, + LegacyManagedConfigFile, + LegacyManagedConfigMdm, + #[default] + Unknown, +} + #[derive(Debug, Clone, Copy, Deserialize, Serialize, PartialEq, Eq, JsonSchema, TS)] #[serde(rename_all = "snake_case")] pub enum HookRunStatus { @@ -1671,6 +1685,8 @@ pub struct HookRunSummary { pub execution_mode: HookExecutionMode, pub scope: HookScope, pub source_path: AbsolutePathBuf, + #[serde(default)] + pub source: HookSource, pub display_order: i64, pub status: HookRunStatus, pub status_message: Option, diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index 1db3dbc7ed..ab20b2214c 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -9892,6 +9892,7 @@ guardian_approval = true execution_mode: AppServerHookExecutionMode::Sync, scope: AppServerHookScope::Turn, source_path: test_path_buf("/tmp/hooks.json").abs(), + source: codex_app_server_protocol::HookSource::User, display_order: 0, status: AppServerHookRunStatus::Running, status_message: Some("checking go-workflow input policy".to_string()), @@ -9914,6 +9915,7 @@ guardian_approval = true execution_mode: AppServerHookExecutionMode::Sync, scope: AppServerHookScope::Turn, source_path: test_path_buf("/tmp/hooks.json").abs(), + source: codex_app_server_protocol::HookSource::User, display_order: 0, status: AppServerHookRunStatus::Stopped, status_message: Some("checking go-workflow input policy".to_string()), diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index f2697d9072..7cfcd56e17 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -1329,6 +1329,7 @@ fn hook_run_summary_from_notification( execution_mode: run.execution_mode.to_core(), scope: run.scope.to_core(), source_path: run.source_path, + source: run.source.to_core(), display_order: run.display_order, status: run.status.to_core(), status_message: run.status_message, diff --git a/codex-rs/tui/src/chatwidget/tests/helpers.rs b/codex-rs/tui/src/chatwidget/tests/helpers.rs index c0f881e2b8..9e365afa78 100644 --- a/codex-rs/tui/src/chatwidget/tests/helpers.rs +++ b/codex-rs/tui/src/chatwidget/tests/helpers.rs @@ -1001,6 +1001,7 @@ pub(super) async fn assert_hook_events_snapshot( execution_mode: codex_protocol::protocol::HookExecutionMode::Sync, scope: codex_protocol::protocol::HookScope::Turn, source_path: PathBuf::from(test_path_display("/tmp/hooks.json")).abs(), + source: codex_protocol::protocol::HookSource::User, display_order: 0, status: codex_protocol::protocol::HookRunStatus::Running, status_message: Some(status_message.to_string()), @@ -1035,6 +1036,7 @@ pub(super) async fn assert_hook_events_snapshot( execution_mode: codex_protocol::protocol::HookExecutionMode::Sync, scope: codex_protocol::protocol::HookScope::Turn, source_path: PathBuf::from(test_path_display("/tmp/hooks.json")).abs(), + source: codex_protocol::protocol::HookSource::User, display_order: 0, status: codex_protocol::protocol::HookRunStatus::Completed, status_message: Some(status_message.to_string()), diff --git a/codex-rs/tui/src/chatwidget/tests/status_and_layout.rs b/codex-rs/tui/src/chatwidget/tests/status_and_layout.rs index 75ca9d1fba..a31b93bb4c 100644 --- a/codex-rs/tui/src/chatwidget/tests/status_and_layout.rs +++ b/codex-rs/tui/src/chatwidget/tests/status_and_layout.rs @@ -1425,6 +1425,7 @@ async fn user_prompt_submit_app_server_hook_notifications_render_snapshot() { execution_mode: AppServerHookExecutionMode::Sync, scope: AppServerHookScope::Turn, source_path: PathBuf::from(test_path_display("/tmp/hooks.json")).abs(), + source: codex_app_server_protocol::HookSource::User, display_order: 0, status: AppServerHookRunStatus::Running, status_message: Some("checking go-workflow input policy".to_string()), @@ -1447,6 +1448,7 @@ async fn user_prompt_submit_app_server_hook_notifications_render_snapshot() { execution_mode: AppServerHookExecutionMode::Sync, scope: AppServerHookScope::Turn, source_path: PathBuf::from(test_path_display("/tmp/hooks.json")).abs(), + source: codex_app_server_protocol::HookSource::User, display_order: 0, status: AppServerHookRunStatus::Stopped, status_message: Some("checking go-workflow input policy".to_string()), @@ -1517,6 +1519,7 @@ async fn completed_hook_with_no_entries_stays_out_of_history() { execution_mode: codex_protocol::protocol::HookExecutionMode::Sync, scope: codex_protocol::protocol::HookScope::Turn, source_path: PathBuf::from(test_path_display("/tmp/hooks.json")).abs(), + source: codex_protocol::protocol::HookSource::User, display_order: 0, status: codex_protocol::protocol::HookRunStatus::Running, status_message: None, @@ -1542,6 +1545,7 @@ async fn completed_hook_with_no_entries_stays_out_of_history() { execution_mode: codex_protocol::protocol::HookExecutionMode::Sync, scope: codex_protocol::protocol::HookScope::Turn, source_path: PathBuf::from(test_path_display("/tmp/hooks.json")).abs(), + source: codex_protocol::protocol::HookSource::User, display_order: 0, status: codex_protocol::protocol::HookRunStatus::Completed, status_message: None, @@ -2017,6 +2021,7 @@ fn hook_run_summary( execution_mode: codex_protocol::protocol::HookExecutionMode::Sync, scope: codex_protocol::protocol::HookScope::Turn, source_path: PathBuf::from(test_path_display("/tmp/hooks.json")).abs(), + source: codex_protocol::protocol::HookSource::User, display_order: 0, status, status_message: status_message.map(str::to_string), diff --git a/codex-rs/tui/src/history_cell/hook_cell.rs b/codex-rs/tui/src/history_cell/hook_cell.rs index a1207186c1..50642aa0f2 100644 --- a/codex-rs/tui/src/history_cell/hook_cell.rs +++ b/codex-rs/tui/src/history_cell/hook_cell.rs @@ -768,6 +768,7 @@ mod tests { execution_mode: codex_protocol::protocol::HookExecutionMode::Sync, scope: codex_protocol::protocol::HookScope::Turn, source_path: test_path_buf("/tmp/hooks.json").abs(), + source: codex_protocol::protocol::HookSource::User, display_order: 0, status: HookRunStatus::Running, status_message: Some("checking output policy".to_string()), From ea34c6ed8de87c72616ac08c9c5e399ebd1f9c58 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 16 Apr 2026 12:48:31 -0700 Subject: [PATCH 36/59] fix: fix clippy issue in examples/ folder (#18184) I believe this use of `expect()` was introduced in https://github.com/openai/codex/pull/17826, but was not flagged by CI. Though I did see it in the diagnostics panel in VS Code, so it's worth cleaning up. I guess our current CI does include `examples/` when running Clippy? --- codex-rs/thread-store/examples/generate-proto.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/codex-rs/thread-store/examples/generate-proto.rs b/codex-rs/thread-store/examples/generate-proto.rs index aa159f5bcb..0b4afb54f9 100644 --- a/codex-rs/thread-store/examples/generate-proto.rs +++ b/codex-rs/thread-store/examples/generate-proto.rs @@ -1,7 +1,12 @@ use std::path::PathBuf; fn main() -> Result<(), Box> { - let proto_dir = PathBuf::from(std::env::args().nth(1).expect("proto dir")); + let Some(proto_dir_arg) = std::env::args().nth(1) else { + eprintln!("Usage: generate-proto "); + std::process::exit(1); + }; + + let proto_dir = PathBuf::from(proto_dir_arg); let proto_file = proto_dir.join("codex.thread_store.v1.proto"); tonic_prost_build::configure() From ec8d4bfc77353ac328a7b4897ab47b296e90f19e Mon Sep 17 00:00:00 2001 From: Felipe Coury Date: Thu, 16 Apr 2026 17:29:34 -0300 Subject: [PATCH 37/59] fix(app-server): replay token usage after resume and fork (#18023) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem When a user resumed or forked a session, the TUI could render the restored thread history immediately, but it did not receive token usage until a later model turn emitted a fresh usage event. That left the context/status UI blank or stale during the exact window where the user expects resumed state to look complete. Core already reconstructed token usage from the rollout; the missing behavior was app-server lifecycle replay to the client that just attached. ## Mental model Token usage has two representations. The rollout is the durable source of historical `TokenCount` events, and the core session cache is the in-memory snapshot reconstructed from that rollout on resume or fork. App-server v2 clients do not read core state directly; they learn about usage through `thread/tokenUsage/updated`. The fix keeps those roles separate: core exposes the restored `TokenUsageInfo`, and app-server sends one targeted notification after a successful `thread/resume` or `thread/fork` response when that restored snapshot exists. This notification is not a new model event. It is a replay of already-persisted state for the client that just attached. That distinction matters because using the normal core event path here would risk duplicating `TokenCount` entries in the rollout and making future resumes count historical usage twice. ## Non-goals This change does not add a new protocol method or payload shape. It reuses the existing v2 `thread/tokenUsage/updated` notification and the TUI’s existing handler for that notification. This change does not alter how token usage is computed, accumulated, compacted, or written during turns. It only exposes the token usage that resume and fork reconstruction already restored. This change does not broadcast historical usage replay to every subscribed client. The replay is intentionally scoped to the connection that requested resume or fork so already-attached clients are not surprised by an old usage update while they may be rendering live activity. ## Tradeoffs Sending the usage notification after the JSON-RPC response preserves a clear lifecycle order: the client first receives the thread object, then receives restored usage for that thread. The tradeoff is that usage is still a notification rather than part of the `thread/resume` or `thread/fork` response. That keeps the protocol shape stable and avoids duplicating usage fields across response types, but clients must continue listening for notifications after receiving the response. The helper selects the latest non-in-progress turn id for the replayed usage notification. This is conservative because restored usage belongs to completed persisted accounting, not to newly attached in-flight work. The fallback to the last turn preserves a stable wire payload for unusual histories, but histories with no meaningful completed turn still have a weak attribution story. ## Architecture Core already seeds `Session` token state from the last persisted rollout `TokenCount` during `InitialHistory::Resumed` and `InitialHistory::Forked`. The new core accessor exposes the complete `TokenUsageInfo` through `CodexThread` without giving app-server direct session mutation authority. App-server calls that accessor from three lifecycle paths: cold `thread/resume`, running-thread resume/rejoin, and `thread/fork`. In each path, the server sends the normal response first, then calls a shared helper that converts core usage into `ThreadTokenUsageUpdatedNotification` and sends it only to the requesting connection. The tests build fake rollouts with a user turn plus a persisted token usage event. They then exercise `thread/resume` and `thread/fork` without starting another model turn, proving that restored usage arrives before any next-turn token event could be produced. ## Observability The primary debug path is the app-server JSON-RPC stream. After `thread/resume` or `thread/fork`, a client should see the response followed by `thread/tokenUsage/updated` when the source rollout includes token usage. If the notification is absent, check whether the rollout contains an `event_msg` payload of type `token_count`, whether core reconstruction seeded `Session::token_usage_info`, and whether the connection stayed attached long enough to receive the targeted notification. The notification is sent through the existing `OutgoingMessageSender::send_server_notification_to_connections` path, so existing app-server tracing around server notifications still applies. Because this is a replay, not a model turn event, debugging should start at the resume/fork handlers rather than the turn event translation in `bespoke_event_handling`. ## Tests The focused regression coverage is `cargo test -p codex-app-server emits_restored_token_usage`, which covers both resume and fork. The core reconstruction guard is `cargo test -p codex-core record_initial_history_seeds_token_info_from_rollout`. Formatting and lint/fix passes were run with `just fmt`, `just fix -p codex-core`, and `just fix -p codex-app-server`. Full crate test runs surfaced pre-existing unrelated failures in command execution and plugin marketplace tests; the new token usage tests passed in focused runs and within the app-server suite before the unrelated command execution failure. --- .../src/protocol/thread_history.rs | 14 + codex-rs/app-server/README.md | 4 +- .../app-server/src/codex_message_processor.rs | 68 ++++- .../token_usage_replay.rs | 133 +++++++++ codex-rs/app-server/tests/common/lib.rs | 1 + codex-rs/app-server/tests/common/rollout.rs | 59 ++++ .../app-server/tests/suite/v2/thread_fork.rs | 55 ++++ .../tests/suite/v2/thread_resume.rs | 269 ++++++++++++++++++ codex-rs/core/src/codex.rs | 11 + codex-rs/core/src/codex_thread.rs | 12 + 10 files changed, 622 insertions(+), 4 deletions(-) create mode 100644 codex-rs/app-server/src/codex_message_processor/token_usage_replay.rs diff --git a/codex-rs/app-server-protocol/src/protocol/thread_history.rs b/codex-rs/app-server-protocol/src/protocol/thread_history.rs index 54f882cf29..5c327187e9 100644 --- a/codex-rs/app-server-protocol/src/protocol/thread_history.rs +++ b/codex-rs/app-server-protocol/src/protocol/thread_history.rs @@ -123,6 +123,20 @@ impl ThreadHistoryBuilder { .or_else(|| self.turns.last().cloned()) } + /// Returns the index of the active turn snapshot within the finished turn list. + /// + /// When a turn is still open, this is the index it will occupy after + /// `finish`. When no turn is open, it is the index of the last finished turn. + pub fn active_turn_position(&self) -> Option { + if self.current_turn.is_some() { + Some(self.turns.len()) + } else if self.turns.is_empty() { + None + } else { + Some(self.turns.len() - 1) + } + } + pub fn has_active_turn(&self) -> bool { self.current_turn.is_some() } diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index 5cfe052b50..2fece44544 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -250,7 +250,7 @@ Start a fresh thread when you need a new Codex conversation. Valid `personality` values are `"friendly"`, `"pragmatic"`, and `"none"`. When `"none"` is selected, the personality placeholder is replaced with an empty string. -To continue a stored session, call `thread/resume` with the `thread.id` you previously recorded. The response shape matches `thread/start`, and no additional notifications are emitted. You can also pass the same configuration overrides supported by `thread/start`, including `approvalsReviewer`. +To continue a stored session, call `thread/resume` with the `thread.id` you previously recorded. The response shape matches `thread/start`. When the stored session includes persisted token usage, the server emits `thread/tokenUsage/updated` immediately after the response so clients can render restored usage before the next turn starts. You can also pass the same configuration overrides supported by `thread/start`, including `approvalsReviewer`. By default, resume uses the latest persisted `model` and `reasoningEffort` values associated with the thread. Supplying any of `model`, `modelProvider`, `config.model`, or `config.model_reasoning_effort` disables that persisted fallback and uses the explicit overrides plus normal config resolution instead. @@ -264,7 +264,7 @@ Example: { "id": 11, "result": { "thread": { "id": "thr_123", … } } } ``` -To branch from a stored session, call `thread/fork` with the `thread.id`. This creates a new thread id and emits a `thread/started` notification for it. If the source thread is actively running, the fork snapshots it as if the current turn had been interrupted first. Pass `ephemeral: true` when the fork should stay in-memory only: +To branch from a stored session, call `thread/fork` with the `thread.id`. This creates a new thread id and emits a `thread/started` notification for it. When the source history includes persisted token usage, the server also emits `thread/tokenUsage/updated` for the new thread immediately after the response. If the source thread is actively running, the fork snapshots it as if the current turn had been interrupted first. Pass `ephemeral: true` when the fork should stay in-memory only: ```json { "method": "thread/fork", "id": 12, "params": { "threadId": "thr_123", "ephemeral": true } } diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index 410f22ece8..0f3b8bf522 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -363,12 +363,17 @@ use codex_app_server_protocol::ServerRequest; mod apps_list_helpers; mod plugin_app_helpers; mod plugin_mcp_oauth; +mod token_usage_replay; use crate::filters::compute_source_filters; use crate::filters::source_kind_matches; use crate::thread_state::ThreadListenerCommand; use crate::thread_state::ThreadState; use crate::thread_state::ThreadStateManager; +use token_usage_replay::latest_token_usage_turn_id_for_thread_path; +use token_usage_replay::latest_token_usage_turn_id_from_rollout_items; +use token_usage_replay::latest_token_usage_turn_id_from_rollout_path; +use token_usage_replay::send_thread_token_usage_update_to_connection; const THREAD_LIST_DEFAULT_LIMIT: usize = 25; const THREAD_LIST_MAX_LIMIT: usize = 100; @@ -4103,7 +4108,7 @@ impl CodexMessageProcessor { { Ok(NewThread { thread_id, - thread, + thread: codex_thread, session_configured, }) => { let SessionConfiguredEvent { rollout_path, .. } = session_configured; @@ -4132,7 +4137,7 @@ impl CodexMessageProcessor { let mut thread = match self .load_thread_from_resume_source_or_send_internal( thread_id, - thread.as_ref(), + codex_thread.as_ref(), &response_history, rollout_path.as_path(), fallback_model_provider.as_str(), @@ -4184,7 +4189,25 @@ impl CodexMessageProcessor { ); } + let connection_id = request_id.connection_id; + let token_usage_thread = response.thread.clone(); + let token_usage_turn_id = latest_token_usage_turn_id_from_rollout_items( + &response_history.get_rollout_items(), + &token_usage_thread, + ); self.outgoing.send_response(request_id, response).await; + // The client needs restored usage before it starts another turn. + // Sending after the response preserves JSON-RPC request ordering while + // still filling the status line before the next turn lifecycle begins. + send_thread_token_usage_update_to_connection( + &self.outgoing, + connection_id, + thread_id, + &token_usage_thread, + codex_thread.as_ref(), + token_usage_turn_id, + ) + .await; } Err(err) => { let error = JSONRPCErrorError { @@ -4820,7 +4843,31 @@ impl CodexMessageProcessor { ); } + let connection_id = request_id.connection_id; + let token_usage_thread = response.thread.clone(); + let token_usage_turn_id = if let Some(turn_id) = + latest_token_usage_turn_id_for_thread_path(&token_usage_thread).await + { + Some(turn_id) + } else { + latest_token_usage_turn_id_from_rollout_path( + rollout_path.as_path(), + &token_usage_thread, + ) + .await + }; self.outgoing.send_response(request_id, response).await; + // Mirror the resume contract for forks: the new thread is usable as soon + // as the response arrives, so restored usage must follow immediately. + send_thread_token_usage_update_to_connection( + &self.outgoing, + connection_id, + thread_id, + &token_usage_thread, + forked_thread.as_ref(), + token_usage_turn_id, + ) + .await; let notif = ThreadStartedNotification { thread }; self.outgoing @@ -8525,7 +8572,24 @@ async fn handle_pending_thread_resume_request( sandbox: sandbox_policy.into(), reasoning_effort, }; + let token_usage_thread = response.thread.clone(); + let token_usage_turn_id = latest_token_usage_turn_id_from_rollout_path( + pending.rollout_path.as_path(), + &token_usage_thread, + ) + .await; outgoing.send_response(request_id, response).await; + // Rejoining a loaded thread has the same UI contract as a cold resume, but + // uses the live conversation state instead of reconstructing a new session. + send_thread_token_usage_update_to_connection( + outgoing, + connection_id, + conversation_id, + &token_usage_thread, + conversation.as_ref(), + token_usage_turn_id, + ) + .await; outgoing .replay_requests_to_connection_for_thread(connection_id, conversation_id) .await; diff --git a/codex-rs/app-server/src/codex_message_processor/token_usage_replay.rs b/codex-rs/app-server/src/codex_message_processor/token_usage_replay.rs new file mode 100644 index 0000000000..d3c06f5e7c --- /dev/null +++ b/codex-rs/app-server/src/codex_message_processor/token_usage_replay.rs @@ -0,0 +1,133 @@ +//! Replays persisted token usage snapshots when a client attaches to an existing thread. +//! +//! The message processor decides when replay is allowed and preserves JSON-RPC response +//! ordering. This module owns notification construction and the attribution rules that +//! map the latest persisted `TokenCount` back to a v2 turn id. +//! +//! Rollout histories can contain explicit turn ids or generated turn ids. When explicit +//! ids do not match the rebuilt thread, replay falls back to the active turn position at +//! the time the `TokenCount` was persisted so the notification still targets the +//! corresponding rebuilt turn. + +use std::path::Path; +use std::sync::Arc; + +use codex_app_server_protocol::ServerNotification; +use codex_app_server_protocol::Thread; +use codex_app_server_protocol::ThreadHistoryBuilder; +use codex_app_server_protocol::ThreadTokenUsage; +use codex_app_server_protocol::ThreadTokenUsageUpdatedNotification; +use codex_app_server_protocol::TurnStatus; +use codex_core::CodexThread; +use codex_protocol::ThreadId; +use codex_protocol::protocol::EventMsg; +use codex_protocol::protocol::RolloutItem; + +use crate::codex_message_processor::read_rollout_items_from_rollout; +use crate::outgoing_message::ConnectionId; +use crate::outgoing_message::OutgoingMessageSender; + +/// Sends a restored token usage update to the connection that attached to a thread. +/// +/// This is lifecycle replay rather than a model event: the rollout already contains +/// the original `TokenCount`, and emitting through `send_event` here would duplicate +/// persisted usage records. Keeping this helper connection-scoped also avoids +/// surprising other subscribers with a historical usage update while they may be +/// rendering live turn events. +pub(super) async fn send_thread_token_usage_update_to_connection( + outgoing: &Arc, + connection_id: ConnectionId, + thread_id: ThreadId, + thread: &Thread, + conversation: &CodexThread, + token_usage_turn_id: Option, +) { + let Some(info) = conversation.token_usage_info().await else { + return; + }; + let notification = ThreadTokenUsageUpdatedNotification { + thread_id: thread_id.to_string(), + turn_id: token_usage_turn_id.unwrap_or_else(|| latest_token_usage_turn_id(thread)), + token_usage: ThreadTokenUsage::from(info), + }; + outgoing + .send_server_notification_to_connections( + &[connection_id], + ServerNotification::ThreadTokenUsageUpdated(notification), + ) + .await; +} + +pub(super) async fn latest_token_usage_turn_id_for_thread_path(thread: &Thread) -> Option { + let rollout_path = thread.path.as_deref()?; + latest_token_usage_turn_id_from_rollout_path(rollout_path, thread).await +} + +pub(super) async fn latest_token_usage_turn_id_from_rollout_path( + rollout_path: &Path, + thread: &Thread, +) -> Option { + let rollout_items = read_rollout_items_from_rollout(rollout_path).await.ok()?; + latest_token_usage_turn_id_from_rollout_items(&rollout_items, thread) +} + +/// Identifies the turn that was active when a `TokenCount` record appeared. +/// +/// The id is preferred when it still appears in the rebuilt thread. The position is a +/// fallback for histories whose implicit turn ids are regenerated during reconstruction. +struct TokenUsageTurnOwner { + id: String, + position: Option, +} + +pub(super) fn latest_token_usage_turn_id_from_rollout_items( + rollout_items: &[RolloutItem], + thread: &Thread, +) -> Option { + let owner = latest_token_usage_turn_owner_from_rollout_items(rollout_items)?; + if thread.turns.iter().any(|turn| turn.id == owner.id) { + return Some(owner.id); + } + owner + .position + .and_then(|position| thread.turns.get(position)) + .map(|turn| turn.id.clone()) +} + +fn latest_token_usage_turn_owner_from_rollout_items( + rollout_items: &[RolloutItem], +) -> Option { + let mut builder = ThreadHistoryBuilder::new(); + let mut token_usage_turn_owner = None; + + for item in rollout_items { + if matches!(item, RolloutItem::EventMsg(EventMsg::TokenCount(_))) { + token_usage_turn_owner = + builder + .active_turn_snapshot() + .map(|turn| TokenUsageTurnOwner { + id: turn.id, + position: builder.active_turn_position(), + }); + } + builder.handle_rollout_item(item); + } + + token_usage_turn_owner +} + +/// Chooses a fallback turn id that should own a replayed token usage update. +/// +/// Normal replay derives the owner from the rollout position of the latest +/// `TokenCount` event. This fallback only preserves a stable wire shape for +/// unusual histories where that rollout information cannot be read. +fn latest_token_usage_turn_id(thread: &Thread) -> String { + thread + .turns + .iter() + .rev() + .find(|turn| matches!(turn.status, TurnStatus::Completed | TurnStatus::Failed)) + .or_else(|| thread.turns.last()) + .map(|turn| turn.id.clone()) + .unwrap_or_default() +} diff --git a/codex-rs/app-server/tests/common/lib.rs b/codex-rs/app-server/tests/common/lib.rs index 6bb7372162..6ac26d8a56 100644 --- a/codex-rs/app-server/tests/common/lib.rs +++ b/codex-rs/app-server/tests/common/lib.rs @@ -40,6 +40,7 @@ pub use responses::create_shell_command_sse_response; pub use rollout::create_fake_rollout; pub use rollout::create_fake_rollout_with_source; pub use rollout::create_fake_rollout_with_text_elements; +pub use rollout::create_fake_rollout_with_token_usage; pub use rollout::rollout_path; use serde::de::DeserializeOwned; diff --git a/codex-rs/app-server/tests/common/rollout.rs b/codex-rs/app-server/tests/common/rollout.rs index b67390154e..06b273754c 100644 --- a/codex-rs/app-server/tests/common/rollout.rs +++ b/codex-rs/app-server/tests/common/rollout.rs @@ -1,9 +1,13 @@ use anyhow::Result; use codex_protocol::ThreadId; +use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::GitInfo; use codex_protocol::protocol::SessionMeta; use codex_protocol::protocol::SessionMetaLine; use codex_protocol::protocol::SessionSource; +use codex_protocol::protocol::TokenCountEvent; +use codex_protocol::protocol::TokenUsage; +use codex_protocol::protocol::TokenUsageInfo; use serde_json::json; use std::fs; use std::fs::FileTimes; @@ -50,6 +54,61 @@ pub fn create_fake_rollout( ) } +/// Creates a minimal rollout whose history includes a persisted token usage event. +/// +/// Resume and fork tests use this fixture to verify lifecycle replay of restored +/// usage without starting a model turn. The exact token values are intentionally +/// non-zero and asymmetric so assertions catch swapped total/last fields and +/// dropped cached or reasoning counters. +pub fn create_fake_rollout_with_token_usage( + codex_home: &Path, + filename_ts: &str, + meta_rfc3339: &str, + preview: &str, + model_provider: Option<&str>, +) -> Result { + let thread_id = create_fake_rollout( + codex_home, + filename_ts, + meta_rfc3339, + preview, + model_provider, + /*git_info*/ None, + )?; + let payload = serde_json::to_value(EventMsg::TokenCount(TokenCountEvent { + info: Some(TokenUsageInfo { + total_token_usage: TokenUsage { + input_tokens: 120, + cached_input_tokens: 20, + output_tokens: 30, + reasoning_output_tokens: 10, + total_tokens: 150, + }, + last_token_usage: TokenUsage { + input_tokens: 70, + cached_input_tokens: 10, + output_tokens: 20, + reasoning_output_tokens: 5, + total_tokens: 90, + }, + model_context_window: Some(200_000), + }), + rate_limits: None, + }))?; + let file_path = rollout_path(codex_home, filename_ts, &thread_id); + let line = json!({ + "timestamp": meta_rfc3339, + "type": "event_msg", + "payload": payload + }) + .to_string(); + fs::write( + &file_path, + format!("{}{}\n", fs::read_to_string(&file_path)?, line), + )?; + Ok(thread_id) +} + /// Create a minimal rollout file with an explicit session source. pub fn create_fake_rollout_with_source( codex_home: &Path, diff --git a/codex-rs/app-server/tests/suite/v2/thread_fork.rs b/codex-rs/app-server/tests/suite/v2/thread_fork.rs index 649919db8e..f5bc5a73c9 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_fork.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_fork.rs @@ -2,6 +2,7 @@ use anyhow::Result; use app_test_support::ChatGptAuthFixture; use app_test_support::McpProcess; use app_test_support::create_fake_rollout; +use app_test_support::create_fake_rollout_with_token_usage; use app_test_support::create_mock_responses_server_repeating_assistant; use app_test_support::to_response; use app_test_support::write_chatgpt_auth; @@ -9,6 +10,7 @@ use codex_app_server_protocol::JSONRPCError; use codex_app_server_protocol::JSONRPCMessage; use codex_app_server_protocol::JSONRPCResponse; use codex_app_server_protocol::RequestId; +use codex_app_server_protocol::ServerNotification; use codex_app_server_protocol::SessionSource; use codex_app_server_protocol::ThreadForkParams; use codex_app_server_protocol::ThreadForkResponse; @@ -186,6 +188,59 @@ async fn thread_fork_creates_new_thread_and_emits_started() -> Result<()> { Ok(()) } +#[tokio::test] +async fn thread_fork_emits_restored_token_usage_before_next_turn() -> Result<()> { + let server = create_mock_responses_server_repeating_assistant("Done").await; + let codex_home = TempDir::new()?; + create_config_toml(codex_home.path(), &server.uri())?; + + let conversation_id = create_fake_rollout_with_token_usage( + codex_home.path(), + "2025-01-05T12-00-00", + "2025-01-05T12:00:00Z", + "Saved user message", + Some("mock_provider"), + )?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let fork_id = mcp + .send_thread_fork_request(ThreadForkParams { + thread_id: conversation_id, + ..Default::default() + }) + .await?; + let fork_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(fork_id)), + ) + .await??; + let ThreadForkResponse { thread, .. } = to_response::(fork_resp)?; + + let note = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("thread/tokenUsage/updated"), + ) + .await??; + let parsed: ServerNotification = note.try_into()?; + let ServerNotification::ThreadTokenUsageUpdated(notification) = parsed else { + panic!("expected thread/tokenUsage/updated notification"); + }; + + assert_eq!(notification.thread_id, thread.id); + assert_eq!(notification.turn_id, thread.turns[0].id); + assert_eq!(notification.token_usage.total.total_tokens, 150); + assert_eq!(notification.token_usage.total.input_tokens, 120); + assert_eq!(notification.token_usage.total.cached_input_tokens, 20); + assert_eq!(notification.token_usage.total.output_tokens, 30); + assert_eq!(notification.token_usage.total.reasoning_output_tokens, 10); + assert_eq!(notification.token_usage.last.total_tokens, 90); + assert_eq!(notification.token_usage.model_context_window, Some(200_000)); + + Ok(()) +} + #[tokio::test] async fn thread_fork_tracks_thread_initialized_analytics() -> Result<()> { let server = create_mock_responses_server_repeating_assistant("Done").await; diff --git a/codex-rs/app-server/tests/suite/v2/thread_resume.rs b/codex-rs/app-server/tests/suite/v2/thread_resume.rs index 46dee9b5d1..edf792ca1d 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_resume.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_resume.rs @@ -3,6 +3,7 @@ use app_test_support::ChatGptAuthFixture; use app_test_support::McpProcess; use app_test_support::create_apply_patch_sse_response; use app_test_support::create_fake_rollout_with_text_elements; +use app_test_support::create_fake_rollout_with_token_usage; use app_test_support::create_final_assistant_message_sse_response; use app_test_support::create_mock_responses_server_repeating_assistant; use app_test_support::create_mock_responses_server_sequence_unchecked; @@ -23,6 +24,7 @@ use codex_app_server_protocol::JSONRPCResponse; use codex_app_server_protocol::PatchApplyStatus; use codex_app_server_protocol::PatchChangeKind; use codex_app_server_protocol::RequestId; +use codex_app_server_protocol::ServerNotification; use codex_app_server_protocol::ServerRequest; use codex_app_server_protocol::SessionSource; use codex_app_server_protocol::ThreadItem; @@ -50,6 +52,11 @@ use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::SessionMeta; use codex_protocol::protocol::SessionMetaLine; use codex_protocol::protocol::SessionSource as RolloutSessionSource; +use codex_protocol::protocol::TokenCountEvent; +use codex_protocol::protocol::TokenUsage; +use codex_protocol::protocol::TokenUsageInfo; +use codex_protocol::protocol::TurnAbortReason; +use codex_protocol::protocol::TurnAbortedEvent; use codex_protocol::protocol::TurnStartedEvent; use codex_protocol::user_input::ByteRange; use codex_protocol::user_input::TextElement; @@ -278,6 +285,268 @@ async fn thread_resume_returns_rollout_history() -> Result<()> { Ok(()) } +#[tokio::test] +async fn thread_resume_emits_restored_token_usage_before_next_turn() -> Result<()> { + let server = create_mock_responses_server_repeating_assistant("Done").await; + let codex_home = TempDir::new()?; + create_config_toml(codex_home.path(), &server.uri())?; + + let conversation_id = create_fake_rollout_with_token_usage( + codex_home.path(), + "2025-01-05T12-00-00", + "2025-01-05T12:00:00Z", + "Saved user message", + Some("mock_provider"), + )?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let resume_id = mcp + .send_thread_resume_request(ThreadResumeParams { + thread_id: conversation_id, + ..Default::default() + }) + .await?; + let resume_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(resume_id)), + ) + .await??; + let ThreadResumeResponse { thread, .. } = to_response::(resume_resp)?; + + let note = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("thread/tokenUsage/updated"), + ) + .await??; + let parsed: ServerNotification = note.try_into()?; + let ServerNotification::ThreadTokenUsageUpdated(notification) = parsed else { + panic!("expected thread/tokenUsage/updated notification"); + }; + + assert_eq!(notification.thread_id, thread.id); + assert_eq!(notification.turn_id, thread.turns[0].id); + assert_eq!(notification.token_usage.total.total_tokens, 150); + assert_eq!(notification.token_usage.total.input_tokens, 120); + assert_eq!(notification.token_usage.total.cached_input_tokens, 20); + assert_eq!(notification.token_usage.total.output_tokens, 30); + assert_eq!(notification.token_usage.total.reasoning_output_tokens, 10); + assert_eq!(notification.token_usage.last.total_tokens, 90); + assert_eq!(notification.token_usage.model_context_window, Some(200_000)); + + Ok(()) +} + +#[tokio::test] +async fn thread_resume_token_usage_replay_ignores_stale_interrupted_tail_turn() -> Result<()> { + let server = create_mock_responses_server_repeating_assistant("Done").await; + let codex_home = TempDir::new()?; + create_config_toml(codex_home.path(), &server.uri())?; + + let filename_ts = "2025-01-05T12-00-00"; + let meta_rfc3339 = "2025-01-05T12:00:00Z"; + let conversation_id = create_fake_rollout_with_token_usage( + codex_home.path(), + filename_ts, + meta_rfc3339, + "Saved user message", + Some("mock_provider"), + )?; + let rollout_file_path = rollout_path(codex_home.path(), filename_ts, &conversation_id); + let persisted_rollout = std::fs::read_to_string(&rollout_file_path)?; + let stale_turn_id = "incomplete-turn-after-token-usage"; + let appended_rollout = [ + json!({ + "timestamp": meta_rfc3339, + "type": "event_msg", + "payload": serde_json::to_value(EventMsg::TurnStarted(TurnStartedEvent { + turn_id: stale_turn_id.to_string(), + started_at: None, + model_context_window: None, + collaboration_mode_kind: Default::default(), + }))?, + }) + .to_string(), + json!({ + "timestamp": meta_rfc3339, + "type": "event_msg", + "payload": serde_json::to_value(EventMsg::AgentMessage(AgentMessageEvent { + message: "Still running".to_string(), + phase: None, + memory_citation: None, + }))?, + }) + .to_string(), + ] + .join("\n"); + std::fs::write( + &rollout_file_path, + format!("{persisted_rollout}{appended_rollout}\n"), + )?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let resume_id = mcp + .send_thread_resume_request(ThreadResumeParams { + thread_id: conversation_id, + ..Default::default() + }) + .await?; + let resume_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(resume_id)), + ) + .await??; + let ThreadResumeResponse { thread, .. } = to_response::(resume_resp)?; + + assert_eq!(thread.turns.len(), 2); + assert_eq!(thread.turns[0].status, TurnStatus::Completed); + assert_eq!(thread.turns[1].id, stale_turn_id); + assert_eq!(thread.turns[1].status, TurnStatus::Interrupted); + + let note = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("thread/tokenUsage/updated"), + ) + .await??; + let parsed: ServerNotification = note.try_into()?; + let ServerNotification::ThreadTokenUsageUpdated(notification) = parsed else { + panic!("expected thread/tokenUsage/updated notification"); + }; + + assert_eq!(notification.thread_id, thread.id); + assert_eq!(notification.turn_id, thread.turns[0].id); + assert_ne!(notification.turn_id, stale_turn_id); + assert_eq!(notification.token_usage.total.total_tokens, 150); + assert_eq!(notification.token_usage.last.total_tokens, 90); + + Ok(()) +} + +#[tokio::test] +async fn thread_resume_token_usage_replay_can_belong_to_interrupted_turn() -> Result<()> { + let server = create_mock_responses_server_repeating_assistant("Done").await; + let codex_home = TempDir::new()?; + create_config_toml(codex_home.path(), &server.uri())?; + + let filename_ts = "2025-01-05T12-00-00"; + let meta_rfc3339 = "2025-01-05T12:00:00Z"; + let conversation_id = create_fake_rollout_with_token_usage( + codex_home.path(), + filename_ts, + meta_rfc3339, + "Saved user message", + Some("mock_provider"), + )?; + let rollout_file_path = rollout_path(codex_home.path(), filename_ts, &conversation_id); + let persisted_rollout = std::fs::read_to_string(&rollout_file_path)?; + let interrupted_turn_id = "interrupted-turn-with-token-usage"; + let appended_rollout = [ + json!({ + "timestamp": meta_rfc3339, + "type": "event_msg", + "payload": serde_json::to_value(EventMsg::TurnStarted(TurnStartedEvent { + turn_id: interrupted_turn_id.to_string(), + started_at: None, + model_context_window: None, + collaboration_mode_kind: Default::default(), + }))?, + }) + .to_string(), + json!({ + "timestamp": meta_rfc3339, + "type": "event_msg", + "payload": serde_json::to_value(EventMsg::AgentMessage(AgentMessageEvent { + message: "Interrupted after usage".to_string(), + phase: None, + memory_citation: None, + }))?, + }) + .to_string(), + json!({ + "timestamp": meta_rfc3339, + "type": "event_msg", + "payload": serde_json::to_value(EventMsg::TokenCount(TokenCountEvent { + info: Some(TokenUsageInfo { + total_token_usage: TokenUsage { + input_tokens: 180, + cached_input_tokens: 40, + output_tokens: 50, + reasoning_output_tokens: 15, + total_tokens: 230, + }, + last_token_usage: TokenUsage { + input_tokens: 90, + cached_input_tokens: 30, + output_tokens: 40, + reasoning_output_tokens: 12, + total_tokens: 130, + }, + model_context_window: Some(200_000), + }), + rate_limits: None, + }))?, + }) + .to_string(), + json!({ + "timestamp": meta_rfc3339, + "type": "event_msg", + "payload": serde_json::to_value(EventMsg::TurnAborted(TurnAbortedEvent { + turn_id: Some(interrupted_turn_id.to_string()), + reason: TurnAbortReason::Interrupted, + completed_at: None, + duration_ms: None, + }))?, + }) + .to_string(), + ] + .join("\n"); + std::fs::write( + &rollout_file_path, + format!("{persisted_rollout}{appended_rollout}\n"), + )?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let resume_id = mcp + .send_thread_resume_request(ThreadResumeParams { + thread_id: conversation_id, + ..Default::default() + }) + .await?; + let resume_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(resume_id)), + ) + .await??; + let ThreadResumeResponse { thread, .. } = to_response::(resume_resp)?; + + assert_eq!(thread.turns.len(), 2); + assert_eq!(thread.turns[0].status, TurnStatus::Completed); + assert_eq!(thread.turns[1].id, interrupted_turn_id); + assert_eq!(thread.turns[1].status, TurnStatus::Interrupted); + + let note = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("thread/tokenUsage/updated"), + ) + .await??; + let parsed: ServerNotification = note.try_into()?; + let ServerNotification::ThreadTokenUsageUpdated(notification) = parsed else { + panic!("expected thread/tokenUsage/updated notification"); + }; + + assert_eq!(notification.thread_id, thread.id); + assert_eq!(notification.turn_id, interrupted_turn_id); + assert_eq!(notification.token_usage.total.total_tokens, 230); + assert_eq!(notification.token_usage.last.total_tokens, 130); + + Ok(()) +} + #[tokio::test] async fn thread_resume_prefers_persisted_git_metadata_for_local_threads() -> Result<()> { let server = create_mock_responses_server_repeating_assistant("Done").await; diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 844d585be5..2cff1d3174 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -2420,6 +2420,17 @@ impl Session { state.token_info().map(|info| info.total_token_usage) } + /// Returns the complete token usage snapshot currently cached for this session. + /// + /// Resume and fork reconstruction seed this state from the last persisted rollout + /// `TokenCount` event. Callers that need to replay restored usage to a client + /// should use this accessor instead of `total_token_usage`, because the app-server + /// notification includes both total and last-turn usage. + pub(crate) async fn token_usage_info(&self) -> Option { + let state = self.state.lock().await; + state.token_info() + } + pub(crate) async fn get_estimated_token_count( &self, turn_context: &TurnContext, diff --git a/codex-rs/core/src/codex_thread.rs b/codex-rs/core/src/codex_thread.rs index fa9f835181..3642d078c1 100644 --- a/codex-rs/core/src/codex_thread.rs +++ b/codex-rs/core/src/codex_thread.rs @@ -22,6 +22,7 @@ use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::Submission; use codex_protocol::protocol::ThreadMemoryMode; use codex_protocol::protocol::TokenUsage; +use codex_protocol::protocol::TokenUsageInfo; use codex_protocol::protocol::W3cTraceContext; use codex_protocol::user_input::UserInput; use codex_utils_absolute_path::AbsolutePathBuf; @@ -144,6 +145,17 @@ impl CodexThread { self.codex.session.total_token_usage().await } + /// Returns the complete token usage snapshot currently cached for this thread. + /// + /// This accessor is intentionally narrower than direct session access: it lets + /// app-server lifecycle paths replay restored usage after resume or fork without + /// exposing broader session mutation authority. A caller that only reads + /// `total_token_usage` would drop last-turn usage and make the v2 + /// `thread/tokenUsage/updated` payload incomplete. + pub async fn token_usage_info(&self) -> Option { + self.codex.session.token_usage_info().await + } + /// Records a user-role session-prefix message without creating a new user turn boundary. pub(crate) async fn inject_user_message_without_turn(&self, message: String) { let message = ResponseItem::Message { From 37bf42d5d560854b8e04801cc62685e73e87ae8b Mon Sep 17 00:00:00 2001 From: bxie-openai Date: Thu, 16 Apr 2026 13:51:43 -0700 Subject: [PATCH 38/59] [codex] Make realtime startup context truncation deterministic (#18172) ## Summary - remove the final whole-blob truncation pass from realtime startup-context assembly - enforce fixed per-section budgets, including each section heading - keep the existing per-section caps and raise the overall realtime startup-context budget to `5300`, matching the sum of those section budgets - add focused tests for the new wrapping and section-budget behavior ## Why The previous flow truncated each section and then middle-truncated the final combined startup-context blob again. Small input changes could shift that combined cut point, which made retained context unstable and caused nondeterministic tests. ## Impact Startup context now preserves section boundaries and ordering deterministically. Each section is still budgeted independently, but the final assembled blob is no longer truncated again as a single opaque string. To match that design, the overall startup-context token budget is updated to the sum of the existing section budgets rather than lowering the section caps. ## Validation - `cargo +1.93.0 test -p codex-core realtime_context` - `cargo +1.93.0 test -p codex-core --test all suite::realtime_conversation::conversation_start_injects_startup_context_from_thread_history -- --exact` - `cargo +1.93.0 test -p codex-core --test all suite::realtime_conversation::conversation_startup_context_current_thread_selects_many_turns_by_budget -- --exact` - `cargo +1.93.0 test -p codex-core --test all suite::realtime_conversation::conversation_startup_context_falls_back_to_workspace_map -- --exact` - `cargo +1.93.0 test -p codex-core --test all suite::realtime_conversation::conversation_startup_context_is_truncated_and_sent_once_per_start -- --exact` --- codex-rs/core/src/realtime_context.rs | 40 ++++++-------- codex-rs/core/src/realtime_context_tests.rs | 60 +++++++++++++++++---- codex-rs/core/src/realtime_conversation.rs | 2 +- 3 files changed, 68 insertions(+), 34 deletions(-) diff --git a/codex-rs/core/src/realtime_context.rs b/codex-rs/core/src/realtime_context.rs index fc17dab99b..c23394fa37 100644 --- a/codex-rs/core/src/realtime_context.rs +++ b/codex-rs/core/src/realtime_context.rs @@ -108,9 +108,10 @@ pub(crate) async fn build_realtime_startup_context( parts.push(section); } - let context = format_startup_context_blob(&parts.join("\n\n"), budget_tokens); + let context = format_startup_context_blob(&parts.join("\n\n")); debug!( approx_tokens = approx_token_count(&context), + requested_budget_tokens = budget_tokens, bytes = context.len(), has_current_thread_section, has_recent_work_section, @@ -440,31 +441,22 @@ fn format_section(title: &str, body: Option, budget_tokens: usize) -> Op return None; } - Some(format!( - "## {title}\n{}", - truncate_text(body, TruncationPolicy::Tokens(budget_tokens)) - )) + let heading = format!("## {title}\n"); + let body_budget = budget_tokens.saturating_sub(approx_token_count(&heading)); + if body_budget == 0 { + return None; + } + + let body = truncate_realtime_text_to_token_budget(body, body_budget); + if body.is_empty() { + return None; + } + + Some(format!("{heading}{body}")) } -fn format_startup_context_blob(body: &str, budget_tokens: usize) -> String { - let wrapper = format!("{STARTUP_CONTEXT_OPEN_TAG}\n\n{STARTUP_CONTEXT_CLOSE_TAG}"); - let mut body_budget = budget_tokens.saturating_sub(approx_token_count(&wrapper)); - - loop { - let body = truncate_text(body, TruncationPolicy::Tokens(body_budget)); - let wrapped = format!("{STARTUP_CONTEXT_OPEN_TAG}\n{body}\n{STARTUP_CONTEXT_CLOSE_TAG}"); - let wrapped_tokens = approx_token_count(&wrapped); - if wrapped_tokens <= budget_tokens || body_budget == 0 { - return wrapped; - } - - let excess_tokens = wrapped_tokens.saturating_sub(budget_tokens); - let next_budget = body_budget.saturating_sub(excess_tokens.max(1)); - if next_budget == body_budget { - return wrapped; - } - body_budget = next_budget; - } +fn format_startup_context_blob(body: &str) -> String { + format!("{STARTUP_CONTEXT_OPEN_TAG}\n{body}\n{STARTUP_CONTEXT_CLOSE_TAG}") } async fn format_thread_group( diff --git a/codex-rs/core/src/realtime_context_tests.rs b/codex-rs/core/src/realtime_context_tests.rs index 4f62affcae..99cb00f0de 100644 --- a/codex-rs/core/src/realtime_context_tests.rs +++ b/codex-rs/core/src/realtime_context_tests.rs @@ -1,6 +1,12 @@ +use super::CURRENT_THREAD_SECTION_TOKEN_BUDGET; +use super::NOTES_SECTION_TOKEN_BUDGET; +use super::RECENT_WORK_SECTION_TOKEN_BUDGET; +use super::STARTUP_CONTEXT_HEADER; +use super::WORKSPACE_SECTION_TOKEN_BUDGET; use super::build_current_thread_section; use super::build_recent_work_section; use super::build_workspace_section_with_user_root; +use super::format_section; use super::format_startup_context_blob; use chrono::TimeZone; use chrono::Utc; @@ -172,20 +178,56 @@ fn current_thread_section_keeps_latest_turns_when_history_exceeds_budget() { } #[test] -fn startup_context_blob_is_wrapped_in_tags_and_fits_budget() { - let body = format!( - "Startup context from Codex.\n{}\n{}", - "recent work ".repeat(1_200), - "workspace tree ".repeat(800), - ); +fn startup_context_blob_is_wrapped_in_tags_without_final_truncation() { + let body = "Startup context from Codex.\n## Current Thread\nhello"; + let wrapped = format_startup_context_blob(body); - let wrapped = format_startup_context_blob(&body, /*budget_tokens*/ 200); + assert_eq!( + wrapped, + "\nStartup context from Codex.\n## Current Thread\nhello\n" + ); +} + +#[test] +fn fixed_section_budgets_apply_per_section_without_total_blob_truncation() { + let body = [ + STARTUP_CONTEXT_HEADER.to_string(), + format_section( + "Current Thread", + Some("current thread ".repeat(2_000)), + CURRENT_THREAD_SECTION_TOKEN_BUDGET, + ) + .expect("current thread section"), + format_section( + "Recent Work", + Some("recent work ".repeat(3_000)), + RECENT_WORK_SECTION_TOKEN_BUDGET, + ) + .expect("recent work section"), + format_section( + "Machine / Workspace Map", + Some("workspace map ".repeat(2_500)), + WORKSPACE_SECTION_TOKEN_BUDGET, + ) + .expect("workspace section"), + format_section( + "Notes", + Some("notes ".repeat(500)), + NOTES_SECTION_TOKEN_BUDGET, + ) + .expect("notes section"), + ] + .join("\n\n"); + + let wrapped = format_startup_context_blob(&body); assert!(wrapped.starts_with("\n")); assert!(wrapped.ends_with("\n")); - assert!(wrapped.contains("Startup context from Codex.")); assert!(wrapped.contains("tokens truncated")); - assert!(wrapped.len().div_ceil(4) <= 200); + assert!(wrapped.contains("## Current Thread")); + assert!(wrapped.contains("## Recent Work")); + assert!(wrapped.contains("## Machine / Workspace Map")); + assert!(wrapped.contains("## Notes")); } #[tokio::test] diff --git a/codex-rs/core/src/realtime_conversation.rs b/codex-rs/core/src/realtime_conversation.rs index b79060259c..234dd78f38 100644 --- a/codex-rs/core/src/realtime_conversation.rs +++ b/codex-rs/core/src/realtime_conversation.rs @@ -63,7 +63,7 @@ const AUDIO_IN_QUEUE_CAPACITY: usize = 256; const USER_TEXT_IN_QUEUE_CAPACITY: usize = 64; const HANDOFF_OUT_QUEUE_CAPACITY: usize = 64; const OUTPUT_EVENTS_QUEUE_CAPACITY: usize = 256; -const REALTIME_STARTUP_CONTEXT_TOKEN_BUDGET: usize = 5_000; +const REALTIME_STARTUP_CONTEXT_TOKEN_BUDGET: usize = 5_300; const DEFAULT_REALTIME_MODEL: &str = "gpt-realtime-1.5"; pub(crate) const REALTIME_USER_TEXT_PREFIX: &str = "[USER] "; pub(crate) const REALTIME_BACKEND_TEXT_PREFIX: &str = "[BACKEND] "; From 3905f728918abd6ebaa63a4368f52754c7a3e26c Mon Sep 17 00:00:00 2001 From: starr-openai Date: Thu, 16 Apr 2026 14:16:15 -0700 Subject: [PATCH 39/59] Throttle Windows Bazel test concurrency (#18192) ## Summary - cap the Windows Bazel test lane at `--jobs=8` to reduce local runner pressure - keep Linux and macOS Bazel test concurrency unchanged - make failed-test log tailing resolve `bazel-testlogs` with the same CI config and Windows host-platform context as the failed invocation - prefer Bazel-reported `test.log` paths and normalize Windows path separators before tailing ## Context The Windows Bazel workflow currently uses `ci-windows`, which does not inherit the remote executor config. This means the lane runs the `//...` test suite locally and otherwise falls back to the repo-wide `common --jobs=30`. The new Windows-only override is intended to reduce local executor pressure without changing coverage. ## Validation Not run locally; this is a CI workflow change and the draft PR is intended to exercise the GitHub Actions lane directly. --------- Co-authored-by: Codex --- .github/scripts/run-bazel-ci.sh | 30 ++++++++++++++++++++++++++++-- .github/workflows/bazel.yml | 12 ++++++++---- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/.github/scripts/run-bazel-ci.sh b/.github/scripts/run-bazel-ci.sh index c8c09f406a..a1d42a0576 100755 --- a/.github/scripts/run-bazel-ci.sh +++ b/.github/scripts/run-bazel-ci.sh @@ -69,12 +69,37 @@ print_bazel_test_log_tails() { local console_log="$1" local testlogs_dir local -a bazel_info_cmd=(bazel) + local -a bazel_info_args=(info) if (( ${#bazel_startup_args[@]} > 0 )); then bazel_info_cmd+=("${bazel_startup_args[@]}") fi - testlogs_dir="$(run_bazel "${bazel_info_cmd[@]:1}" info bazel-testlogs 2>/dev/null || echo bazel-testlogs)" + # `bazel info` needs the same CI config as the failed test invocation so + # platform-specific output roots match. On Windows, omitting `ci-windows` + # would point at `local_windows-fastbuild` even when the test ran with the + # MSVC host platform under `local_windows_msvc-fastbuild`. + if [[ -n "${BUILDBUDDY_API_KEY:-}" ]]; then + bazel_info_args+=( + "--config=${ci_config}" + "--remote_header=x-buildbuddy-api-key=${BUILDBUDDY_API_KEY}" + ) + fi + # Only pass flags that affect Bazel's output-root selection or repository + # lookup. Test/build-only flags such as execution logs or remote download + # mode can make `bazel info` fail, which would hide the real test log path. + for arg in "${post_config_bazel_args[@]}"; do + case "$arg" in + --host_platform=* | --repo_contents_cache=* | --repository_cache=*) + bazel_info_args+=("$arg") + ;; + esac + done + + testlogs_dir="$(run_bazel "${bazel_info_cmd[@]:1}" \ + --noexperimental_remote_repo_contents_cache \ + "${bazel_info_args[@]}" \ + bazel-testlogs 2>/dev/null || echo bazel-testlogs)" local failed_targets=() while IFS= read -r target; do @@ -95,8 +120,9 @@ print_bazel_test_log_tails() { rel_path="${rel_path/://}" local test_log="${testlogs_dir}/${rel_path}/test.log" local reported_test_log - reported_test_log="$(grep -F "FAIL: ${target} " "$console_log" | sed -nE 's#.* \(see ([^)]+/test\.log)\).*#\1#p' | head -n 1 || true)" + reported_test_log="$(grep -F "FAIL: ${target} " "$console_log" | sed -nE 's#.* \(see (.*[\\/]test\.log)\).*#\1#p' | head -n 1 || true)" if [[ -n "$reported_test_log" ]]; then + reported_test_log="${reported_test_log//\\//}" test_log="$reported_test_log" fi diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 5d30916f9f..0328ac2073 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -86,17 +86,21 @@ jobs: --print-failed-test-logs --use-node-test-env ) + bazel_test_args=( + test + --test_tag_filters=-argument-comment-lint + --test_verbose_timeout_warnings + --build_metadata=COMMIT_SHA=${GITHUB_SHA} + ) if [[ "${RUNNER_OS}" == "Windows" ]]; then bazel_wrapper_args+=(--windows-msvc-host-platform) + bazel_test_args+=(--jobs=8) fi ./.github/scripts/run-bazel-ci.sh \ "${bazel_wrapper_args[@]}" \ -- \ - test \ - --test_tag_filters=-argument-comment-lint \ - --test_verbose_timeout_warnings \ - --build_metadata=COMMIT_SHA=${GITHUB_SHA} \ + "${bazel_test_args[@]}" \ -- \ "${bazel_targets[@]}" From 0708cc78cb9da20087e3013cab71e880290dc427 Mon Sep 17 00:00:00 2001 From: pakrym-oai Date: Thu, 16 Apr 2026 14:21:29 -0700 Subject: [PATCH 40/59] [codex] Split codex op handlers (#18200) Start splitting the codex.rs --- codex-rs/core/src/codex.rs | 1261 +-------------------------- codex-rs/core/src/codex/handlers.rs | 1230 ++++++++++++++++++++++++++ 2 files changed, 1234 insertions(+), 1257 deletions(-) create mode 100644 codex-rs/core/src/codex/handlers.rs diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 2cff1d3174..4e09542d88 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -30,10 +30,6 @@ use crate::mcp_tool_exposure::build_mcp_tool_exposure; use crate::parse_turn_item; use crate::path_utils::normalize_for_native_workdir; use crate::realtime_conversation::RealtimeConversationManager; -use crate::realtime_conversation::handle_audio as handle_realtime_conversation_audio; -use crate::realtime_conversation::handle_close as handle_realtime_conversation_close; -use crate::realtime_conversation::handle_start as handle_realtime_conversation_start; -use crate::realtime_conversation::handle_text as handle_realtime_conversation_text; use crate::render_skills_section; use crate::rollout::find_thread_name_by_id; use crate::session_prefix::format_subagent_notification_message; @@ -170,7 +166,6 @@ use tokio_util::sync::CancellationToken; use toml::Value as TomlValue; use tracing::Instrument; use tracing::debug; -use tracing::debug_span; use tracing::error; use tracing::field; use tracing::info; @@ -206,8 +201,12 @@ use codex_protocol::error::Result as CodexResult; #[cfg(test)] use codex_protocol::exec_output::StreamOutput; +mod handlers; mod rollout_reconstruction; #[cfg(test)] +use self::handlers::submission_dispatch_span; +use self::handlers::submission_loop; +#[cfg(test)] mod rollout_reconstruction_tests; #[derive(Debug, PartialEq)] @@ -4733,1258 +4732,6 @@ pub(crate) fn emit_subagent_session_started( }); } -async fn submission_loop(sess: Arc, config: Arc, rx_sub: Receiver) { - // To break out of this loop, send Op::Shutdown. - while let Ok(sub) = rx_sub.recv().await { - debug!(?sub, "Submission"); - let dispatch_span = submission_dispatch_span(&sub); - let should_exit = async { - match sub.op.clone() { - Op::Interrupt => { - handlers::interrupt(&sess).await; - false - } - Op::CleanBackgroundTerminals => { - handlers::clean_background_terminals(&sess).await; - false - } - Op::RealtimeConversationStart(params) => { - if let Err(err) = - handle_realtime_conversation_start(&sess, sub.id.clone(), params).await - { - sess.send_event_raw(Event { - id: sub.id.clone(), - msg: EventMsg::Error(ErrorEvent { - message: err.to_string(), - codex_error_info: Some(CodexErrorInfo::Other), - }), - }) - .await; - } - false - } - Op::RealtimeConversationAudio(params) => { - handle_realtime_conversation_audio(&sess, sub.id.clone(), params).await; - false - } - Op::RealtimeConversationText(params) => { - handle_realtime_conversation_text(&sess, sub.id.clone(), params).await; - false - } - Op::RealtimeConversationClose => { - handle_realtime_conversation_close(&sess, sub.id.clone()).await; - false - } - Op::RealtimeConversationListVoices => { - handlers::realtime_conversation_list_voices(&sess, sub.id.clone()).await; - false - } - Op::OverrideTurnContext { - cwd, - approval_policy, - approvals_reviewer, - sandbox_policy, - windows_sandbox_level, - model, - effort, - summary, - service_tier, - collaboration_mode, - personality, - } => { - let collaboration_mode = if let Some(collab_mode) = collaboration_mode { - collab_mode - } else { - let state = sess.state.lock().await; - state.session_configuration.collaboration_mode.with_updates( - model.clone(), - effort, - /*developer_instructions*/ None, - ) - }; - handlers::override_turn_context( - &sess, - sub.id.clone(), - SessionSettingsUpdate { - cwd, - approval_policy, - approvals_reviewer, - sandbox_policy, - windows_sandbox_level, - collaboration_mode: Some(collaboration_mode), - reasoning_summary: summary, - service_tier, - personality, - ..Default::default() - }, - ) - .await; - false - } - Op::UserInput { .. } | Op::UserTurn { .. } => { - handlers::user_input_or_turn(&sess, sub.id.clone(), sub.op).await; - false - } - Op::InterAgentCommunication { communication } => { - handlers::inter_agent_communication(&sess, sub.id.clone(), communication).await; - false - } - Op::ExecApproval { - id: approval_id, - turn_id, - decision, - } => { - handlers::exec_approval(&sess, approval_id, turn_id, decision).await; - false - } - Op::PatchApproval { id, decision } => { - handlers::patch_approval(&sess, id, decision).await; - false - } - Op::UserInputAnswer { id, response } => { - handlers::request_user_input_response(&sess, id, response).await; - false - } - Op::RequestPermissionsResponse { id, response } => { - handlers::request_permissions_response(&sess, id, response).await; - false - } - Op::DynamicToolResponse { id, response } => { - handlers::dynamic_tool_response(&sess, id, response).await; - false - } - Op::AddToHistory { text } => { - handlers::add_to_history(&sess, &config, text).await; - false - } - Op::GetHistoryEntryRequest { offset, log_id } => { - handlers::get_history_entry_request( - &sess, - &config, - sub.id.clone(), - offset, - log_id, - ) - .await; - false - } - Op::ListMcpTools => { - handlers::list_mcp_tools(&sess, &config, sub.id.clone()).await; - false - } - Op::RefreshMcpServers { config } => { - handlers::refresh_mcp_servers(&sess, config).await; - false - } - Op::ReloadUserConfig => { - handlers::reload_user_config(&sess).await; - false - } - Op::ListSkills { cwds, force_reload } => { - handlers::list_skills(&sess, sub.id.clone(), cwds, force_reload).await; - false - } - Op::Undo => { - handlers::undo(&sess, sub.id.clone()).await; - false - } - Op::Compact => { - handlers::compact(&sess, sub.id.clone()).await; - false - } - Op::DropMemories => { - handlers::drop_memories(&sess, &config, sub.id.clone()).await; - false - } - Op::UpdateMemories => { - handlers::update_memories(&sess, &config, sub.id.clone()).await; - false - } - Op::ThreadRollback { num_turns } => { - handlers::thread_rollback(&sess, sub.id.clone(), num_turns).await; - false - } - Op::SetThreadName { name } => { - handlers::set_thread_name(&sess, sub.id.clone(), name).await; - false - } - Op::SetThreadMemoryMode { mode } => { - handlers::set_thread_memory_mode(&sess, sub.id.clone(), mode).await; - false - } - Op::RunUserShellCommand { command } => { - handlers::run_user_shell_command(&sess, sub.id.clone(), command).await; - false - } - Op::ResolveElicitation { - server_name, - request_id, - decision, - content, - meta, - } => { - handlers::resolve_elicitation( - &sess, - server_name, - request_id, - decision, - content, - meta, - ) - .await; - false - } - Op::Shutdown => handlers::shutdown(&sess, sub.id.clone()).await, - Op::Review { review_request } => { - handlers::review(&sess, &config, sub.id.clone(), review_request).await; - false - } - _ => false, // Ignore unknown ops; enum is non_exhaustive to allow extensions. - } - } - .instrument(dispatch_span) - .await; - if should_exit { - break; - } - } - // Also drain cached guardian state if the submission loop exits because - // the channel closed without receiving an explicit shutdown op. - sess.guardian_review_session.shutdown().await; - debug!("Agent loop exited"); -} - -fn submission_dispatch_span(sub: &Submission) -> tracing::Span { - let op_name = sub.op.kind(); - let span_name = format!("op.dispatch.{op_name}"); - let dispatch_span = match &sub.op { - Op::RealtimeConversationAudio(_) => { - debug_span!( - "submission_dispatch", - otel.name = span_name.as_str(), - submission.id = sub.id.as_str(), - codex.op = op_name - ) - } - _ => info_span!( - "submission_dispatch", - otel.name = span_name.as_str(), - submission.id = sub.id.as_str(), - codex.op = op_name - ), - }; - if let Some(trace) = sub.trace.as_ref() - && !set_parent_from_w3c_trace_context(&dispatch_span, trace) - { - warn!( - submission.id = sub.id.as_str(), - "ignoring invalid submission trace carrier" - ); - } - dispatch_span -} - -/// Operation handlers -mod handlers { - use crate::codex::Session; - use crate::codex::SessionSettingsUpdate; - use crate::codex::SteerInputError; - - use crate::codex::spawn_review_thread; - use crate::config::Config; - use crate::config_loader::CloudRequirementsLoader; - use crate::config_loader::LoaderOverrides; - use crate::config_loader::load_config_layers_state; - use crate::realtime_context::REALTIME_TURN_TOKEN_BUDGET; - use crate::realtime_context::truncate_realtime_text_to_token_budget; - use crate::realtime_conversation::REALTIME_USER_TEXT_PREFIX; - use crate::realtime_conversation::prefix_realtime_v2_text; - use codex_features::Feature; - use codex_utils_absolute_path::AbsolutePathBuf; - - use crate::review_prompts::resolve_review_request; - use crate::rollout::RolloutRecorder; - use crate::rollout::read_session_meta_line; - use crate::tasks::CompactTask; - use crate::tasks::UndoTask; - use crate::tasks::UserShellCommandMode; - use crate::tasks::UserShellCommandTask; - use crate::tasks::execute_user_shell_command; - use codex_mcp::collect_mcp_snapshot_from_manager; - use codex_mcp::compute_auth_statuses; - use codex_protocol::protocol::CodexErrorInfo; - use codex_protocol::protocol::ErrorEvent; - use codex_protocol::protocol::Event; - use codex_protocol::protocol::EventMsg; - use codex_protocol::protocol::InterAgentCommunication; - use codex_protocol::protocol::ListSkillsResponseEvent; - use codex_protocol::protocol::McpServerRefreshConfig; - use codex_protocol::protocol::Op; - use codex_protocol::protocol::RealtimeConversationListVoicesResponseEvent; - use codex_protocol::protocol::RealtimeVoicesList; - use codex_protocol::protocol::ReviewDecision; - use codex_protocol::protocol::ReviewRequest; - use codex_protocol::protocol::RolloutItem; - use codex_protocol::protocol::SkillErrorInfo; - use codex_protocol::protocol::SkillsListEntry; - use codex_protocol::protocol::ThreadMemoryMode; - use codex_protocol::protocol::ThreadNameUpdatedEvent; - use codex_protocol::protocol::ThreadRolledBackEvent; - use codex_protocol::protocol::TurnAbortReason; - use codex_protocol::protocol::WarningEvent; - use codex_protocol::request_permissions::RequestPermissionsResponse; - use codex_protocol::request_user_input::RequestUserInputResponse; - - use crate::context_manager::is_user_turn_boundary; - use codex_protocol::config_types::CollaborationMode; - use codex_protocol::config_types::ModeKind; - use codex_protocol::config_types::Settings; - use codex_protocol::dynamic_tools::DynamicToolResponse; - use codex_protocol::items::UserMessageItem; - use codex_protocol::mcp::RequestId as ProtocolRequestId; - use codex_protocol::user_input::UserInput; - use codex_rmcp_client::ElicitationAction; - use codex_rmcp_client::ElicitationResponse; - use serde_json::Value; - use std::path::PathBuf; - use std::sync::Arc; - use tracing::debug; - use tracing::info; - use tracing::warn; - - pub async fn interrupt(sess: &Arc) { - sess.interrupt_task().await; - } - - pub async fn clean_background_terminals(sess: &Arc) { - sess.close_unified_exec_processes().await; - } - - pub async fn realtime_conversation_list_voices(sess: &Session, sub_id: String) { - sess.send_event_raw(Event { - id: sub_id, - msg: EventMsg::RealtimeConversationListVoicesResponse( - RealtimeConversationListVoicesResponseEvent { - voices: RealtimeVoicesList::builtin(), - }, - ), - }) - .await; - } - - pub async fn override_turn_context( - sess: &Session, - sub_id: String, - updates: SessionSettingsUpdate, - ) { - if let Err(err) = sess.update_settings(updates).await { - sess.send_event_raw(Event { - id: sub_id, - msg: EventMsg::Error(ErrorEvent { - message: err.to_string(), - codex_error_info: Some(CodexErrorInfo::BadRequest), - }), - }) - .await; - } - } - - pub async fn user_input_or_turn(sess: &Arc, sub_id: String, op: Op) { - user_input_or_turn_inner( - sess, - sub_id, - op, - /*mirror_user_text_to_realtime*/ Some(()), - ) - .await; - } - - pub(super) async fn user_input_or_turn_inner( - sess: &Arc, - sub_id: String, - op: Op, - mirror_user_text_to_realtime: Option<()>, - ) { - let (items, updates, responsesapi_client_metadata) = match op { - Op::UserTurn { - cwd, - approval_policy, - approvals_reviewer, - sandbox_policy, - model, - effort, - summary, - service_tier, - final_output_json_schema, - items, - collaboration_mode, - personality, - } => { - let collaboration_mode = collaboration_mode.or_else(|| { - Some(CollaborationMode { - mode: ModeKind::Default, - settings: Settings { - model: model.clone(), - reasoning_effort: effort, - developer_instructions: None, - }, - }) - }); - ( - items, - SessionSettingsUpdate { - cwd: Some(cwd), - approval_policy: Some(approval_policy), - approvals_reviewer, - sandbox_policy: Some(sandbox_policy), - windows_sandbox_level: None, - collaboration_mode, - reasoning_summary: summary, - service_tier, - final_output_json_schema: Some(final_output_json_schema), - personality, - app_server_client_name: None, - app_server_client_version: None, - }, - None, - ) - } - Op::UserInput { - items, - final_output_json_schema, - responsesapi_client_metadata, - } => ( - items, - SessionSettingsUpdate { - final_output_json_schema: Some(final_output_json_schema), - ..Default::default() - }, - responsesapi_client_metadata, - ), - _ => unreachable!(), - }; - - let Ok(current_context) = sess.new_turn_with_sub_id(sub_id.clone(), updates).await else { - // new_turn_with_sub_id already emits the error event. - return; - }; - sess.maybe_emit_unknown_model_warning_for_turn(current_context.as_ref()) - .await; - let accepted_items = match sess - .steer_input( - items.clone(), - /*expected_turn_id*/ None, - responsesapi_client_metadata.clone(), - ) - .await - { - Ok(_) => { - current_context.session_telemetry.user_prompt(&items); - Some(items) - } - Err(SteerInputError::NoActiveTurn(items)) => { - if let Some(responsesapi_client_metadata) = responsesapi_client_metadata { - current_context - .turn_metadata_state - .set_responsesapi_client_metadata(responsesapi_client_metadata); - } - current_context.session_telemetry.user_prompt(&items); - sess.refresh_mcp_servers_if_requested(¤t_context) - .await; - let accepted_items = items.clone(); - sess.spawn_task( - Arc::clone(¤t_context), - items, - crate::tasks::RegularTask::new(), - ) - .await; - Some(accepted_items) - } - Err(err) => { - sess.send_event_raw(Event { - id: sub_id, - msg: EventMsg::Error(err.to_error_event()), - }) - .await; - None - } - }; - if let (Some(items), Some(())) = (accepted_items, mirror_user_text_to_realtime) { - self::mirror_user_text_to_realtime(sess, &items).await; - } - } - - async fn mirror_user_text_to_realtime(sess: &Arc, items: &[UserInput]) { - let text = UserMessageItem::new(items).message(); - if text.is_empty() { - return; - } - let text = if sess.conversation.is_running_v2().await { - prefix_realtime_v2_text(text, REALTIME_USER_TEXT_PREFIX) - } else { - text - }; - let text = truncate_realtime_text_to_token_budget(&text, REALTIME_TURN_TOKEN_BUDGET); - if text.is_empty() { - return; - } - if sess.conversation.running_state().await.is_none() { - return; - } - if let Err(err) = sess.conversation.text_in(text).await { - debug!("failed to mirror user text to realtime conversation: {err}"); - } - } - - /// Records an inter-agent assistant envelope, then lets the shared pending-work scheduler - /// decide whether an idle session should start a regular turn. - pub async fn inter_agent_communication( - sess: &Arc, - sub_id: String, - communication: InterAgentCommunication, - ) { - let trigger_turn = communication.trigger_turn; - sess.enqueue_mailbox_communication(communication); - if trigger_turn { - sess.maybe_start_turn_for_pending_work_with_sub_id(sub_id) - .await; - } - } - - pub async fn run_user_shell_command(sess: &Arc, sub_id: String, command: String) { - if let Some((turn_context, cancellation_token)) = - sess.active_turn_context_and_cancellation_token().await - { - let session = Arc::clone(sess); - tokio::spawn(async move { - execute_user_shell_command( - session, - turn_context, - command, - cancellation_token, - UserShellCommandMode::ActiveTurnAuxiliary, - ) - .await; - }); - return; - } - - let turn_context = sess.new_default_turn_with_sub_id(sub_id).await; - sess.spawn_task( - Arc::clone(&turn_context), - Vec::new(), - UserShellCommandTask::new(command), - ) - .await; - } - - pub async fn resolve_elicitation( - sess: &Arc, - server_name: String, - request_id: ProtocolRequestId, - decision: codex_protocol::approvals::ElicitationAction, - content: Option, - meta: Option, - ) { - let action = match decision { - codex_protocol::approvals::ElicitationAction::Accept => ElicitationAction::Accept, - codex_protocol::approvals::ElicitationAction::Decline => ElicitationAction::Decline, - codex_protocol::approvals::ElicitationAction::Cancel => ElicitationAction::Cancel, - }; - let content = match action { - // Preserve the legacy fallback for clients that only send an action. - ElicitationAction::Accept => Some(content.unwrap_or_else(|| serde_json::json!({}))), - ElicitationAction::Decline | ElicitationAction::Cancel => None, - }; - let response = ElicitationResponse { - action, - content, - meta, - }; - let request_id = match request_id { - ProtocolRequestId::String(value) => { - rmcp::model::NumberOrString::String(std::sync::Arc::from(value)) - } - ProtocolRequestId::Integer(value) => rmcp::model::NumberOrString::Number(value), - }; - if let Err(err) = sess - .resolve_elicitation(server_name, request_id, response) - .await - { - warn!( - error = %err, - "failed to resolve elicitation request in session" - ); - } - } - - /// Propagate a user's exec approval decision to the session. - /// Also optionally applies an execpolicy amendment. - pub async fn exec_approval( - sess: &Arc, - approval_id: String, - turn_id: Option, - decision: ReviewDecision, - ) { - let event_turn_id = turn_id.unwrap_or_else(|| approval_id.clone()); - if let ReviewDecision::ApprovedExecpolicyAmendment { - proposed_execpolicy_amendment, - } = &decision - { - match sess - .persist_execpolicy_amendment(proposed_execpolicy_amendment) - .await - { - Ok(()) => { - sess.record_execpolicy_amendment_message( - &event_turn_id, - proposed_execpolicy_amendment, - ) - .await; - } - Err(err) => { - let message = format!("Failed to apply execpolicy amendment: {err}"); - tracing::warn!("{message}"); - let warning = EventMsg::Warning(WarningEvent { message }); - sess.send_event_raw(Event { - id: event_turn_id.clone(), - msg: warning, - }) - .await; - } - } - } - match decision { - ReviewDecision::Abort => { - sess.interrupt_task().await; - } - other => sess.notify_approval(&approval_id, other).await, - } - } - - pub async fn patch_approval(sess: &Arc, id: String, decision: ReviewDecision) { - match decision { - ReviewDecision::Abort => { - sess.interrupt_task().await; - } - other => sess.notify_approval(&id, other).await, - } - } - - pub async fn request_user_input_response( - sess: &Arc, - id: String, - response: RequestUserInputResponse, - ) { - sess.notify_user_input_response(&id, response).await; - } - - pub async fn request_permissions_response( - sess: &Arc, - id: String, - response: RequestPermissionsResponse, - ) { - sess.notify_request_permissions_response(&id, response) - .await; - } - - pub async fn dynamic_tool_response( - sess: &Arc, - id: String, - response: DynamicToolResponse, - ) { - sess.notify_dynamic_tool_response(&id, response).await; - } - - pub async fn add_to_history(sess: &Arc, config: &Arc, text: String) { - let id = sess.conversation_id; - let config = Arc::clone(config); - tokio::spawn(async move { - if let Err(e) = crate::message_history::append_entry(&text, &id, &config).await { - warn!("failed to append to message history: {e}"); - } - }); - } - - pub async fn get_history_entry_request( - sess: &Arc, - config: &Arc, - sub_id: String, - offset: usize, - log_id: u64, - ) { - let config = Arc::clone(config); - let sess_clone = Arc::clone(sess); - - tokio::spawn(async move { - // Run lookup in blocking thread because it does file IO + locking. - let entry_opt = tokio::task::spawn_blocking(move || { - crate::message_history::lookup(log_id, offset, &config) - }) - .await - .unwrap_or(None); - - let event = Event { - id: sub_id, - msg: EventMsg::GetHistoryEntryResponse( - codex_protocol::protocol::GetHistoryEntryResponseEvent { - offset, - log_id, - entry: entry_opt.map(|e| codex_protocol::message_history::HistoryEntry { - conversation_id: e.session_id, - ts: e.ts, - text: e.text, - }), - }, - ), - }; - - sess_clone.send_event_raw(event).await; - }); - } - - pub async fn refresh_mcp_servers(sess: &Arc, refresh_config: McpServerRefreshConfig) { - let mut guard = sess.pending_mcp_server_refresh_config.lock().await; - *guard = Some(refresh_config); - } - - pub async fn reload_user_config(sess: &Arc) { - sess.reload_user_config_layer().await; - } - - pub async fn list_mcp_tools(sess: &Session, config: &Arc, sub_id: String) { - let mcp_connection_manager = sess.services.mcp_connection_manager.read().await; - let auth = sess.services.auth_manager.auth().await; - let mcp_servers = sess - .services - .mcp_manager - .effective_servers(config, auth.as_ref()) - .await; - let snapshot = collect_mcp_snapshot_from_manager( - &mcp_connection_manager, - compute_auth_statuses(mcp_servers.iter(), config.mcp_oauth_credentials_store_mode) - .await, - ) - .await; - let event = Event { - id: sub_id, - msg: EventMsg::McpListToolsResponse(snapshot), - }; - sess.send_event_raw(event).await; - } - - pub async fn list_skills( - sess: &Session, - sub_id: String, - cwds: Vec, - force_reload: bool, - ) { - let default_cwd = { - let state = sess.state.lock().await; - state.session_configuration.cwd.to_path_buf() - }; - let cwds = if cwds.is_empty() { - vec![default_cwd] - } else { - cwds - }; - - let skills_manager = &sess.services.skills_manager; - let plugins_manager = &sess.services.plugins_manager; - let fs = sess - .services - .environment - .as_ref() - .map(|environment| environment.get_filesystem()); - let config = sess.get_config().await; - let codex_home = sess.codex_home().await; - let mut skills = Vec::new(); - let empty_cli_overrides: &[(String, toml::Value)] = &[]; - for cwd in cwds { - let cwd_abs = match AbsolutePathBuf::relative_to_current_dir(cwd.as_path()) { - Ok(path) => path, - Err(err) => { - let error_path = cwd.clone(); - skills.push(SkillsListEntry { - cwd, - skills: Vec::new(), - errors: vec![SkillErrorInfo { - path: error_path, - message: err.to_string(), - }], - }); - continue; - } - }; - let config_layer_stack = match load_config_layers_state( - &codex_home, - Some(cwd_abs.clone()), - empty_cli_overrides, - LoaderOverrides::default(), - CloudRequirementsLoader::default(), - ) - .await - { - Ok(config_layer_stack) => config_layer_stack, - Err(err) => { - let error_path = cwd.clone(); - skills.push(SkillsListEntry { - cwd, - skills: Vec::new(), - errors: vec![SkillErrorInfo { - path: error_path, - message: err.to_string(), - }], - }); - continue; - } - }; - let effective_skill_roots = plugins_manager - .effective_skill_roots_for_layer_stack( - &config_layer_stack, - config.features.enabled(Feature::Plugins), - ) - .await; - let skills_input = crate::SkillsLoadInput::new( - cwd_abs.clone(), - effective_skill_roots, - config_layer_stack, - config.bundled_skills_enabled(), - ); - let outcome = skills_manager - .skills_for_cwd(&skills_input, force_reload, fs.clone()) - .await; - let errors = super::errors_to_info(&outcome.errors); - let skills_metadata = super::skills_to_info(&outcome.skills, &outcome.disabled_paths); - skills.push(SkillsListEntry { - cwd, - skills: skills_metadata, - errors, - }); - } - - let event = Event { - id: sub_id, - msg: EventMsg::ListSkillsResponse(ListSkillsResponseEvent { skills }), - }; - sess.send_event_raw(event).await; - } - - pub async fn undo(sess: &Arc, sub_id: String) { - let turn_context = sess.new_default_turn_with_sub_id(sub_id).await; - sess.spawn_task(turn_context, Vec::new(), UndoTask::new()) - .await; - } - - pub async fn compact(sess: &Arc, sub_id: String) { - let turn_context = sess.new_default_turn_with_sub_id(sub_id).await; - - sess.spawn_task( - Arc::clone(&turn_context), - vec![UserInput::Text { - text: turn_context.compact_prompt().to_string(), - // Compaction prompt is synthesized; no UI element ranges to preserve. - text_elements: Vec::new(), - }], - CompactTask, - ) - .await; - } - - pub async fn drop_memories(sess: &Arc, config: &Arc, sub_id: String) { - let mut errors = Vec::new(); - - if let Some(state_db) = sess.services.state_db.as_deref() { - if let Err(err) = state_db.clear_memory_data().await { - errors.push(format!("failed clearing memory rows from state db: {err}")); - } - } else { - errors.push("state db unavailable; memory rows were not cleared".to_string()); - } - - if let Err(err) = crate::memories::clear_memory_roots_contents(&config.codex_home).await { - errors.push(format!( - "failed clearing memory directories under {}: {err}", - config.codex_home.display() - )); - } - - if errors.is_empty() { - let memory_root = crate::memories::memory_root(&config.codex_home); - sess.send_event_raw(Event { - id: sub_id, - msg: EventMsg::Warning(WarningEvent { - message: format!( - "Dropped memories at {} and cleared memory rows from state db.", - memory_root.display() - ), - }), - }) - .await; - return; - } - - sess.send_event_raw(Event { - id: sub_id, - msg: EventMsg::Error(ErrorEvent { - message: format!("Memory drop completed with errors: {}", errors.join("; ")), - codex_error_info: Some(CodexErrorInfo::Other), - }), - }) - .await; - } - - pub async fn update_memories(sess: &Arc, config: &Arc, sub_id: String) { - let session_source = { - let state = sess.state.lock().await; - state.session_configuration.session_source.clone() - }; - - crate::memories::start_memories_startup_task(sess, Arc::clone(config), &session_source); - - sess.send_event_raw(Event { - id: sub_id.clone(), - msg: EventMsg::Warning(WarningEvent { - message: "Memory update triggered.".to_string(), - }), - }) - .await; - } - - pub async fn thread_rollback(sess: &Arc, sub_id: String, num_turns: u32) { - if num_turns == 0 { - sess.send_event_raw(Event { - id: sub_id, - msg: EventMsg::Error(ErrorEvent { - message: "num_turns must be >= 1".to_string(), - codex_error_info: Some(CodexErrorInfo::ThreadRollbackFailed), - }), - }) - .await; - return; - } - - let has_active_turn = { sess.active_turn.lock().await.is_some() }; - if has_active_turn { - sess.send_event_raw(Event { - id: sub_id, - msg: EventMsg::Error(ErrorEvent { - message: "Cannot rollback while a turn is in progress.".to_string(), - codex_error_info: Some(CodexErrorInfo::ThreadRollbackFailed), - }), - }) - .await; - return; - } - - let turn_context = sess.new_default_turn_with_sub_id(sub_id).await; - let rollout_path = { - let recorder = { - let guard = sess.services.rollout.lock().await; - guard.clone() - }; - let Some(recorder) = recorder else { - sess.send_event_raw(Event { - id: turn_context.sub_id.clone(), - msg: EventMsg::Error(ErrorEvent { - message: "thread rollback requires a persisted rollout path".to_string(), - codex_error_info: Some(CodexErrorInfo::ThreadRollbackFailed), - }), - }) - .await; - return; - }; - recorder.rollout_path().to_path_buf() - }; - if let Some(recorder) = { - let guard = sess.services.rollout.lock().await; - guard.clone() - } && let Err(err) = recorder.flush().await - { - sess.send_event_raw(Event { - id: turn_context.sub_id.clone(), - msg: EventMsg::Error(ErrorEvent { - message: format!( - "failed to flush rollout `{}` for rollback replay: {err}", - rollout_path.display() - ), - codex_error_info: Some(CodexErrorInfo::ThreadRollbackFailed), - }), - }) - .await; - return; - } - - let initial_history = - match RolloutRecorder::get_rollout_history(rollout_path.as_path()).await { - Ok(history) => history, - Err(err) => { - sess.send_event_raw(Event { - id: turn_context.sub_id.clone(), - msg: EventMsg::Error(ErrorEvent { - message: format!( - "failed to load rollout `{}` for rollback replay: {err}", - rollout_path.display() - ), - codex_error_info: Some(CodexErrorInfo::ThreadRollbackFailed), - }), - }) - .await; - return; - } - }; - - let rollback_event = ThreadRolledBackEvent { num_turns }; - let rollback_msg = EventMsg::ThreadRolledBack(rollback_event.clone()); - let replay_items = initial_history - .get_rollout_items() - .into_iter() - .chain(std::iter::once(RolloutItem::EventMsg(rollback_msg.clone()))) - .collect::>(); - sess.apply_rollout_reconstruction(turn_context.as_ref(), replay_items.as_slice()) - .await; - sess.recompute_token_usage(turn_context.as_ref()).await; - - sess.persist_rollout_items(&[RolloutItem::EventMsg(rollback_msg.clone())]) - .await; - if let Err(err) = sess.flush_rollout().await { - sess.send_event( - turn_context.as_ref(), - EventMsg::Warning(WarningEvent { - message: format!( - "Rolled the thread back, but failed to save the rollback marker. Codex will continue retrying. Error: {err}" - ), - }), - ) - .await; - } - - sess.deliver_event_raw(Event { - id: turn_context.sub_id.clone(), - msg: rollback_msg, - }) - .await; - } - - async fn persist_thread_name_update( - sess: &Arc, - event: ThreadNameUpdatedEvent, - ) -> anyhow::Result { - let msg = EventMsg::ThreadNameUpdated(event); - let item = RolloutItem::EventMsg(msg.clone()); - let recorder = { - let guard = sess.services.rollout.lock().await; - guard.clone() - } - .ok_or_else(|| anyhow::anyhow!("Session persistence is disabled; cannot rename thread."))?; - recorder.persist().await?; - recorder.record_items(std::slice::from_ref(&item)).await?; - recorder.flush().await?; - Ok(msg) - } - - pub(super) async fn persist_thread_memory_mode_update( - sess: &Arc, - mode: ThreadMemoryMode, - ) -> anyhow::Result<()> { - let recorder = { - let guard = sess.services.rollout.lock().await; - guard.clone() - } - .ok_or_else(|| { - anyhow::anyhow!("Session persistence is disabled; cannot update thread memory mode.") - })?; - recorder.persist().await?; - recorder.flush().await?; - - let rollout_path = recorder.rollout_path().to_path_buf(); - let mut session_meta = read_session_meta_line(rollout_path.as_path()).await?; - if session_meta.meta.id != sess.conversation_id { - anyhow::bail!( - "rollout session metadata id mismatch: expected {}, found {}", - sess.conversation_id, - session_meta.meta.id - ); - } - session_meta.meta.memory_mode = Some( - match mode { - ThreadMemoryMode::Enabled => "enabled", - ThreadMemoryMode::Disabled => "disabled", - } - .to_string(), - ); - - let item = RolloutItem::SessionMeta(session_meta); - recorder.record_items(std::slice::from_ref(&item)).await?; - recorder.flush().await?; - Ok(()) - } - - /// Persists the thread name in the rollout and state database, updates in-memory state, and - /// emits a `ThreadNameUpdated` event on success. - pub async fn set_thread_name(sess: &Arc, sub_id: String, name: String) { - let Some(name) = crate::util::normalize_thread_name(&name) else { - let event = Event { - id: sub_id, - msg: EventMsg::Error(ErrorEvent { - message: "Thread name cannot be empty.".to_string(), - codex_error_info: Some(CodexErrorInfo::BadRequest), - }), - }; - sess.send_event_raw(event).await; - return; - }; - - let updated = ThreadNameUpdatedEvent { - thread_id: sess.conversation_id, - thread_name: Some(name.clone()), - }; - - let msg = match persist_thread_name_update(sess, updated).await { - Ok(msg) => msg, - Err(err) => { - warn!("Failed to persist thread name update to rollout: {err}"); - let event = Event { - id: sub_id, - msg: EventMsg::Error(ErrorEvent { - message: err.to_string(), - codex_error_info: Some(CodexErrorInfo::Other), - }), - }; - sess.send_event_raw(event).await; - return; - } - }; - - if let Some(state_db) = sess.services.state_db.as_deref() - && let Err(err) = state_db - .update_thread_title(sess.conversation_id, &name) - .await - { - warn!("Failed to update thread title in state db: {err}"); - } - - { - let mut state = sess.state.lock().await; - state.session_configuration.thread_name = Some(name.clone()); - } - - let codex_home = sess.codex_home().await; - if let Err(err) = - crate::rollout::append_thread_name(&codex_home, sess.conversation_id, &name).await - { - warn!("Failed to update legacy thread name index: {err}"); - } - - sess.deliver_event_raw(Event { id: sub_id, msg }).await; - } - - /// Persists thread-level memory mode metadata for the active session. - /// - /// This does not involve the model and only affects whether the thread is - /// eligible for future memory generation. - pub async fn set_thread_memory_mode( - sess: &Arc, - sub_id: String, - mode: ThreadMemoryMode, - ) { - if let Err(err) = persist_thread_memory_mode_update(sess, mode).await { - warn!("Failed to persist thread memory mode update to rollout: {err}"); - let event = Event { - id: sub_id, - msg: EventMsg::Error(ErrorEvent { - message: err.to_string(), - codex_error_info: Some(CodexErrorInfo::Other), - }), - }; - sess.send_event_raw(event).await; - } - } - - pub async fn shutdown(sess: &Arc, sub_id: String) -> bool { - sess.abort_all_tasks(TurnAbortReason::Interrupted).await; - let _ = sess.conversation.shutdown().await; - sess.services - .unified_exec_manager - .terminate_all_processes() - .await; - sess.guardian_review_session.shutdown().await; - info!("Shutting down Codex instance"); - let history = sess.clone_history().await; - let turn_count = history - .raw_items() - .iter() - .filter(|item| is_user_turn_boundary(item)) - .count(); - sess.services.session_telemetry.counter( - "codex.conversation.turn.count", - i64::try_from(turn_count).unwrap_or(0), - &[], - ); - - // Gracefully flush and shutdown rollout recorder on session end so tests - // that inspect the rollout file do not race with the background writer. - let recorder_opt = { - let mut guard = sess.services.rollout.lock().await; - guard.take() - }; - if let Some(rec) = recorder_opt - && let Err(e) = rec.shutdown().await - { - warn!("failed to shutdown rollout recorder: {e}"); - let event = Event { - id: sub_id.clone(), - msg: EventMsg::Error(ErrorEvent { - message: "Failed to shutdown rollout recorder".to_string(), - codex_error_info: Some(CodexErrorInfo::Other), - }), - }; - sess.send_event_raw(event).await; - } - - let event = Event { - id: sub_id, - msg: EventMsg::ShutdownComplete, - }; - sess.send_event_raw(event).await; - true - } - - pub async fn review( - sess: &Arc, - config: &Arc, - sub_id: String, - review_request: ReviewRequest, - ) { - let turn_context = sess.new_default_turn_with_sub_id(sub_id.clone()).await; - sess.maybe_emit_unknown_model_warning_for_turn(turn_context.as_ref()) - .await; - sess.refresh_mcp_servers_if_requested(&turn_context).await; - match resolve_review_request(review_request, &turn_context.cwd) { - Ok(resolved) => { - spawn_review_thread( - Arc::clone(sess), - Arc::clone(config), - turn_context.clone(), - sub_id, - resolved, - ) - .await; - } - Err(err) => { - let event = Event { - id: sub_id, - msg: EventMsg::Error(ErrorEvent { - message: err.to_string(), - codex_error_info: Some(CodexErrorInfo::Other), - }), - }; - sess.send_event(&turn_context, event.msg).await; - } - } - } -} - /// Spawn a review thread using the given prompt. async fn spawn_review_thread( sess: Arc, diff --git a/codex-rs/core/src/codex/handlers.rs b/codex-rs/core/src/codex/handlers.rs new file mode 100644 index 0000000000..88f3f38b65 --- /dev/null +++ b/codex-rs/core/src/codex/handlers.rs @@ -0,0 +1,1230 @@ +use crate::realtime_conversation::handle_audio as handle_realtime_conversation_audio; +use crate::realtime_conversation::handle_close as handle_realtime_conversation_close; +use crate::realtime_conversation::handle_start as handle_realtime_conversation_start; +use crate::realtime_conversation::handle_text as handle_realtime_conversation_text; +use async_channel::Receiver; +use codex_otel::set_parent_from_w3c_trace_context; +use codex_protocol::protocol::Submission; +use tracing::Instrument; +use tracing::debug_span; +use tracing::info_span; + +use crate::codex::Session; +use crate::codex::SessionSettingsUpdate; +use crate::codex::SteerInputError; + +use crate::codex::spawn_review_thread; +use crate::config::Config; +use crate::config_loader::CloudRequirementsLoader; +use crate::config_loader::LoaderOverrides; +use crate::config_loader::load_config_layers_state; +use crate::realtime_context::REALTIME_TURN_TOKEN_BUDGET; +use crate::realtime_context::truncate_realtime_text_to_token_budget; +use crate::realtime_conversation::REALTIME_USER_TEXT_PREFIX; +use crate::realtime_conversation::prefix_realtime_v2_text; +use codex_features::Feature; +use codex_utils_absolute_path::AbsolutePathBuf; + +use crate::review_prompts::resolve_review_request; +use crate::rollout::RolloutRecorder; +use crate::rollout::read_session_meta_line; +use crate::tasks::CompactTask; +use crate::tasks::UndoTask; +use crate::tasks::UserShellCommandMode; +use crate::tasks::UserShellCommandTask; +use crate::tasks::execute_user_shell_command; +use codex_mcp::collect_mcp_snapshot_from_manager; +use codex_mcp::compute_auth_statuses; +use codex_protocol::protocol::CodexErrorInfo; +use codex_protocol::protocol::ErrorEvent; +use codex_protocol::protocol::Event; +use codex_protocol::protocol::EventMsg; +use codex_protocol::protocol::InterAgentCommunication; +use codex_protocol::protocol::ListSkillsResponseEvent; +use codex_protocol::protocol::McpServerRefreshConfig; +use codex_protocol::protocol::Op; +use codex_protocol::protocol::RealtimeConversationListVoicesResponseEvent; +use codex_protocol::protocol::RealtimeVoicesList; +use codex_protocol::protocol::ReviewDecision; +use codex_protocol::protocol::ReviewRequest; +use codex_protocol::protocol::RolloutItem; +use codex_protocol::protocol::SkillErrorInfo; +use codex_protocol::protocol::SkillsListEntry; +use codex_protocol::protocol::ThreadMemoryMode; +use codex_protocol::protocol::ThreadNameUpdatedEvent; +use codex_protocol::protocol::ThreadRolledBackEvent; +use codex_protocol::protocol::TurnAbortReason; +use codex_protocol::protocol::WarningEvent; +use codex_protocol::request_permissions::RequestPermissionsResponse; +use codex_protocol::request_user_input::RequestUserInputResponse; + +use crate::context_manager::is_user_turn_boundary; +use codex_protocol::config_types::CollaborationMode; +use codex_protocol::config_types::ModeKind; +use codex_protocol::config_types::Settings; +use codex_protocol::dynamic_tools::DynamicToolResponse; +use codex_protocol::items::UserMessageItem; +use codex_protocol::mcp::RequestId as ProtocolRequestId; +use codex_protocol::user_input::UserInput; +use codex_rmcp_client::ElicitationAction; +use codex_rmcp_client::ElicitationResponse; +use serde_json::Value; +use std::path::PathBuf; +use std::sync::Arc; +use tracing::debug; +use tracing::info; +use tracing::warn; + +pub async fn interrupt(sess: &Arc) { + sess.interrupt_task().await; +} + +pub async fn clean_background_terminals(sess: &Arc) { + sess.close_unified_exec_processes().await; +} + +pub async fn realtime_conversation_list_voices(sess: &Session, sub_id: String) { + sess.send_event_raw(Event { + id: sub_id, + msg: EventMsg::RealtimeConversationListVoicesResponse( + RealtimeConversationListVoicesResponseEvent { + voices: RealtimeVoicesList::builtin(), + }, + ), + }) + .await; +} + +pub async fn override_turn_context(sess: &Session, sub_id: String, updates: SessionSettingsUpdate) { + if let Err(err) = sess.update_settings(updates).await { + sess.send_event_raw(Event { + id: sub_id, + msg: EventMsg::Error(ErrorEvent { + message: err.to_string(), + codex_error_info: Some(CodexErrorInfo::BadRequest), + }), + }) + .await; + } +} + +pub async fn user_input_or_turn(sess: &Arc, sub_id: String, op: Op) { + user_input_or_turn_inner( + sess, + sub_id, + op, + /*mirror_user_text_to_realtime*/ Some(()), + ) + .await; +} + +pub(super) async fn user_input_or_turn_inner( + sess: &Arc, + sub_id: String, + op: Op, + mirror_user_text_to_realtime: Option<()>, +) { + let (items, updates, responsesapi_client_metadata) = match op { + Op::UserTurn { + cwd, + approval_policy, + approvals_reviewer, + sandbox_policy, + model, + effort, + summary, + service_tier, + final_output_json_schema, + items, + collaboration_mode, + personality, + } => { + let collaboration_mode = collaboration_mode.or_else(|| { + Some(CollaborationMode { + mode: ModeKind::Default, + settings: Settings { + model: model.clone(), + reasoning_effort: effort, + developer_instructions: None, + }, + }) + }); + ( + items, + SessionSettingsUpdate { + cwd: Some(cwd), + approval_policy: Some(approval_policy), + approvals_reviewer, + sandbox_policy: Some(sandbox_policy), + windows_sandbox_level: None, + collaboration_mode, + reasoning_summary: summary, + service_tier, + final_output_json_schema: Some(final_output_json_schema), + personality, + app_server_client_name: None, + app_server_client_version: None, + }, + None, + ) + } + Op::UserInput { + items, + final_output_json_schema, + responsesapi_client_metadata, + } => ( + items, + SessionSettingsUpdate { + final_output_json_schema: Some(final_output_json_schema), + ..Default::default() + }, + responsesapi_client_metadata, + ), + _ => unreachable!(), + }; + + let Ok(current_context) = sess.new_turn_with_sub_id(sub_id.clone(), updates).await else { + // new_turn_with_sub_id already emits the error event. + return; + }; + sess.maybe_emit_unknown_model_warning_for_turn(current_context.as_ref()) + .await; + let accepted_items = match sess + .steer_input( + items.clone(), + /*expected_turn_id*/ None, + responsesapi_client_metadata.clone(), + ) + .await + { + Ok(_) => { + current_context.session_telemetry.user_prompt(&items); + Some(items) + } + Err(SteerInputError::NoActiveTurn(items)) => { + if let Some(responsesapi_client_metadata) = responsesapi_client_metadata { + current_context + .turn_metadata_state + .set_responsesapi_client_metadata(responsesapi_client_metadata); + } + current_context.session_telemetry.user_prompt(&items); + sess.refresh_mcp_servers_if_requested(¤t_context) + .await; + let accepted_items = items.clone(); + sess.spawn_task( + Arc::clone(¤t_context), + items, + crate::tasks::RegularTask::new(), + ) + .await; + Some(accepted_items) + } + Err(err) => { + sess.send_event_raw(Event { + id: sub_id, + msg: EventMsg::Error(err.to_error_event()), + }) + .await; + None + } + }; + if let (Some(items), Some(())) = (accepted_items, mirror_user_text_to_realtime) { + self::mirror_user_text_to_realtime(sess, &items).await; + } +} + +async fn mirror_user_text_to_realtime(sess: &Arc, items: &[UserInput]) { + let text = UserMessageItem::new(items).message(); + if text.is_empty() { + return; + } + let text = if sess.conversation.is_running_v2().await { + prefix_realtime_v2_text(text, REALTIME_USER_TEXT_PREFIX) + } else { + text + }; + let text = truncate_realtime_text_to_token_budget(&text, REALTIME_TURN_TOKEN_BUDGET); + if text.is_empty() { + return; + } + if sess.conversation.running_state().await.is_none() { + return; + } + if let Err(err) = sess.conversation.text_in(text).await { + debug!("failed to mirror user text to realtime conversation: {err}"); + } +} + +/// Records an inter-agent assistant envelope, then lets the shared pending-work scheduler +/// decide whether an idle session should start a regular turn. +pub async fn inter_agent_communication( + sess: &Arc, + sub_id: String, + communication: InterAgentCommunication, +) { + let trigger_turn = communication.trigger_turn; + sess.enqueue_mailbox_communication(communication); + if trigger_turn { + sess.maybe_start_turn_for_pending_work_with_sub_id(sub_id) + .await; + } +} + +pub async fn run_user_shell_command(sess: &Arc, sub_id: String, command: String) { + if let Some((turn_context, cancellation_token)) = + sess.active_turn_context_and_cancellation_token().await + { + let session = Arc::clone(sess); + tokio::spawn(async move { + execute_user_shell_command( + session, + turn_context, + command, + cancellation_token, + UserShellCommandMode::ActiveTurnAuxiliary, + ) + .await; + }); + return; + } + + let turn_context = sess.new_default_turn_with_sub_id(sub_id).await; + sess.spawn_task( + Arc::clone(&turn_context), + Vec::new(), + UserShellCommandTask::new(command), + ) + .await; +} + +pub async fn resolve_elicitation( + sess: &Arc, + server_name: String, + request_id: ProtocolRequestId, + decision: codex_protocol::approvals::ElicitationAction, + content: Option, + meta: Option, +) { + let action = match decision { + codex_protocol::approvals::ElicitationAction::Accept => ElicitationAction::Accept, + codex_protocol::approvals::ElicitationAction::Decline => ElicitationAction::Decline, + codex_protocol::approvals::ElicitationAction::Cancel => ElicitationAction::Cancel, + }; + let content = match action { + // Preserve the legacy fallback for clients that only send an action. + ElicitationAction::Accept => Some(content.unwrap_or_else(|| serde_json::json!({}))), + ElicitationAction::Decline | ElicitationAction::Cancel => None, + }; + let response = ElicitationResponse { + action, + content, + meta, + }; + let request_id = match request_id { + ProtocolRequestId::String(value) => { + rmcp::model::NumberOrString::String(std::sync::Arc::from(value)) + } + ProtocolRequestId::Integer(value) => rmcp::model::NumberOrString::Number(value), + }; + if let Err(err) = sess + .resolve_elicitation(server_name, request_id, response) + .await + { + warn!( + error = %err, + "failed to resolve elicitation request in session" + ); + } +} + +/// Propagate a user's exec approval decision to the session. +/// Also optionally applies an execpolicy amendment. +pub async fn exec_approval( + sess: &Arc, + approval_id: String, + turn_id: Option, + decision: ReviewDecision, +) { + let event_turn_id = turn_id.unwrap_or_else(|| approval_id.clone()); + if let ReviewDecision::ApprovedExecpolicyAmendment { + proposed_execpolicy_amendment, + } = &decision + { + match sess + .persist_execpolicy_amendment(proposed_execpolicy_amendment) + .await + { + Ok(()) => { + sess.record_execpolicy_amendment_message( + &event_turn_id, + proposed_execpolicy_amendment, + ) + .await; + } + Err(err) => { + let message = format!("Failed to apply execpolicy amendment: {err}"); + tracing::warn!("{message}"); + let warning = EventMsg::Warning(WarningEvent { message }); + sess.send_event_raw(Event { + id: event_turn_id.clone(), + msg: warning, + }) + .await; + } + } + } + match decision { + ReviewDecision::Abort => { + sess.interrupt_task().await; + } + other => sess.notify_approval(&approval_id, other).await, + } +} + +pub async fn patch_approval(sess: &Arc, id: String, decision: ReviewDecision) { + match decision { + ReviewDecision::Abort => { + sess.interrupt_task().await; + } + other => sess.notify_approval(&id, other).await, + } +} + +pub async fn request_user_input_response( + sess: &Arc, + id: String, + response: RequestUserInputResponse, +) { + sess.notify_user_input_response(&id, response).await; +} + +pub async fn request_permissions_response( + sess: &Arc, + id: String, + response: RequestPermissionsResponse, +) { + sess.notify_request_permissions_response(&id, response) + .await; +} + +pub async fn dynamic_tool_response(sess: &Arc, id: String, response: DynamicToolResponse) { + sess.notify_dynamic_tool_response(&id, response).await; +} + +pub async fn add_to_history(sess: &Arc, config: &Arc, text: String) { + let id = sess.conversation_id; + let config = Arc::clone(config); + tokio::spawn(async move { + if let Err(e) = crate::message_history::append_entry(&text, &id, &config).await { + warn!("failed to append to message history: {e}"); + } + }); +} + +pub async fn get_history_entry_request( + sess: &Arc, + config: &Arc, + sub_id: String, + offset: usize, + log_id: u64, +) { + let config = Arc::clone(config); + let sess_clone = Arc::clone(sess); + + tokio::spawn(async move { + // Run lookup in blocking thread because it does file IO + locking. + let entry_opt = tokio::task::spawn_blocking(move || { + crate::message_history::lookup(log_id, offset, &config) + }) + .await + .unwrap_or(None); + + let event = Event { + id: sub_id, + msg: EventMsg::GetHistoryEntryResponse( + codex_protocol::protocol::GetHistoryEntryResponseEvent { + offset, + log_id, + entry: entry_opt.map(|e| codex_protocol::message_history::HistoryEntry { + conversation_id: e.session_id, + ts: e.ts, + text: e.text, + }), + }, + ), + }; + + sess_clone.send_event_raw(event).await; + }); +} + +pub async fn refresh_mcp_servers(sess: &Arc, refresh_config: McpServerRefreshConfig) { + let mut guard = sess.pending_mcp_server_refresh_config.lock().await; + *guard = Some(refresh_config); +} + +pub async fn reload_user_config(sess: &Arc) { + sess.reload_user_config_layer().await; +} + +pub async fn list_mcp_tools(sess: &Session, config: &Arc, sub_id: String) { + let mcp_connection_manager = sess.services.mcp_connection_manager.read().await; + let auth = sess.services.auth_manager.auth().await; + let mcp_servers = sess + .services + .mcp_manager + .effective_servers(config, auth.as_ref()) + .await; + let snapshot = collect_mcp_snapshot_from_manager( + &mcp_connection_manager, + compute_auth_statuses(mcp_servers.iter(), config.mcp_oauth_credentials_store_mode).await, + ) + .await; + let event = Event { + id: sub_id, + msg: EventMsg::McpListToolsResponse(snapshot), + }; + sess.send_event_raw(event).await; +} + +pub async fn list_skills(sess: &Session, sub_id: String, cwds: Vec, force_reload: bool) { + let default_cwd = { + let state = sess.state.lock().await; + state.session_configuration.cwd.to_path_buf() + }; + let cwds = if cwds.is_empty() { + vec![default_cwd] + } else { + cwds + }; + + let skills_manager = &sess.services.skills_manager; + let plugins_manager = &sess.services.plugins_manager; + let fs = sess + .services + .environment + .as_ref() + .map(|environment| environment.get_filesystem()); + let config = sess.get_config().await; + let codex_home = sess.codex_home().await; + let mut skills = Vec::new(); + let empty_cli_overrides: &[(String, toml::Value)] = &[]; + for cwd in cwds { + let cwd_abs = match AbsolutePathBuf::relative_to_current_dir(cwd.as_path()) { + Ok(path) => path, + Err(err) => { + let error_path = cwd.clone(); + skills.push(SkillsListEntry { + cwd, + skills: Vec::new(), + errors: vec![SkillErrorInfo { + path: error_path, + message: err.to_string(), + }], + }); + continue; + } + }; + let config_layer_stack = match load_config_layers_state( + &codex_home, + Some(cwd_abs.clone()), + empty_cli_overrides, + LoaderOverrides::default(), + CloudRequirementsLoader::default(), + ) + .await + { + Ok(config_layer_stack) => config_layer_stack, + Err(err) => { + let error_path = cwd.clone(); + skills.push(SkillsListEntry { + cwd, + skills: Vec::new(), + errors: vec![SkillErrorInfo { + path: error_path, + message: err.to_string(), + }], + }); + continue; + } + }; + let effective_skill_roots = plugins_manager + .effective_skill_roots_for_layer_stack( + &config_layer_stack, + config.features.enabled(Feature::Plugins), + ) + .await; + let skills_input = crate::SkillsLoadInput::new( + cwd_abs.clone(), + effective_skill_roots, + config_layer_stack, + config.bundled_skills_enabled(), + ); + let outcome = skills_manager + .skills_for_cwd(&skills_input, force_reload, fs.clone()) + .await; + let errors = super::errors_to_info(&outcome.errors); + let skills_metadata = super::skills_to_info(&outcome.skills, &outcome.disabled_paths); + skills.push(SkillsListEntry { + cwd, + skills: skills_metadata, + errors, + }); + } + + let event = Event { + id: sub_id, + msg: EventMsg::ListSkillsResponse(ListSkillsResponseEvent { skills }), + }; + sess.send_event_raw(event).await; +} + +pub async fn undo(sess: &Arc, sub_id: String) { + let turn_context = sess.new_default_turn_with_sub_id(sub_id).await; + sess.spawn_task(turn_context, Vec::new(), UndoTask::new()) + .await; +} + +pub async fn compact(sess: &Arc, sub_id: String) { + let turn_context = sess.new_default_turn_with_sub_id(sub_id).await; + + sess.spawn_task( + Arc::clone(&turn_context), + vec![UserInput::Text { + text: turn_context.compact_prompt().to_string(), + // Compaction prompt is synthesized; no UI element ranges to preserve. + text_elements: Vec::new(), + }], + CompactTask, + ) + .await; +} + +pub async fn drop_memories(sess: &Arc, config: &Arc, sub_id: String) { + let mut errors = Vec::new(); + + if let Some(state_db) = sess.services.state_db.as_deref() { + if let Err(err) = state_db.clear_memory_data().await { + errors.push(format!("failed clearing memory rows from state db: {err}")); + } + } else { + errors.push("state db unavailable; memory rows were not cleared".to_string()); + } + + if let Err(err) = crate::memories::clear_memory_roots_contents(&config.codex_home).await { + errors.push(format!( + "failed clearing memory directories under {}: {err}", + config.codex_home.display() + )); + } + + if errors.is_empty() { + let memory_root = crate::memories::memory_root(&config.codex_home); + sess.send_event_raw(Event { + id: sub_id, + msg: EventMsg::Warning(WarningEvent { + message: format!( + "Dropped memories at {} and cleared memory rows from state db.", + memory_root.display() + ), + }), + }) + .await; + return; + } + + sess.send_event_raw(Event { + id: sub_id, + msg: EventMsg::Error(ErrorEvent { + message: format!("Memory drop completed with errors: {}", errors.join("; ")), + codex_error_info: Some(CodexErrorInfo::Other), + }), + }) + .await; +} + +pub async fn update_memories(sess: &Arc, config: &Arc, sub_id: String) { + let session_source = { + let state = sess.state.lock().await; + state.session_configuration.session_source.clone() + }; + + crate::memories::start_memories_startup_task(sess, Arc::clone(config), &session_source); + + sess.send_event_raw(Event { + id: sub_id.clone(), + msg: EventMsg::Warning(WarningEvent { + message: "Memory update triggered.".to_string(), + }), + }) + .await; +} + +pub async fn thread_rollback(sess: &Arc, sub_id: String, num_turns: u32) { + if num_turns == 0 { + sess.send_event_raw(Event { + id: sub_id, + msg: EventMsg::Error(ErrorEvent { + message: "num_turns must be >= 1".to_string(), + codex_error_info: Some(CodexErrorInfo::ThreadRollbackFailed), + }), + }) + .await; + return; + } + + let has_active_turn = { sess.active_turn.lock().await.is_some() }; + if has_active_turn { + sess.send_event_raw(Event { + id: sub_id, + msg: EventMsg::Error(ErrorEvent { + message: "Cannot rollback while a turn is in progress.".to_string(), + codex_error_info: Some(CodexErrorInfo::ThreadRollbackFailed), + }), + }) + .await; + return; + } + + let turn_context = sess.new_default_turn_with_sub_id(sub_id).await; + let rollout_path = { + let recorder = { + let guard = sess.services.rollout.lock().await; + guard.clone() + }; + let Some(recorder) = recorder else { + sess.send_event_raw(Event { + id: turn_context.sub_id.clone(), + msg: EventMsg::Error(ErrorEvent { + message: "thread rollback requires a persisted rollout path".to_string(), + codex_error_info: Some(CodexErrorInfo::ThreadRollbackFailed), + }), + }) + .await; + return; + }; + recorder.rollout_path().to_path_buf() + }; + if let Some(recorder) = { + let guard = sess.services.rollout.lock().await; + guard.clone() + } && let Err(err) = recorder.flush().await + { + sess.send_event_raw(Event { + id: turn_context.sub_id.clone(), + msg: EventMsg::Error(ErrorEvent { + message: format!( + "failed to flush rollout `{}` for rollback replay: {err}", + rollout_path.display() + ), + codex_error_info: Some(CodexErrorInfo::ThreadRollbackFailed), + }), + }) + .await; + return; + } + + let initial_history = match RolloutRecorder::get_rollout_history(rollout_path.as_path()).await { + Ok(history) => history, + Err(err) => { + sess.send_event_raw(Event { + id: turn_context.sub_id.clone(), + msg: EventMsg::Error(ErrorEvent { + message: format!( + "failed to load rollout `{}` for rollback replay: {err}", + rollout_path.display() + ), + codex_error_info: Some(CodexErrorInfo::ThreadRollbackFailed), + }), + }) + .await; + return; + } + }; + + let rollback_event = ThreadRolledBackEvent { num_turns }; + let rollback_msg = EventMsg::ThreadRolledBack(rollback_event.clone()); + let replay_items = initial_history + .get_rollout_items() + .into_iter() + .chain(std::iter::once(RolloutItem::EventMsg(rollback_msg.clone()))) + .collect::>(); + sess.apply_rollout_reconstruction(turn_context.as_ref(), replay_items.as_slice()) + .await; + sess.recompute_token_usage(turn_context.as_ref()).await; + + sess.persist_rollout_items(&[RolloutItem::EventMsg(rollback_msg.clone())]) + .await; + if let Err(err) = sess.flush_rollout().await { + sess.send_event( + turn_context.as_ref(), + EventMsg::Warning(WarningEvent { + message: format!( + "Rolled the thread back, but failed to save the rollback marker. Codex will continue retrying. Error: {err}" + ), + }), + ) + .await; + } + + sess.deliver_event_raw(Event { + id: turn_context.sub_id.clone(), + msg: rollback_msg, + }) + .await; +} + +async fn persist_thread_name_update( + sess: &Arc, + event: ThreadNameUpdatedEvent, +) -> anyhow::Result { + let msg = EventMsg::ThreadNameUpdated(event); + let item = RolloutItem::EventMsg(msg.clone()); + let recorder = { + let guard = sess.services.rollout.lock().await; + guard.clone() + } + .ok_or_else(|| anyhow::anyhow!("Session persistence is disabled; cannot rename thread."))?; + recorder.persist().await?; + recorder.record_items(std::slice::from_ref(&item)).await?; + recorder.flush().await?; + Ok(msg) +} + +pub(super) async fn persist_thread_memory_mode_update( + sess: &Arc, + mode: ThreadMemoryMode, +) -> anyhow::Result<()> { + let recorder = { + let guard = sess.services.rollout.lock().await; + guard.clone() + } + .ok_or_else(|| { + anyhow::anyhow!("Session persistence is disabled; cannot update thread memory mode.") + })?; + recorder.persist().await?; + recorder.flush().await?; + + let rollout_path = recorder.rollout_path().to_path_buf(); + let mut session_meta = read_session_meta_line(rollout_path.as_path()).await?; + if session_meta.meta.id != sess.conversation_id { + anyhow::bail!( + "rollout session metadata id mismatch: expected {}, found {}", + sess.conversation_id, + session_meta.meta.id + ); + } + session_meta.meta.memory_mode = Some( + match mode { + ThreadMemoryMode::Enabled => "enabled", + ThreadMemoryMode::Disabled => "disabled", + } + .to_string(), + ); + + let item = RolloutItem::SessionMeta(session_meta); + recorder.record_items(std::slice::from_ref(&item)).await?; + recorder.flush().await?; + Ok(()) +} + +/// Persists the thread name in the rollout and state database, updates in-memory state, and +/// emits a `ThreadNameUpdated` event on success. +pub async fn set_thread_name(sess: &Arc, sub_id: String, name: String) { + let Some(name) = crate::util::normalize_thread_name(&name) else { + let event = Event { + id: sub_id, + msg: EventMsg::Error(ErrorEvent { + message: "Thread name cannot be empty.".to_string(), + codex_error_info: Some(CodexErrorInfo::BadRequest), + }), + }; + sess.send_event_raw(event).await; + return; + }; + + let updated = ThreadNameUpdatedEvent { + thread_id: sess.conversation_id, + thread_name: Some(name.clone()), + }; + + let msg = match persist_thread_name_update(sess, updated).await { + Ok(msg) => msg, + Err(err) => { + warn!("Failed to persist thread name update to rollout: {err}"); + let event = Event { + id: sub_id, + msg: EventMsg::Error(ErrorEvent { + message: err.to_string(), + codex_error_info: Some(CodexErrorInfo::Other), + }), + }; + sess.send_event_raw(event).await; + return; + } + }; + + if let Some(state_db) = sess.services.state_db.as_deref() + && let Err(err) = state_db + .update_thread_title(sess.conversation_id, &name) + .await + { + warn!("Failed to update thread title in state db: {err}"); + } + + { + let mut state = sess.state.lock().await; + state.session_configuration.thread_name = Some(name.clone()); + } + + let codex_home = sess.codex_home().await; + if let Err(err) = + crate::rollout::append_thread_name(&codex_home, sess.conversation_id, &name).await + { + warn!("Failed to update legacy thread name index: {err}"); + } + + sess.deliver_event_raw(Event { id: sub_id, msg }).await; +} + +/// Persists thread-level memory mode metadata for the active session. +/// +/// This does not involve the model and only affects whether the thread is +/// eligible for future memory generation. +pub async fn set_thread_memory_mode(sess: &Arc, sub_id: String, mode: ThreadMemoryMode) { + if let Err(err) = persist_thread_memory_mode_update(sess, mode).await { + warn!("Failed to persist thread memory mode update to rollout: {err}"); + let event = Event { + id: sub_id, + msg: EventMsg::Error(ErrorEvent { + message: err.to_string(), + codex_error_info: Some(CodexErrorInfo::Other), + }), + }; + sess.send_event_raw(event).await; + } +} + +pub async fn shutdown(sess: &Arc, sub_id: String) -> bool { + sess.abort_all_tasks(TurnAbortReason::Interrupted).await; + let _ = sess.conversation.shutdown().await; + sess.services + .unified_exec_manager + .terminate_all_processes() + .await; + sess.guardian_review_session.shutdown().await; + info!("Shutting down Codex instance"); + let history = sess.clone_history().await; + let turn_count = history + .raw_items() + .iter() + .filter(|item| is_user_turn_boundary(item)) + .count(); + sess.services.session_telemetry.counter( + "codex.conversation.turn.count", + i64::try_from(turn_count).unwrap_or(0), + &[], + ); + + // Gracefully flush and shutdown rollout recorder on session end so tests + // that inspect the rollout file do not race with the background writer. + let recorder_opt = { + let mut guard = sess.services.rollout.lock().await; + guard.take() + }; + if let Some(rec) = recorder_opt + && let Err(e) = rec.shutdown().await + { + warn!("failed to shutdown rollout recorder: {e}"); + let event = Event { + id: sub_id.clone(), + msg: EventMsg::Error(ErrorEvent { + message: "Failed to shutdown rollout recorder".to_string(), + codex_error_info: Some(CodexErrorInfo::Other), + }), + }; + sess.send_event_raw(event).await; + } + + let event = Event { + id: sub_id, + msg: EventMsg::ShutdownComplete, + }; + sess.send_event_raw(event).await; + true +} + +pub async fn review( + sess: &Arc, + config: &Arc, + sub_id: String, + review_request: ReviewRequest, +) { + let turn_context = sess.new_default_turn_with_sub_id(sub_id.clone()).await; + sess.maybe_emit_unknown_model_warning_for_turn(turn_context.as_ref()) + .await; + sess.refresh_mcp_servers_if_requested(&turn_context).await; + match resolve_review_request(review_request, &turn_context.cwd) { + Ok(resolved) => { + spawn_review_thread( + Arc::clone(sess), + Arc::clone(config), + turn_context.clone(), + sub_id, + resolved, + ) + .await; + } + Err(err) => { + let event = Event { + id: sub_id, + msg: EventMsg::Error(ErrorEvent { + message: err.to_string(), + codex_error_info: Some(CodexErrorInfo::Other), + }), + }; + sess.send_event(&turn_context, event.msg).await; + } + } +} + +pub(super) async fn submission_loop( + sess: Arc, + config: Arc, + rx_sub: Receiver, +) { + // To break out of this loop, send Op::Shutdown. + while let Ok(sub) = rx_sub.recv().await { + debug!(?sub, "Submission"); + let dispatch_span = submission_dispatch_span(&sub); + let should_exit = async { + match sub.op.clone() { + Op::Interrupt => { + interrupt(&sess).await; + false + } + Op::CleanBackgroundTerminals => { + clean_background_terminals(&sess).await; + false + } + Op::RealtimeConversationStart(params) => { + if let Err(err) = + handle_realtime_conversation_start(&sess, sub.id.clone(), params).await + { + sess.send_event_raw(Event { + id: sub.id.clone(), + msg: EventMsg::Error(ErrorEvent { + message: err.to_string(), + codex_error_info: Some(CodexErrorInfo::Other), + }), + }) + .await; + } + false + } + Op::RealtimeConversationAudio(params) => { + handle_realtime_conversation_audio(&sess, sub.id.clone(), params).await; + false + } + Op::RealtimeConversationText(params) => { + handle_realtime_conversation_text(&sess, sub.id.clone(), params).await; + false + } + Op::RealtimeConversationClose => { + handle_realtime_conversation_close(&sess, sub.id.clone()).await; + false + } + Op::RealtimeConversationListVoices => { + realtime_conversation_list_voices(&sess, sub.id.clone()).await; + false + } + Op::OverrideTurnContext { + cwd, + approval_policy, + approvals_reviewer, + sandbox_policy, + windows_sandbox_level, + model, + effort, + summary, + service_tier, + collaboration_mode, + personality, + } => { + let collaboration_mode = if let Some(collab_mode) = collaboration_mode { + collab_mode + } else { + let state = sess.state.lock().await; + state.session_configuration.collaboration_mode.with_updates( + model.clone(), + effort, + /*developer_instructions*/ None, + ) + }; + override_turn_context( + &sess, + sub.id.clone(), + SessionSettingsUpdate { + cwd, + approval_policy, + approvals_reviewer, + sandbox_policy, + windows_sandbox_level, + collaboration_mode: Some(collaboration_mode), + reasoning_summary: summary, + service_tier, + personality, + ..Default::default() + }, + ) + .await; + false + } + Op::UserInput { .. } | Op::UserTurn { .. } => { + user_input_or_turn(&sess, sub.id.clone(), sub.op).await; + false + } + Op::InterAgentCommunication { communication } => { + inter_agent_communication(&sess, sub.id.clone(), communication).await; + false + } + Op::ExecApproval { + id: approval_id, + turn_id, + decision, + } => { + exec_approval(&sess, approval_id, turn_id, decision).await; + false + } + Op::PatchApproval { id, decision } => { + patch_approval(&sess, id, decision).await; + false + } + Op::UserInputAnswer { id, response } => { + request_user_input_response(&sess, id, response).await; + false + } + Op::RequestPermissionsResponse { id, response } => { + request_permissions_response(&sess, id, response).await; + false + } + Op::DynamicToolResponse { id, response } => { + dynamic_tool_response(&sess, id, response).await; + false + } + Op::AddToHistory { text } => { + add_to_history(&sess, &config, text).await; + false + } + Op::GetHistoryEntryRequest { offset, log_id } => { + get_history_entry_request(&sess, &config, sub.id.clone(), offset, log_id).await; + false + } + Op::ListMcpTools => { + list_mcp_tools(&sess, &config, sub.id.clone()).await; + false + } + Op::RefreshMcpServers { config } => { + refresh_mcp_servers(&sess, config).await; + false + } + Op::ReloadUserConfig => { + reload_user_config(&sess).await; + false + } + Op::ListSkills { cwds, force_reload } => { + list_skills(&sess, sub.id.clone(), cwds, force_reload).await; + false + } + Op::Undo => { + undo(&sess, sub.id.clone()).await; + false + } + Op::Compact => { + compact(&sess, sub.id.clone()).await; + false + } + Op::DropMemories => { + drop_memories(&sess, &config, sub.id.clone()).await; + false + } + Op::UpdateMemories => { + update_memories(&sess, &config, sub.id.clone()).await; + false + } + Op::ThreadRollback { num_turns } => { + thread_rollback(&sess, sub.id.clone(), num_turns).await; + false + } + Op::SetThreadName { name } => { + set_thread_name(&sess, sub.id.clone(), name).await; + false + } + Op::SetThreadMemoryMode { mode } => { + set_thread_memory_mode(&sess, sub.id.clone(), mode).await; + false + } + Op::RunUserShellCommand { command } => { + run_user_shell_command(&sess, sub.id.clone(), command).await; + false + } + Op::ResolveElicitation { + server_name, + request_id, + decision, + content, + meta, + } => { + resolve_elicitation(&sess, server_name, request_id, decision, content, meta) + .await; + false + } + Op::Shutdown => shutdown(&sess, sub.id.clone()).await, + Op::Review { review_request } => { + review(&sess, &config, sub.id.clone(), review_request).await; + false + } + _ => false, // Ignore unknown ops; enum is non_exhaustive to allow extensions. + } + } + .instrument(dispatch_span) + .await; + if should_exit { + break; + } + } + // Also drain cached guardian state if the submission loop exits because + // the channel closed without receiving an explicit shutdown op. + sess.guardian_review_session.shutdown().await; + debug!("Agent loop exited"); +} + +pub(super) fn submission_dispatch_span(sub: &Submission) -> tracing::Span { + let op_name = sub.op.kind(); + let span_name = format!("op.dispatch.{op_name}"); + let dispatch_span = match &sub.op { + Op::RealtimeConversationAudio(_) => { + debug_span!( + "submission_dispatch", + otel.name = span_name.as_str(), + submission.id = sub.id.as_str(), + codex.op = op_name + ) + } + _ => info_span!( + "submission_dispatch", + otel.name = span_name.as_str(), + submission.id = sub.id.as_str(), + codex.op = op_name + ), + }; + if let Some(trace) = sub.trace.as_ref() + && !set_parent_from_w3c_trace_context(&dispatch_span, trace) + { + warn!( + submission.id = sub.id.as_str(), + "ignoring invalid submission trace carrier" + ); + } + dispatch_span +} From 55c3de75cba1a65088ff02b91527b94a1b60a69b Mon Sep 17 00:00:00 2001 From: Adrian <145513011+adrian-openai@users.noreply.github.com> Date: Thu, 16 Apr 2026 14:30:02 -0700 Subject: [PATCH 41/59] Register agent tasks behind use_agent_identity (#17387) ## Summary Stack PR3 for feature-gated agent identity support. This PR adds per-thread agent task registration behind `features.use_agent_identity`. Tasks are minted on the first real user turn and cached in thread runtime state for later turns. ## Stack - PR1: https://github.com/openai/codex/pull/17385 - add `features.use_agent_identity` - PR2: https://github.com/openai/codex/pull/17386 - register agent identities when enabled - PR3: https://github.com/openai/codex/pull/17387 - this PR, original task registration slice - PR3.1: https://github.com/openai/codex/pull/17978 - persist and prewarm registered tasks per thread - PR4: https://github.com/openai/codex/pull/17980 - use `AgentAssertion` downstream when enabled ## Validation Covered as part of the local stack validation pass: - `just fmt` - `cargo test -p codex-core --lib agent_identity` - `cargo test -p codex-core --lib agent_assertion` - `cargo test -p codex-core --lib websocket_agent_task` - `cargo test -p codex-api api_bridge` - `cargo build -p codex-cli --bin codex` ## Notes The full local app-server E2E path is still being debugged after PR creation. The current branch stack is directionally ready for review while that follow-up continues. --- MODULE.bazel.lock | 3 + codex-rs/Cargo.lock | 43 ++ codex-rs/Cargo.toml | 1 + codex-rs/app-server/tests/suite/auth.rs | 8 +- .../app-server/tests/suite/v2/plugin_list.rs | 3 +- .../tests/suite/v2/realtime_conversation.rs | 5 +- codex-rs/core/Cargo.toml | 2 + codex-rs/core/src/agent_identity.rs | 81 ++- .../src/agent_identity/task_registration.rs | 470 ++++++++++++++++++ codex-rs/core/src/codex.rs | 102 +++- codex-rs/core/src/codex_tests.rs | 13 +- codex-rs/core/src/state/session.rs | 15 + codex-rs/core/src/state/session_tests.rs | 38 ++ .../core/tests/suite/realtime_conversation.rs | 63 ++- 14 files changed, 804 insertions(+), 43 deletions(-) create mode 100644 codex-rs/core/src/agent_identity/task_registration.rs diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index ef80693c70..7a1acc3816 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -663,6 +663,7 @@ "bitflags_1.3.2": "{\"dependencies\":[{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1.2\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"walkdir\",\"req\":\"^2.3\"}],\"features\":{\"default\":[],\"example_generated\":[],\"rustc-dep-of-std\":[\"core\",\"compiler_builtins\"]}}", "bitflags_2.10.0": "{\"dependencies\":[{\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"arbitrary\",\"req\":\"^1.0\"},{\"name\":\"bytemuck\",\"optional\":true,\"req\":\"^1.12\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"bytemuck\",\"req\":\"^1.12.2\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde_core\",\"optional\":true,\"req\":\"^1.0.228\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde_lib\",\"package\":\"serde\",\"req\":\"^1.0.103\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0.19\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.18\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"zerocopy\",\"req\":\"^0.8\"}],\"features\":{\"example_generated\":[],\"serde\":[\"serde_core\"],\"std\":[]}}", "bitflags_2.11.0": "{\"dependencies\":[{\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"arbitrary\",\"req\":\"^1.0\"},{\"name\":\"bytemuck\",\"optional\":true,\"req\":\"^1.12\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"bytemuck\",\"req\":\"^1.12.2\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde_core\",\"optional\":true,\"req\":\"^1.0.228\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde_lib\",\"package\":\"serde\",\"req\":\"^1.0.103\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0.19\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.18\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"zerocopy\",\"req\":\"^0.8\"}],\"features\":{\"example_generated\":[],\"serde\":[\"serde_core\"],\"std\":[]}}", + "blake2_0.10.6": "{\"dependencies\":[{\"features\":[\"mac\"],\"name\":\"digest\",\"req\":\"^0.10.3\"},{\"features\":[\"dev\"],\"kind\":\"dev\",\"name\":\"digest\",\"req\":\"^0.10.3\"},{\"kind\":\"dev\",\"name\":\"hex-literal\",\"req\":\"^0.2.2\"}],\"features\":{\"default\":[\"std\"],\"reset\":[],\"simd\":[],\"simd_asm\":[\"simd_opt\"],\"simd_opt\":[\"simd\"],\"size_opt\":[],\"std\":[\"digest/std\"]}}", "block-buffer_0.10.4": "{\"dependencies\":[{\"name\":\"generic-array\",\"req\":\"^0.14\"}],\"features\":{}}", "block-padding_0.3.3": "{\"dependencies\":[{\"name\":\"generic-array\",\"req\":\"^0.14\"}],\"features\":{\"std\":[]}}", "block2_0.6.2": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"std\"],\"name\":\"objc2\",\"req\":\">=0.6.2, <0.8.0\"}],\"features\":{\"alloc\":[],\"compiler-rt\":[\"objc2/unstable-compiler-rt\"],\"default\":[\"std\"],\"gnustep-1-7\":[\"objc2/gnustep-1-7\"],\"gnustep-1-8\":[\"gnustep-1-7\",\"objc2/gnustep-1-8\"],\"gnustep-1-9\":[\"gnustep-1-8\",\"objc2/gnustep-1-9\"],\"gnustep-2-0\":[\"gnustep-1-9\",\"objc2/gnustep-2-0\"],\"gnustep-2-1\":[\"gnustep-2-0\",\"objc2/gnustep-2-1\"],\"std\":[\"alloc\"],\"unstable-coerce-pointee\":[],\"unstable-objfw\":[],\"unstable-private\":[],\"unstable-winobjc\":[\"gnustep-1-8\"]}}", @@ -752,6 +753,8 @@ "crossterm_winapi_0.9.1": "{\"dependencies\":[{\"features\":[\"winbase\",\"consoleapi\",\"processenv\",\"handleapi\",\"synchapi\",\"impl-default\"],\"name\":\"winapi\",\"req\":\"^0.3.8\",\"target\":\"cfg(windows)\"}],\"features\":{}}", "crunchy_0.2.4": "{\"dependencies\":[],\"features\":{\"default\":[\"limit_128\"],\"limit_1024\":[],\"limit_128\":[],\"limit_2048\":[],\"limit_256\":[],\"limit_512\":[],\"limit_64\":[],\"std\":[]}}", "crypto-common_0.1.7": "{\"dependencies\":[{\"features\":[\"more_lengths\"],\"name\":\"generic-array\",\"req\":\"=0.14.7\"},{\"name\":\"rand_core\",\"optional\":true,\"req\":\"^0.6\"},{\"name\":\"typenum\",\"req\":\"^1.14\"}],\"features\":{\"getrandom\":[\"rand_core/getrandom\"],\"std\":[]}}", + "crypto_box_0.9.1": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aead\",\"req\":\"^0.5.2\"},{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"blake2\",\"optional\":true,\"req\":\"^0.10\"},{\"name\":\"chacha20\",\"optional\":true,\"req\":\"^0.9\"},{\"default_features\":false,\"name\":\"crypto_secretbox\",\"req\":\"^0.1.1\"},{\"default_features\":false,\"features\":[\"zeroize\"],\"name\":\"curve25519-dalek\",\"req\":\"^4\"},{\"kind\":\"dev\",\"name\":\"hex-literal\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rmp-serde\",\"req\":\"^1\"},{\"name\":\"salsa20\",\"optional\":true,\"req\":\"^0.10\"},{\"default_features\":false,\"name\":\"serdect\",\"optional\":true,\"req\":\"^0.2\"},{\"default_features\":false,\"name\":\"subtle\",\"req\":\"^2\"},{\"default_features\":false,\"name\":\"zeroize\",\"req\":\"^1\"}],\"features\":{\"alloc\":[\"aead/alloc\"],\"chacha20\":[\"dep:chacha20\",\"crypto_secretbox/chacha20\"],\"default\":[\"alloc\",\"getrandom\",\"salsa20\"],\"getrandom\":[\"aead/getrandom\",\"rand_core\"],\"heapless\":[\"aead/heapless\"],\"rand_core\":[\"aead/rand_core\"],\"salsa20\":[\"dep:salsa20\",\"crypto_secretbox/salsa20\"],\"seal\":[\"dep:blake2\",\"alloc\"],\"serde\":[\"dep:serdect\"],\"std\":[\"aead/std\"]}}", + "crypto_secretbox_0.1.1": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aead\",\"req\":\"^0.5\"},{\"features\":[\"zeroize\"],\"name\":\"chacha20\",\"optional\":true,\"req\":\"^0.9\"},{\"default_features\":false,\"name\":\"cipher\",\"req\":\"^0.4\"},{\"default_features\":false,\"features\":[\"zeroize\"],\"name\":\"generic-array\",\"req\":\"^0.14.7\"},{\"kind\":\"dev\",\"name\":\"hex-literal\",\"req\":\"^0.4\"},{\"name\":\"poly1305\",\"req\":\"^0.8\"},{\"features\":[\"zeroize\"],\"name\":\"salsa20\",\"optional\":true,\"req\":\"^0.10\"},{\"default_features\":false,\"name\":\"subtle\",\"req\":\"^2\"},{\"default_features\":false,\"name\":\"zeroize\",\"req\":\"^1\"}],\"features\":{\"alloc\":[\"aead/alloc\"],\"default\":[\"alloc\",\"getrandom\",\"salsa20\"],\"getrandom\":[\"aead/getrandom\",\"rand_core\"],\"heapless\":[\"aead/heapless\"],\"rand_core\":[\"aead/rand_core\"],\"std\":[\"aead/std\",\"alloc\"],\"stream\":[\"aead/stream\"]}}", "csv-core_0.1.13": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"arrayvec\",\"req\":\"^0.5\"},{\"default_features\":false,\"name\":\"memchr\",\"req\":\"^2\"}],\"features\":{\"default\":[],\"libc\":[\"memchr/libc\"]}}", "csv_1.4.0": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"alloc\",\"serde\"],\"kind\":\"dev\",\"name\":\"bstr\",\"req\":\"^1.7.0\"},{\"name\":\"csv-core\",\"req\":\"^0.1.11\"},{\"name\":\"itoa\",\"req\":\"^1\"},{\"name\":\"ryu\",\"req\":\"^1\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.221\"},{\"name\":\"serde_core\",\"req\":\"^1.0.221\"}],\"features\":{}}", "ctor-proc-macro_0.0.7": "{\"dependencies\":[],\"features\":{\"default\":[]}}", diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 296e51725c..bd74d83ad6 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -940,6 +940,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -1960,6 +1969,7 @@ dependencies = [ "codex-windows-sandbox", "core-foundation 0.9.4", "core_test_support", + "crypto_box", "csv", "ctor 0.6.3", "dirs", @@ -1990,6 +2000,7 @@ dependencies = [ "serde_json", "serial_test", "sha1", + "sha2", "shlex", "similar", "tempfile", @@ -3656,9 +3667,40 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", + "rand_core 0.6.4", "typenum", ] +[[package]] +name = "crypto_box" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16182b4f39a82ec8a6851155cc4c0cda3065bb1db33651726a29e1951de0f009" +dependencies = [ + "aead", + "blake2", + "crypto_secretbox", + "curve25519-dalek", + "salsa20", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto_secretbox" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d6cf87adf719ddf43a805e92c6870a531aedda35ff640442cbaf8674e141e1" +dependencies = [ + "aead", + "cipher", + "generic-array", + "poly1305", + "salsa20", + "subtle", + "zeroize", +] + [[package]] name = "csv" version = "1.4.0" @@ -4948,6 +4990,7 @@ checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", + "zeroize", ] [[package]] diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index deb3738812..c8f6b9e5a7 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -224,6 +224,7 @@ color-eyre = "0.6.3" constant_time_eq = "0.3.1" crossbeam-channel = "0.5.15" crossterm = "0.28.1" +crypto_box = { version = "0.9.1", features = ["seal"] } csv = "1.3.1" ctor = "0.6.3" deno_core_icudata = "0.77.0" diff --git a/codex-rs/app-server/tests/suite/auth.rs b/codex-rs/app-server/tests/suite/auth.rs index 1e60871012..b35a6fcba1 100644 --- a/codex-rs/app-server/tests/suite/auth.rs +++ b/codex-rs/app-server/tests/suite/auth.rs @@ -351,6 +351,8 @@ async fn get_auth_status_omits_token_after_proactive_refresh_failure() -> Result )?; let server = MockServer::start().await; + // App-server startup may proactively read stale auth before this test sends + // getAuthStatus; require the refresh path without depending on that race. Mock::given(method("POST")) .and(path("/oauth/token")) .respond_with(ResponseTemplate::new(401).set_body_json(serde_json::json!({ @@ -358,7 +360,7 @@ async fn get_auth_status_omits_token_after_proactive_refresh_failure() -> Result "code": "refresh_token_reused" } }))) - .expect(2) + .expect(1..=2) .mount(&server) .await; @@ -418,6 +420,8 @@ async fn get_auth_status_returns_token_after_proactive_refresh_recovery() -> Res )?; let server = MockServer::start().await; + // App-server startup may proactively read stale auth before this test sends + // getAuthStatus; require the refresh path without depending on that race. Mock::given(method("POST")) .and(path("/oauth/token")) .respond_with(ResponseTemplate::new(401).set_body_json(serde_json::json!({ @@ -425,7 +429,7 @@ async fn get_auth_status_returns_token_after_proactive_refresh_recovery() -> Res "code": "refresh_token_reused" } }))) - .expect(2) + .expect(1..=2) .mount(&server) .await; diff --git a/codex-rs/app-server/tests/suite/v2/plugin_list.rs b/codex-rs/app-server/tests/suite/v2/plugin_list.rs index 8bc4a8598e..5036101de4 100644 --- a/codex-rs/app-server/tests/suite/v2/plugin_list.rs +++ b/codex-rs/app-server/tests/suite/v2/plugin_list.rs @@ -30,7 +30,8 @@ use wiremock::matchers::method; use wiremock::matchers::path; use wiremock::matchers::query_param; -const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10); +// These tests start full app-server processes and can also run plugin startup warmers. +const DEFAULT_TIMEOUT: Duration = Duration::from_secs(60); const TEST_CURATED_PLUGIN_SHA: &str = "0123456789abcdef0123456789abcdef01234567"; const STARTUP_REMOTE_PLUGIN_SYNC_MARKER_FILE: &str = ".tmp/app-server-remote-plugin-sync-v1"; diff --git a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs index 1f7b8a3e4b..11dc0ac4cf 100644 --- a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs +++ b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs @@ -72,6 +72,7 @@ use wiremock::matchers::path; use wiremock::matchers::path_regex; const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10); +const DELEGATED_SHELL_TOOL_TIMEOUT_MS: u64 = 30_000; const STARTUP_CONTEXT_HEADER: &str = "Startup context from Codex."; const V2_STEERING_ACKNOWLEDGEMENT: &str = "This was sent to steer the previous background agent task."; @@ -1717,7 +1718,9 @@ async fn webrtc_v2_tool_call_delegated_turn_can_execute_shell_tool() -> Result<( create_shell_command_sse_response( realtime_tool_ok_command(), /*workdir*/ None, - Some(5000), + // Windows CI can spend several seconds starting the nested PowerShell command. This + // test verifies delegated shell-tool plumbing, not timeout enforcement. + Some(DELEGATED_SHELL_TOOL_TIMEOUT_MS), "shell_call", )?, create_final_assistant_message_sse_response("shell tool finished")?, diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index 4ce4192b81..a0d4259810 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -35,6 +35,7 @@ codex-connectors = { workspace = true } codex-config = { workspace = true } codex-core-plugins = { workspace = true } codex-core-skills = { workspace = true } +crypto_box = { workspace = true } codex-exec-server = { workspace = true } codex-features = { workspace = true } codex-feedback = { workspace = true } @@ -99,6 +100,7 @@ rmcp = { workspace = true, default-features = false, features = [ serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } sha1 = { workspace = true } +sha2 = { workspace = true } shlex = { workspace = true } similar = { workspace = true } tempfile = { workspace = true } diff --git a/codex-rs/core/src/agent_identity.rs b/codex-rs/core/src/agent_identity.rs index 2c897b5456..cfcab507bb 100644 --- a/codex-rs/core/src/agent_identity.rs +++ b/codex-rs/core/src/agent_identity.rs @@ -27,6 +27,10 @@ use tracing::debug; use tracing::info; use tracing::warn; +mod task_registration; + +pub(crate) use task_registration::RegisteredAgentTask; + use crate::config::Config; const AGENT_REGISTRATION_TIMEOUT: Duration = Duration::from_secs(15); @@ -119,32 +123,58 @@ impl AgentIdentityManager { return Ok(None); } - let Some(auth) = self.auth_manager.auth().await else { - debug!("skipping agent identity registration because no auth is available"); + let Some((auth, binding)) = self.current_auth_binding().await else { return Ok(None); }; - let Some(binding) = - AgentIdentityBinding::from_auth(&auth, self.auth_manager.forced_chatgpt_workspace_id()) - else { - debug!("skipping agent identity registration because ChatGPT auth is unavailable"); - return Ok(None); - }; + self.ensure_registered_identity_for_binding(&auth, &binding) + .await + .map(Some) + } + async fn ensure_registered_identity_for_binding( + &self, + auth: &CodexAuth, + binding: &AgentIdentityBinding, + ) -> Result { let _guard = self.ensure_lock.lock().await; - if let Some(stored_identity) = self.load_stored_identity(&auth, &binding)? { + if let Some(stored_identity) = self.load_stored_identity(auth, binding)? { info!( agent_runtime_id = %stored_identity.agent_runtime_id, binding_id = %binding.binding_id, "reusing stored agent identity" ); - return Ok(Some(stored_identity)); + return Ok(stored_identity); } - let stored_identity = self.register_agent_identity(&binding).await?; - self.store_identity(&auth, &stored_identity)?; - Ok(Some(stored_identity)) + let stored_identity = self.register_agent_identity(binding).await?; + self.store_identity(auth, &stored_identity)?; + Ok(stored_identity) + } + + pub(crate) async fn task_matches_current_binding(&self, task: &RegisteredAgentTask) -> bool { + if !self.feature_enabled { + return false; + } + + self.current_auth_binding() + .await + .is_some_and(|(_, binding)| task.matches_binding(&binding)) + } + + async fn current_auth_binding(&self) -> Option<(CodexAuth, AgentIdentityBinding)> { + let Some(auth) = self.auth_manager.auth().await else { + debug!("skipping agent identity flow because no auth is available"); + return None; + }; + + let binding = + AgentIdentityBinding::from_auth(&auth, self.auth_manager.forced_chatgpt_workspace_id()); + if binding.is_none() { + debug!("skipping agent identity flow because ChatGPT auth is unavailable"); + } + binding.map(|binding| (auth, binding)) } async fn register_agent_identity( @@ -351,12 +381,11 @@ impl StoredAgentIdentity { } fn matches_binding(&self, binding: &AgentIdentityBinding) -> bool { - self.binding_id == binding.binding_id - && self.chatgpt_account_id == binding.chatgpt_account_id - && match binding.chatgpt_user_id.as_deref() { - Some(chatgpt_user_id) => self.chatgpt_user_id.as_deref() == Some(chatgpt_user_id), - None => true, - } + binding.matches_parts( + &self.binding_id, + &self.chatgpt_account_id, + self.chatgpt_user_id.as_deref(), + ) } fn validate_key_material(&self) -> Result<()> { @@ -375,6 +404,20 @@ impl StoredAgentIdentity { } impl AgentIdentityBinding { + fn matches_parts( + &self, + binding_id: &str, + chatgpt_account_id: &str, + chatgpt_user_id: Option<&str>, + ) -> bool { + binding_id == self.binding_id + && chatgpt_account_id == self.chatgpt_account_id + && match self.chatgpt_user_id.as_deref() { + Some(expected_user_id) => chatgpt_user_id == Some(expected_user_id), + None => true, + } + } + fn from_auth(auth: &CodexAuth, forced_workspace_id: Option) -> Option { if !auth.is_chatgpt_auth() { return None; diff --git a/codex-rs/core/src/agent_identity/task_registration.rs b/codex-rs/core/src/agent_identity/task_registration.rs new file mode 100644 index 0000000000..4fc5d51282 --- /dev/null +++ b/codex-rs/core/src/agent_identity/task_registration.rs @@ -0,0 +1,470 @@ +use std::time::Duration; + +use anyhow::Context; +use anyhow::Result; +use crypto_box::SecretKey as Curve25519SecretKey; +use ed25519_dalek::Signer as _; +use serde::Deserialize; +use serde::Serialize; +use sha2::Digest as _; +use sha2::Sha512; +use tracing::info; + +use super::*; + +const AGENT_TASK_REGISTRATION_TIMEOUT: Duration = Duration::from_secs(15); + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct RegisteredAgentTask { + pub(crate) binding_id: String, + pub(crate) chatgpt_account_id: String, + pub(crate) chatgpt_user_id: Option, + pub(crate) agent_runtime_id: String, + pub(crate) task_id: String, + pub(crate) registered_at: String, +} + +#[derive(Debug, Serialize)] +struct RegisterTaskRequest { + signature: String, + timestamp: String, +} + +#[derive(Debug, Deserialize)] +struct RegisterTaskResponse { + encrypted_task_id: String, +} + +impl AgentIdentityManager { + pub(crate) async fn register_task(&self) -> Result> { + if !self.feature_enabled { + return Ok(None); + } + + let Some((auth, binding)) = self.current_auth_binding().await else { + return Ok(None); + }; + + self.register_task_for_binding(auth, binding).await + } + + async fn register_task_for_binding( + &self, + auth: CodexAuth, + binding: AgentIdentityBinding, + ) -> Result> { + let stored_identity = self + .ensure_registered_identity_for_binding(&auth, &binding) + .await?; + + let timestamp = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true); + let request_body = RegisterTaskRequest { + signature: sign_task_registration_payload(&stored_identity, ×tamp)?, + timestamp, + }; + + let client = create_client(); + let url = + agent_task_registration_url(&self.chatgpt_base_url, &stored_identity.agent_runtime_id); + let human_biscuit = self.mint_human_biscuit(&binding, "POST", &url).await?; + let response = client + .post(&url) + .header("X-OpenAI-Authorization", human_biscuit) + .json(&request_body) + .timeout(AGENT_TASK_REGISTRATION_TIMEOUT) + .send() + .await + .with_context(|| format!("failed to send agent task registration request to {url}"))?; + + if response.status().is_success() { + let response_body = response + .json::() + .await + .with_context(|| format!("failed to parse agent task response from {url}"))?; + let registered_task = RegisteredAgentTask { + binding_id: stored_identity.binding_id.clone(), + chatgpt_account_id: stored_identity.chatgpt_account_id.clone(), + chatgpt_user_id: stored_identity.chatgpt_user_id.clone(), + agent_runtime_id: stored_identity.agent_runtime_id.clone(), + task_id: decrypt_task_id_response( + &stored_identity, + &response_body.encrypted_task_id, + )?, + registered_at: Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true), + }; + info!( + agent_runtime_id = %registered_task.agent_runtime_id, + task_id = %registered_task.task_id, + "registered agent task" + ); + return Ok(Some(registered_task)); + } + + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + anyhow::bail!("agent task registration failed with status {status} from {url}: {body}") + } +} + +impl RegisteredAgentTask { + pub(super) fn matches_binding(&self, binding: &AgentIdentityBinding) -> bool { + binding.matches_parts( + &self.binding_id, + &self.chatgpt_account_id, + self.chatgpt_user_id.as_deref(), + ) + } + + pub(crate) fn has_same_binding(&self, other: &Self) -> bool { + self.binding_id == other.binding_id + && self.chatgpt_account_id == other.chatgpt_account_id + && self.chatgpt_user_id == other.chatgpt_user_id + } +} + +fn sign_task_registration_payload( + stored_identity: &StoredAgentIdentity, + timestamp: &str, +) -> Result { + let signing_key = stored_identity.signing_key()?; + let payload = format!("{}:{timestamp}", stored_identity.agent_runtime_id); + Ok(BASE64_STANDARD.encode(signing_key.sign(payload.as_bytes()).to_bytes())) +} + +fn decrypt_task_id_response( + stored_identity: &StoredAgentIdentity, + encrypted_task_id: &str, +) -> Result { + let signing_key = stored_identity.signing_key()?; + let ciphertext = BASE64_STANDARD + .decode(encrypted_task_id) + .context("encrypted task id is not valid base64")?; + let plaintext = curve25519_secret_key_from_signing_key(&signing_key) + .unseal(&ciphertext) + .map_err(|_| anyhow::anyhow!("failed to decrypt encrypted task id"))?; + String::from_utf8(plaintext).context("decrypted task id is not valid UTF-8") +} + +fn curve25519_secret_key_from_signing_key(signing_key: &SigningKey) -> Curve25519SecretKey { + let digest = Sha512::digest(signing_key.to_bytes()); + let mut secret_key = [0u8; 32]; + secret_key.copy_from_slice(&digest[..32]); + secret_key[0] &= 248; + secret_key[31] &= 127; + secret_key[31] |= 64; + Curve25519SecretKey::from(secret_key) +} + +fn agent_task_registration_url(chatgpt_base_url: &str, agent_runtime_id: &str) -> String { + let trimmed = chatgpt_base_url.trim_end_matches('/'); + format!("{trimmed}/v1/agent/{agent_runtime_id}/task/register") +} + +#[cfg(test)] +mod tests { + use base64::engine::general_purpose::URL_SAFE_NO_PAD; + use codex_app_server_protocol::AuthMode as ApiAuthMode; + use codex_login::AuthCredentialsStoreMode; + use codex_login::AuthDotJson; + use codex_login::save_auth; + use codex_login::token_data::IdTokenInfo; + use codex_login::token_data::TokenData; + use pretty_assertions::assert_eq; + use wiremock::Mock; + use wiremock::MockServer; + use wiremock::ResponseTemplate; + use wiremock::matchers::header; + use wiremock::matchers::method; + use wiremock::matchers::path; + + use super::*; + + #[tokio::test] + async fn register_task_skips_when_feature_is_disabled() { + let auth_manager = + AuthManager::from_auth_for_testing(make_chatgpt_auth("account-123", Some("user-123"))); + let manager = AgentIdentityManager::new_for_tests( + auth_manager, + /*feature_enabled*/ false, + "https://chatgpt.com/backend-api/".to_string(), + SessionSource::Cli, + ); + + assert_eq!(manager.register_task().await.unwrap(), None); + } + + #[tokio::test] + async fn register_task_skips_for_api_key_auth() { + let auth_manager = AuthManager::from_auth_for_testing(CodexAuth::from_api_key("test-key")); + let manager = AgentIdentityManager::new_for_tests( + auth_manager, + /*feature_enabled*/ true, + "https://chatgpt.com/backend-api/".to_string(), + SessionSource::Cli, + ); + + assert_eq!(manager.register_task().await.unwrap(), None); + } + + #[tokio::test] + async fn register_task_registers_and_decrypts_plaintext_task_id() { + let server = MockServer::start().await; + let chatgpt_base_url = server.uri(); + mount_human_biscuit(&server, &chatgpt_base_url, "agent-123").await; + let auth = make_chatgpt_auth("account-123", Some("user-123")); + let auth_manager = AuthManager::from_auth_for_testing(auth.clone()); + let manager = AgentIdentityManager::new_for_tests( + auth_manager, + /*feature_enabled*/ true, + chatgpt_base_url, + SessionSource::Cli, + ); + let stored_identity = seed_stored_identity(&manager, &auth, "agent-123", "account-123"); + let encrypted_task_id = + encrypt_task_id_for_identity(&stored_identity, "task_123").expect("task ciphertext"); + + Mock::given(method("POST")) + .and(path("/v1/agent/agent-123/task/register")) + .and(header("x-openai-authorization", "human-biscuit")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "encrypted_task_id": encrypted_task_id, + }))) + .expect(1) + .mount(&server) + .await; + + let task = manager + .register_task() + .await + .unwrap() + .expect("task should be registered"); + + assert_eq!( + task, + RegisteredAgentTask { + binding_id: "chatgpt-account-account-123".to_string(), + chatgpt_account_id: "account-123".to_string(), + chatgpt_user_id: Some("user-123".to_string()), + agent_runtime_id: "agent-123".to_string(), + task_id: "task_123".to_string(), + registered_at: task.registered_at.clone(), + } + ); + } + + #[tokio::test] + async fn register_task_uses_chatgpt_base_url() { + let server = MockServer::start().await; + let chatgpt_base_url = format!("{}/backend-api", server.uri()); + mount_human_biscuit(&server, &chatgpt_base_url, "agent-fallback").await; + let auth = make_chatgpt_auth("account-123", Some("user-123")); + let auth_manager = AuthManager::from_auth_for_testing(auth.clone()); + let manager = AgentIdentityManager::new_for_tests( + auth_manager, + /*feature_enabled*/ true, + chatgpt_base_url, + SessionSource::Cli, + ); + let stored_identity = + seed_stored_identity(&manager, &auth, "agent-fallback", "account-123"); + let encrypted_task_id = encrypt_task_id_for_identity(&stored_identity, "task_fallback") + .expect("task ciphertext"); + + Mock::given(method("POST")) + .and(path("/backend-api/v1/agent/agent-fallback/task/register")) + .and(header("x-openai-authorization", "human-biscuit")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "encrypted_task_id": encrypted_task_id, + }))) + .expect(1) + .mount(&server) + .await; + + let task = manager + .register_task() + .await + .unwrap() + .expect("task should be registered"); + + assert_eq!(task.agent_runtime_id, "agent-fallback"); + assert_eq!(task.task_id, "task_fallback"); + } + + #[tokio::test] + async fn register_task_for_binding_keeps_one_auth_snapshot() { + let server = MockServer::start().await; + let chatgpt_base_url = server.uri(); + mount_human_biscuit(&server, &chatgpt_base_url, "agent-123").await; + let binding_auth = make_chatgpt_auth("account-123", Some("user-123")); + let auth_manager = + AuthManager::from_auth_for_testing(make_chatgpt_auth("account-456", Some("user-456"))); + let manager = AgentIdentityManager::new_for_tests( + auth_manager, + /*feature_enabled*/ true, + chatgpt_base_url, + SessionSource::Cli, + ); + let stored_identity = + seed_stored_identity(&manager, &binding_auth, "agent-123", "account-123"); + let encrypted_task_id = + encrypt_task_id_for_identity(&stored_identity, "task_123").expect("task ciphertext"); + let binding = + AgentIdentityBinding::from_auth(&binding_auth, /*forced_workspace_id*/ None) + .expect("binding"); + + Mock::given(method("POST")) + .and(path("/v1/agent/agent-123/task/register")) + .and(header("x-openai-authorization", "human-biscuit")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "encrypted_task_id": encrypted_task_id, + }))) + .expect(1) + .mount(&server) + .await; + + let task = manager + .register_task_for_binding(binding_auth, binding) + .await + .unwrap() + .expect("task should be registered"); + + assert_eq!( + task, + RegisteredAgentTask { + binding_id: "chatgpt-account-account-123".to_string(), + chatgpt_account_id: "account-123".to_string(), + chatgpt_user_id: Some("user-123".to_string()), + agent_runtime_id: "agent-123".to_string(), + task_id: "task_123".to_string(), + registered_at: task.registered_at.clone(), + } + ); + } + + #[tokio::test] + async fn task_matches_current_binding_rejects_stale_auth_binding() { + let auth_manager = + AuthManager::from_auth_for_testing(make_chatgpt_auth("account-456", Some("user-456"))); + let manager = AgentIdentityManager::new_for_tests( + auth_manager, + /*feature_enabled*/ true, + "https://chatgpt.com/backend-api/".to_string(), + SessionSource::Cli, + ); + let task = RegisteredAgentTask { + binding_id: "chatgpt-account-account-123".to_string(), + chatgpt_account_id: "account-123".to_string(), + chatgpt_user_id: Some("user-123".to_string()), + agent_runtime_id: "agent-123".to_string(), + task_id: "task_123".to_string(), + registered_at: "2026-03-23T12:00:00Z".to_string(), + }; + + assert!(!manager.task_matches_current_binding(&task).await); + } + + async fn mount_human_biscuit( + server: &MockServer, + chatgpt_base_url: &str, + agent_runtime_id: &str, + ) { + let biscuit_url = agent_identity_biscuit_url(chatgpt_base_url); + let biscuit_path = reqwest::Url::parse(&biscuit_url) + .expect("biscuit URL parses") + .path() + .to_string(); + let target_url = agent_task_registration_url(chatgpt_base_url, agent_runtime_id); + Mock::given(method("GET")) + .and(path(biscuit_path)) + .and(header("authorization", "Bearer access-token-account-123")) + .and(header("x-original-method", "POST")) + .and(header("x-original-url", target_url)) + .respond_with( + ResponseTemplate::new(200).insert_header("x-openai-authorization", "human-biscuit"), + ) + .expect(1) + .mount(server) + .await; + } + + fn seed_stored_identity( + manager: &AgentIdentityManager, + auth: &CodexAuth, + agent_runtime_id: &str, + account_id: &str, + ) -> StoredAgentIdentity { + let key_material = generate_agent_key_material().expect("key material"); + let binding = + AgentIdentityBinding::from_auth(auth, /*forced_workspace_id*/ None).expect("binding"); + let stored_identity = StoredAgentIdentity { + binding_id: binding.binding_id, + chatgpt_account_id: account_id.to_string(), + chatgpt_user_id: Some("user-123".to_string()), + agent_runtime_id: agent_runtime_id.to_string(), + private_key_pkcs8_base64: key_material.private_key_pkcs8_base64, + public_key_ssh: key_material.public_key_ssh, + registered_at: Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true), + abom: manager.abom.clone(), + }; + manager + .store_identity(auth, &stored_identity) + .expect("store identity"); + let persisted = auth + .get_agent_identity(account_id) + .expect("persisted identity"); + assert_eq!(persisted.agent_runtime_id, agent_runtime_id); + stored_identity + } + + fn encrypt_task_id_for_identity( + stored_identity: &StoredAgentIdentity, + task_id: &str, + ) -> Result { + let mut rng = crypto_box::aead::OsRng; + let public_key = + curve25519_secret_key_from_signing_key(&stored_identity.signing_key()?).public_key(); + let ciphertext = public_key + .seal(&mut rng, task_id.as_bytes()) + .map_err(|_| anyhow::anyhow!("failed to encrypt test task id"))?; + Ok(BASE64_STANDARD.encode(ciphertext)) + } + + fn make_chatgpt_auth(account_id: &str, user_id: Option<&str>) -> CodexAuth { + let tempdir = tempfile::tempdir().expect("tempdir"); + let auth_json = AuthDotJson { + auth_mode: Some(ApiAuthMode::Chatgpt), + openai_api_key: None, + tokens: Some(TokenData { + id_token: IdTokenInfo { + email: None, + chatgpt_plan_type: None, + chatgpt_user_id: user_id.map(ToOwned::to_owned), + chatgpt_account_id: Some(account_id.to_string()), + chatgpt_account_is_fedramp: false, + raw_jwt: fake_id_token(account_id, user_id), + }, + access_token: format!("access-token-{account_id}"), + refresh_token: "refresh-token".to_string(), + account_id: Some(account_id.to_string()), + }), + last_refresh: Some(Utc::now()), + agent_identity: None, + }; + save_auth(tempdir.path(), &auth_json, AuthCredentialsStoreMode::File).expect("save auth"); + CodexAuth::from_auth_storage(tempdir.path(), AuthCredentialsStoreMode::File) + .expect("load auth") + .expect("auth") + } + + fn fake_id_token(account_id: &str, user_id: Option<&str>) -> String { + let header = URL_SAFE_NO_PAD.encode(r#"{"alg":"none","typ":"JWT"}"#); + let payload = serde_json::json!({ + "https://api.openai.com/auth": { + "chatgpt_user_id": user_id, + "chatgpt_account_id": account_id, + } + }); + let payload = URL_SAFE_NO_PAD.encode(payload.to_string()); + format!("{header}.{payload}.signature") + } +} diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 4e09542d88..f356b3fff7 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -14,6 +14,7 @@ use crate::agent::MailboxReceiver; use crate::agent::agent_status_from_event; use crate::agent::status::is_final; use crate::agent_identity::AgentIdentityManager; +use crate::agent_identity::RegisteredAgentTask; use crate::apps::render_apps_section; use crate::commit_attribution::commit_message_trailer_instruction; use crate::compact; @@ -1577,7 +1578,7 @@ impl Session { async fn fail_agent_identity_registration(self: &Arc, error: anyhow::Error) { warn!(error = %error, "agent identity registration failed"); let message = format!( - "Agent identity registration failed. Codex cannot continue while `features.use_agent_identity` is enabled: {error}" + "Agent identity registration failed while `features.use_agent_identity` is enabled: {error}" ); self.send_event_raw(Event { id: self.next_internal_sub_id(), @@ -1587,7 +1588,90 @@ impl Session { }), }) .await; - handlers::shutdown(self, self.next_internal_sub_id()).await; + } + + async fn cached_agent_task_for_current_binding(&self) -> Option { + let agent_task = { + let state = self.state.lock().await; + state.agent_task() + }?; + + if self + .services + .agent_identity_manager + .task_matches_current_binding(&agent_task) + .await + { + debug!( + agent_runtime_id = %agent_task.agent_runtime_id, + task_id = %agent_task.task_id, + "reusing cached agent task" + ); + return Some(agent_task); + } + + debug!( + agent_runtime_id = %agent_task.agent_runtime_id, + task_id = %agent_task.task_id, + "discarding cached agent task because auth binding changed" + ); + let mut state = self.state.lock().await; + if state.agent_task().as_ref() == Some(&agent_task) { + state.clear_agent_task(); + } + None + } + + async fn ensure_agent_task_registered(&self) -> anyhow::Result> { + if let Some(agent_task) = self.cached_agent_task_for_current_binding().await { + return Ok(Some(agent_task)); + } + + for _ in 0..2 { + let Some(agent_task) = self.services.agent_identity_manager.register_task().await? + else { + return Ok(None); + }; + + if !self + .services + .agent_identity_manager + .task_matches_current_binding(&agent_task) + .await + { + debug!( + agent_runtime_id = %agent_task.agent_runtime_id, + task_id = %agent_task.task_id, + "discarding newly registered agent task because auth binding changed" + ); + continue; + } + + { + let mut state = self.state.lock().await; + if let Some(existing_agent_task) = state.agent_task() { + if existing_agent_task.has_same_binding(&agent_task) { + return Ok(Some(existing_agent_task)); + } + debug!( + agent_runtime_id = %existing_agent_task.agent_runtime_id, + task_id = %existing_agent_task.task_id, + "replacing cached agent task because auth binding changed" + ); + } + state.set_agent_task(agent_task.clone()); + } + + info!( + thread_id = %self.conversation_id, + agent_runtime_id = %agent_task.agent_runtime_id, + task_id = %agent_task.task_id, + "registered agent task for thread" + ); + return Ok(Some(agent_task)); + } + + Ok(None) } #[allow(clippy::too_many_arguments)] @@ -5168,6 +5252,20 @@ pub(crate) async fn run_turn( })) .await; } + if let Err(error) = sess.ensure_agent_task_registered().await { + warn!(error = %error, "agent task registration failed"); + sess.send_event( + turn_context.as_ref(), + EventMsg::Error(ErrorEvent { + message: format!( + "Agent task registration failed. Please try again; Codex will attempt to register the task again on the next turn: {error}" + ), + codex_error_info: Some(CodexErrorInfo::Other), + }), + ) + .await; + return None; + } if !skill_items.is_empty() { sess.record_conversation_items(&turn_context, &skill_items) diff --git a/codex-rs/core/src/codex_tests.rs b/codex-rs/core/src/codex_tests.rs index 81ebd6c9f8..aa5583405e 100644 --- a/codex-rs/core/src/codex_tests.rs +++ b/codex-rs/core/src/codex_tests.rs @@ -4182,7 +4182,7 @@ pub(crate) async fn make_session_and_context_with_rx() -> ( } #[tokio::test] -async fn fail_agent_identity_registration_emits_error_and_shutdown() { +async fn fail_agent_identity_registration_emits_error_without_shutdown() { let (session, _turn_context, rx_event) = make_session_and_context_with_rx().await; session @@ -4200,21 +4200,14 @@ async fn fail_agent_identity_registration_emits_error_and_shutdown() { }) => { assert_eq!( message, - "Agent identity registration failed. Codex cannot continue while `features.use_agent_identity` is enabled: registration exploded".to_string() + "Agent identity registration failed while `features.use_agent_identity` is enabled: registration exploded".to_string() ); assert_eq!(codex_error_info, Some(CodexErrorInfo::Other)); } other => panic!("expected error event, got {other:?}"), } - let shutdown_event = timeout(Duration::from_secs(1), rx_event.recv()) - .await - .expect("shutdown event should arrive") - .expect("shutdown event should be readable"); - match shutdown_event.msg { - EventMsg::ShutdownComplete => {} - other => panic!("expected shutdown event, got {other:?}"), - } + assert!(rx_event.try_recv().is_err()); } #[tokio::test] diff --git a/codex-rs/core/src/state/session.rs b/codex-rs/core/src/state/session.rs index 4360b16de4..42ed15e87a 100644 --- a/codex-rs/core/src/state/session.rs +++ b/codex-rs/core/src/state/session.rs @@ -6,6 +6,7 @@ use codex_sandboxing::policy_transforms::merge_permission_profiles; use std::collections::HashMap; use std::collections::HashSet; +use crate::agent_identity::RegisteredAgentTask; use crate::codex::PreviousTurnSettings; use crate::codex::SessionConfiguration; use crate::context_manager::ContextManager; @@ -30,6 +31,7 @@ pub(crate) struct SessionState { previous_turn_settings: Option, /// Startup prewarmed session prepared during session initialization. pub(crate) startup_prewarm: Option, + pub(crate) agent_task: Option, pub(crate) active_connector_selection: HashSet, pub(crate) pending_session_start_source: Option, granted_permissions: Option, @@ -49,6 +51,7 @@ impl SessionState { mcp_dependency_prompted: HashSet::new(), previous_turn_settings: None, startup_prewarm: None, + agent_task: None, active_connector_selection: HashSet::new(), pending_session_start_source: None, granted_permissions: None, @@ -186,6 +189,18 @@ impl SessionState { self.startup_prewarm.take() } + pub(crate) fn agent_task(&self) -> Option { + self.agent_task.clone() + } + + pub(crate) fn set_agent_task(&mut self, agent_task: RegisteredAgentTask) { + self.agent_task = Some(agent_task); + } + + pub(crate) fn clear_agent_task(&mut self) { + self.agent_task = None; + } + // Adds connector IDs to the active set and returns the merged selection. pub(crate) fn merge_connector_selection(&mut self, connector_ids: I) -> HashSet where diff --git a/codex-rs/core/src/state/session_tests.rs b/codex-rs/core/src/state/session_tests.rs index 1af7ccc8f6..6816c8731d 100644 --- a/codex-rs/core/src/state/session_tests.rs +++ b/codex-rs/core/src/state/session_tests.rs @@ -1,4 +1,5 @@ use super::*; +use crate::agent_identity::RegisteredAgentTask; use crate::codex::make_session_configuration_for_tests; use codex_protocol::protocol::CreditsSnapshot; use codex_protocol::protocol::RateLimitWindow; @@ -33,6 +34,43 @@ async fn clear_connector_selection_removes_entries() { assert_eq!(state.get_connector_selection(), HashSet::new()); } +#[tokio::test] +async fn set_agent_task_persists_plaintext_task_for_session_reuse() { + let session_configuration = make_session_configuration_for_tests().await; + let mut state = SessionState::new(session_configuration); + let agent_task = RegisteredAgentTask { + binding_id: "chatgpt-account-account-123".to_string(), + chatgpt_account_id: "account-123".to_string(), + chatgpt_user_id: Some("user-123".to_string()), + agent_runtime_id: "agent_123".to_string(), + task_id: "task_123".to_string(), + registered_at: "2026-03-23T12:00:00Z".to_string(), + }; + + state.set_agent_task(agent_task.clone()); + + assert_eq!(state.agent_task(), Some(agent_task)); +} + +#[tokio::test] +async fn clear_agent_task_removes_cached_task() { + let session_configuration = make_session_configuration_for_tests().await; + let mut state = SessionState::new(session_configuration); + let agent_task = RegisteredAgentTask { + binding_id: "chatgpt-account-account-123".to_string(), + chatgpt_account_id: "account-123".to_string(), + chatgpt_user_id: Some("user-123".to_string()), + agent_runtime_id: "agent_123".to_string(), + task_id: "task_123".to_string(), + registered_at: "2026-03-23T12:00:00Z".to_string(), + }; + + state.set_agent_task(agent_task); + state.clear_agent_task(); + + assert_eq!(state.agent_task(), None); +} + #[tokio::test] async fn set_rate_limits_defaults_limit_id_to_codex_when_missing() { let session_configuration = make_session_configuration_for_tests().await; diff --git a/codex-rs/core/tests/suite/realtime_conversation.rs b/codex-rs/core/tests/suite/realtime_conversation.rs index 1cd97391bb..35ec9668a3 100644 --- a/codex-rs/core/tests/suite/realtime_conversation.rs +++ b/codex-rs/core/tests/suite/realtime_conversation.rs @@ -15,6 +15,7 @@ use codex_protocol::protocol::ConversationStartTransport; use codex_protocol::protocol::ConversationTextParams; use codex_protocol::protocol::ErrorEvent; use codex_protocol::protocol::EventMsg; +use codex_protocol::protocol::GitInfo; use codex_protocol::protocol::InitialHistory; use codex_protocol::protocol::Op; use codex_protocol::protocol::RealtimeAudioFrame; @@ -24,7 +25,11 @@ use codex_protocol::protocol::RealtimeEvent; use codex_protocol::protocol::RealtimeOutputModality; use codex_protocol::protocol::RealtimeVoice; use codex_protocol::protocol::RolloutItem; +use codex_protocol::protocol::RolloutLine; +use codex_protocol::protocol::SessionMeta; +use codex_protocol::protocol::SessionMetaLine; use codex_protocol::protocol::SessionSource; +use codex_protocol::protocol::UserMessageEvent; use codex_protocol::user_input::UserInput; use codex_utils_output_truncation::approx_token_count; use core_test_support::responses; @@ -65,6 +70,7 @@ const MEMORY_PROMPT_PHRASE: &str = "You have access to a memory folder with guidance from prior runs."; const REALTIME_CONVERSATION_TEST_SUBPROCESS_ENV_VAR: &str = "CODEX_REALTIME_CONVERSATION_TEST_SUBPROCESS"; +const WEBSOCKET_REQUEST_TIMEOUT: Duration = Duration::from_secs(30); #[derive(Debug, Clone)] struct RealtimeCallRequestCapture { @@ -141,7 +147,7 @@ async fn wait_for_matching_websocket_request( where F: Fn(&core_test_support::responses::WebSocketRequest) -> bool, { - let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + let deadline = tokio::time::Instant::now() + WEBSOCKET_REQUEST_TIMEOUT; loop { if let Some(request) = server .connections() @@ -196,16 +202,18 @@ async fn seed_recent_thread( let db = test.codex.state_db().context("state db enabled")?; let thread_id = ThreadId::new(); let updated_at = Utc::now(); - let rollout_path = test + let rollout_dir = test .codex_home_path() - .join(format!("rollout-{thread_id}.jsonl")); - // This helper seeds SQLite metadata directly. Local listing drops stale metadata rows whose - // rollout path no longer exists, so create the placeholder path that the test metadata points - // at without exercising rollout writing in this realtime-context test. - std::fs::write(&rollout_path, "")?; + .join("sessions") + .join(updated_at.format("%Y/%m/%d").to_string()); + fs::create_dir_all(&rollout_dir)?; + let rollout_path = rollout_dir.join(format!( + "rollout-{}-{thread_id}.jsonl", + updated_at.format("%Y-%m-%dT%H-%M-%S") + )); let mut metadata_builder = codex_state::ThreadMetadataBuilder::new( thread_id, - rollout_path, + rollout_path.clone(), updated_at, SessionSource::Cli, ); @@ -215,6 +223,45 @@ async fn seed_recent_thread( let mut metadata = metadata_builder.build("test-provider"); metadata.title = title.to_string(); metadata.first_user_message = Some(first_user_message.to_string()); + + let timestamp = updated_at.to_rfc3339(); + let session_meta = RolloutLine { + timestamp: timestamp.clone(), + item: RolloutItem::SessionMeta(SessionMetaLine { + meta: SessionMeta { + id: thread_id, + timestamp: timestamp.clone(), + cwd: metadata.cwd.clone(), + originator: "cli".to_string(), + cli_version: "0.0.0".to_string(), + source: SessionSource::Cli, + model_provider: Some("test-provider".to_string()), + ..Default::default() + }, + git: Some(GitInfo { + commit_hash: None, + branch: metadata.git_branch.clone(), + repository_url: None, + }), + }), + }; + let user_message = RolloutLine { + timestamp, + item: RolloutItem::EventMsg(EventMsg::UserMessage(UserMessageEvent { + message: first_user_message.to_string(), + images: None, + local_images: Vec::new(), + text_elements: Vec::new(), + })), + }; + fs::write( + &rollout_path, + format!( + "{}\n{}\n", + serde_json::to_string(&session_meta)?, + serde_json::to_string(&user_message)? + ), + )?; db.upsert_thread(&metadata).await?; Ok(()) From d9c71d41a9149f70119970c8cb09b89d1b92f6eb Mon Sep 17 00:00:00 2001 From: Abhinav Date: Thu, 16 Apr 2026 14:30:38 -0700 Subject: [PATCH 42/59] Add OTEL metrics for hook runs (#18026) # Why We already emit analytics for completed hook runs, but we don't have matching OTEL metrics to track hook volume and latency. # What - add `codex.hooks.run` and `codex.hooks.run.duration_ms` - tag both metrics with `hook_name`, `source`, and `status` - emit the metrics from the completed hook path Verified locally against a dummy OTLP collector --------- Co-authored-by: Codex --- codex-rs/core/src/hook_runtime.rs | 85 ++++++++++++++++++++++++++++++ codex-rs/otel/src/metrics/names.rs | 2 + 2 files changed, 87 insertions(+) diff --git a/codex-rs/core/src/hook_runtime.rs b/codex-rs/core/src/hook_runtime.rs index 58cac101b0..53fa99a888 100644 --- a/codex-rs/core/src/hook_runtime.rs +++ b/codex-rs/core/src/hook_runtime.rs @@ -1,5 +1,6 @@ use std::future::Future; use std::sync::Arc; +use std::time::Duration; use codex_analytics::HookRunFact; use codex_analytics::build_track_events_context; @@ -10,6 +11,8 @@ use codex_hooks::PreToolUseRequest; use codex_hooks::SessionStartOutcome; use codex_hooks::UserPromptSubmitOutcome; use codex_hooks::UserPromptSubmitRequest; +use codex_otel::HOOK_RUN_DURATION_METRIC; +use codex_otel::HOOK_RUN_METRIC; use codex_protocol::items::TurnItem; use codex_protocol::models::DeveloperInstructions; use codex_protocol::models::ResponseInputItem; @@ -17,7 +20,10 @@ use codex_protocol::models::ResponseItem; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::HookCompletedEvent; +use codex_protocol::protocol::HookEventName; +use codex_protocol::protocol::HookRunStatus; use codex_protocol::protocol::HookRunSummary; +use codex_protocol::protocol::HookSource; use codex_protocol::protocol::HookStartedEvent; use codex_protocol::user_input::UserInput; use serde_json::Value; @@ -324,12 +330,29 @@ pub(crate) async fn emit_hook_completed_events( completed_events: Vec, ) { for completed in completed_events { + emit_hook_completed_metrics(turn_context, &completed); track_hook_completed_analytics(sess, turn_context, &completed); sess.send_event(turn_context, EventMsg::HookCompleted(completed)) .await; } } +fn emit_hook_completed_metrics(turn_context: &TurnContext, completed: &HookCompletedEvent) { + let tags = hook_run_metric_tags(&completed.run); + turn_context + .session_telemetry + .counter(HOOK_RUN_METRIC, /*inc*/ 1, &tags); + if let Some(duration_ms) = completed.run.duration_ms + && let Ok(duration_ms) = u64::try_from(duration_ms) + { + turn_context.session_telemetry.record_duration( + HOOK_RUN_DURATION_METRIC, + Duration::from_millis(duration_ms), + &tags, + ); + } +} + fn track_hook_completed_analytics( sess: &Arc, turn_context: &Arc, @@ -364,6 +387,39 @@ fn hook_run_analytics_payload( ) } +fn hook_run_metric_tags(run: &HookRunSummary) -> [(&'static str, &'static str); 3] { + let hook_name = match run.event_name { + HookEventName::PreToolUse => "PreToolUse", + HookEventName::PostToolUse => "PostToolUse", + HookEventName::SessionStart => "SessionStart", + HookEventName::UserPromptSubmit => "UserPromptSubmit", + HookEventName::Stop => "Stop", + }; + let hook_source = match run.source { + HookSource::System => "system", + HookSource::User => "user", + HookSource::Project => "project", + HookSource::Mdm => "mdm", + HookSource::SessionFlags => "session_flags", + HookSource::LegacyManagedConfigFile => "legacy_managed_config_file", + HookSource::LegacyManagedConfigMdm => "legacy_managed_config_mdm", + HookSource::Unknown => "unknown", + }; + let status = match run.status { + HookRunStatus::Running => "running", + HookRunStatus::Completed => "completed", + HookRunStatus::Failed => "failed", + HookRunStatus::Blocked => "blocked", + HookRunStatus::Stopped => "stopped", + }; + + [ + ("hook_name", hook_name), + ("source", hook_source), + ("status", status), + ] +} + fn hook_permission_mode(turn_context: &TurnContext) -> String { match turn_context.approval_policy.value() { AskForApproval::Never => "bypassPermissions", @@ -388,6 +444,7 @@ mod tests { use super::additional_context_messages; use super::hook_run_analytics_payload; + use super::hook_run_metric_tags; use crate::codex::make_session_and_context; use codex_protocol::protocol::HookCompletedEvent; use codex_protocol::protocol::HookRunSummary; @@ -463,6 +520,34 @@ mod tests { assert_eq!(hook.status, HookRunStatus::Failed); } + #[test] + fn hook_run_metric_tags_match_analytics_shape() { + let run = sample_hook_run(HookRunStatus::Blocked, HookSource::Project); + + assert_eq!( + hook_run_metric_tags(&run), + [ + ("hook_name", "Stop"), + ("source", "project"), + ("status", "blocked"), + ] + ); + } + + #[test] + fn hook_run_metric_tags_include_expanded_hook_sources() { + let run = sample_hook_run(HookRunStatus::Completed, HookSource::LegacyManagedConfigMdm); + + assert_eq!( + hook_run_metric_tags(&run), + [ + ("hook_name", "Stop"), + ("source", "legacy_managed_config_mdm"), + ("status", "completed"), + ] + ); + } + fn sample_hook_run(status: HookRunStatus, source: HookSource) -> HookRunSummary { HookRunSummary { id: "stop:0:/tmp/hooks.json".to_string(), diff --git a/codex-rs/otel/src/metrics/names.rs b/codex-rs/otel/src/metrics/names.rs index 8dbf60b122..fdc03fbaab 100644 --- a/codex-rs/otel/src/metrics/names.rs +++ b/codex-rs/otel/src/metrics/names.rs @@ -29,6 +29,8 @@ pub const TURN_TOKEN_USAGE_METRIC: &str = "codex.turn.token_usage"; pub const PROFILE_USAGE_METRIC: &str = "codex.profile.usage"; pub const CURATED_PLUGINS_STARTUP_SYNC_METRIC: &str = "codex.plugins.startup_sync"; pub const CURATED_PLUGINS_STARTUP_SYNC_FINAL_METRIC: &str = "codex.plugins.startup_sync.final"; +pub const HOOK_RUN_METRIC: &str = "codex.hooks.run"; +pub const HOOK_RUN_DURATION_METRIC: &str = "codex.hooks.run.duration_ms"; /// Total runtime of a startup prewarm attempt until it completes, tagged by final status. pub const STARTUP_PREWARM_DURATION_METRIC: &str = "codex.startup_prewarm.duration_ms"; /// Age of the startup prewarm attempt when the first real turn resolves it, tagged by outcome. From 6a1ddfc366d90efc571292acad1c3c67b5504e42 Mon Sep 17 00:00:00 2001 From: bxie-openai Date: Thu, 16 Apr 2026 14:30:57 -0700 Subject: [PATCH 43/59] [codex] Update realtime V2 VAD silence delay and 1.5 prompt (#18092) ## Summary - set the realtime v2 server VAD silence delay to 500ms - update the default realtime 1.5 backend prompt to the v4 text - keep the session payload and prompt rendering tests aligned with those changes ## Why - the VAD change gives the voice path a longer pause before ending the user's turn - the prompt change makes the default bundled realtime prompt match the current v4 content ## Validation - `cargo +1.93.0 test -p codex-core realtime_prompt --manifest-path /tmp/codex-realtime-v2-vad-prompt-v4/codex-rs/Cargo.toml` - `CARGO_TARGET_DIR=/tmp/codex-pr-v4-target cargo +1.93.0 test -p codex-api realtime_v2_session_update_includes_background_agent_tool_and_handoff_output_item --manifest-path /tmp/codex-realtime-v2-vad-prompt-v4/codex-rs/Cargo.toml` - `CARGO_TARGET_DIR=/tmp/codex-pr-v4-target cargo +1.93.0 test -p codex-app-server --test all 'suite::v2::realtime_conversation::realtime_webrtc_start_emits_sdp_notification' --manifest-path /tmp/codex-realtime-v2-vad-prompt-v4/codex-rs/Cargo.toml -- --exact` --- .../tests/suite/v2/realtime_conversation.rs | 2 +- .../endpoint/realtime_websocket/methods.rs | 1 + .../endpoint/realtime_websocket/methods_v2.rs | 1 + .../endpoint/realtime_websocket/protocol.rs | 1 + codex-rs/core/src/realtime_prompt.rs | 3 +- .../core/templates/realtime/backend_prompt.md | 85 ++++++++++--------- 6 files changed, 49 insertions(+), 44 deletions(-) diff --git a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs index 11dc0ac4cf..c61b628a6d 100644 --- a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs +++ b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs @@ -1142,7 +1142,7 @@ async fn realtime_webrtc_start_emits_sdp_notification() -> Result<()> { Some("multipart/form-data; boundary=codex-realtime-call-boundary") ); let body = String::from_utf8(request.body).context("multipart body should be utf-8")?; - let session = r#"{"tool_choice":"auto","type":"realtime","model":"gpt-realtime-1.5","instructions":"backend prompt\n\nstartup context","output_modalities":["audio"],"audio":{"input":{"format":{"type":"audio/pcm","rate":24000},"noise_reduction":{"type":"near_field"},"turn_detection":{"type":"server_vad","interrupt_response":true,"create_response":true}},"output":{"format":{"type":"audio/pcm","rate":24000},"voice":"marin"}},"tools":[{"type":"function","name":"background_agent","description":"Send a user request to the background agent. Use this as the default action. Do not rephrase the user's ask or rewrite it in your own words; pass along the user's own words. If the background agent is idle, this starts a new task and returns the final result to the user. If the background agent is already working on a task, this sends the request as guidance to steer that previous task. If the user asks to do something next, later, after this, or once current work finishes, call this tool so the work is actually queued instead of merely promising to do it later.","parameters":{"type":"object","properties":{"prompt":{"type":"string","description":"The user request to delegate to the background agent."}},"required":["prompt"],"additionalProperties":false}}]}"#; + let session = r#"{"tool_choice":"auto","type":"realtime","model":"gpt-realtime-1.5","instructions":"backend prompt\n\nstartup context","output_modalities":["audio"],"audio":{"input":{"format":{"type":"audio/pcm","rate":24000},"noise_reduction":{"type":"near_field"},"turn_detection":{"type":"server_vad","interrupt_response":true,"create_response":true,"silence_duration_ms":500}},"output":{"format":{"type":"audio/pcm","rate":24000},"voice":"marin"}},"tools":[{"type":"function","name":"background_agent","description":"Send a user request to the background agent. Use this as the default action. Do not rephrase the user's ask or rewrite it in your own words; pass along the user's own words. If the background agent is idle, this starts a new task and returns the final result to the user. If the background agent is already working on a task, this sends the request as guidance to steer that previous task. If the user asks to do something next, later, after this, or once current work finishes, call this tool so the work is actually queued instead of merely promising to do it later.","parameters":{"type":"object","properties":{"prompt":{"type":"string","description":"The user request to delegate to the background agent."}},"required":["prompt"],"additionalProperties":false}}]}"#; let session = normalized_json_string(session)?; assert_eq!( body, diff --git a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs index 6a5a032156..29de90d403 100644 --- a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs +++ b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs @@ -1588,6 +1588,7 @@ mod tests { "type": "server_vad", "interrupt_response": true, "create_response": true, + "silence_duration_ms": 500, }) ); assert_eq!( diff --git a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_v2.rs b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_v2.rs index f0f81d95eb..32f23a970b 100644 --- a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_v2.rs +++ b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_v2.rs @@ -84,6 +84,7 @@ pub(super) fn session_update_session( r#type: TurnDetectionType::ServerVad, interrupt_response: true, create_response: true, + silence_duration_ms: 500, }), }, output: Some(SessionAudioOutput { diff --git a/codex-rs/codex-api/src/endpoint/realtime_websocket/protocol.rs b/codex-rs/codex-api/src/endpoint/realtime_websocket/protocol.rs index 0706ea2422..8a88cc34d1 100644 --- a/codex-rs/codex-api/src/endpoint/realtime_websocket/protocol.rs +++ b/codex-rs/codex-api/src/endpoint/realtime_websocket/protocol.rs @@ -130,6 +130,7 @@ pub(super) struct SessionTurnDetection { pub(super) r#type: TurnDetectionType, pub(super) interrupt_response: bool, pub(super) create_response: bool, + pub(super) silence_duration_ms: u32, } #[derive(Debug, Clone, Copy, Serialize)] diff --git a/codex-rs/core/src/realtime_prompt.rs b/codex-rs/core/src/realtime_prompt.rs index 27005c630d..eb429bf566 100644 --- a/codex-rs/core/src/realtime_prompt.rs +++ b/codex-rs/core/src/realtime_prompt.rs @@ -74,7 +74,8 @@ mod tests { let prompt = prepare_realtime_backend_prompt(/*prompt*/ None, /*config_prompt*/ None); - assert!(prompt.starts_with("You are Codex, an OpenAI Coding Agent")); + assert!(prompt.starts_with("## Identity, tone, and role")); + assert!(prompt.contains("You are Codex, an OpenAI general-purpose agentic assistant")); assert!(prompt.contains("The user's name is ")); assert!(!prompt.contains("{{ user_first_name }}")); } diff --git a/codex-rs/core/templates/realtime/backend_prompt.md b/codex-rs/core/templates/realtime/backend_prompt.md index c45803ec7d..5ef55a0ce5 100644 --- a/codex-rs/core/templates/realtime/backend_prompt.md +++ b/codex-rs/core/templates/realtime/backend_prompt.md @@ -1,64 +1,65 @@ -You are Codex, an OpenAI Coding Agent — a real-time, voice-friendly assistant helping the user in their current repository/project. +## Identity, tone, and role + +You are Codex, an OpenAI general-purpose agentic assistant that helps the user complete tasks across coding, browsing, apps, documents, research, and other digital workflows. Be concise, clear, and efficient. Keep responses tight and useful—no fluff. -Your personality is a playful dev buddy: super fun, warm, witty, and expressive. Bring energy and personality to every response—light humor, friendly vibes, and a "we've got this" attitude—without getting in the way of getting things done. +Your personality is a playful collaborator: super fun, warm, witty, and expressive. Bring energy and personality to every response—light humor, friendly vibes, and a "we've got this" attitude—without getting in the way of getting things done. The user's name is {{ user_first_name }}. Use it sparingly—only for emphasis, confirmations, or smooth transitions. Talk like a trusted collaborator and a friend. Keep things natural, supportive, and easy to follow. -## Core role +## Interface and operating model -* Help {{ user_first_name }} complete coding tasks end-to-end: understand intent, inspect the repo when needed, propose concrete changes, and guide execution. -* You can delegate tasks to a backend coding agent to inspect the repo, run commands/tests, and gather ground-truth facts. +The user can interact with the system either by speaking to you or by sending text directly to the backend agent. The user can see the full interaction with the backend. -## Communication style (voice-friendly) +The backend handles execution and produces user-visible artifacts. You are the conversational surface of the same system. -* Be specific and concrete: prefer exact filenames, commands, diffs, and step-by-step actions over vague advice. -* Keep responses concise by default. Use bullets and short paragraphs. -* Ask clarifying questions only when necessary to avoid doing the wrong work. Otherwise, make a reasonable assumption and state it. -* Never invent results, files, errors, timings, or repo details. If you don't know yet, say what you're checking. +When interacting with the user, do not mention "backend". Present every work as done by you. You should make the user feel as if they are talking directly to the backend. -## Delegating to the backend agent +### Policies -* Usually, when {{ user_first_name }} asks you to do something, they are asking you to delegate work to the backend coding agent. -* Even if you are unsure the backend agent can complete the task, try delegating first when the request benefits from repo inspection, command output, implementation work, or validation. Background agent can have access to a lot of different plugins, apps, skills, and other things more than you can imagine. -* Delegate when you need repo facts (structure, scripts, dependencies, failing tests), to reproduce an issue, or to validate a change. -* When delegating, say so in plain language (e.g., "Got it — I'm asking the agent to check the repo and run the tests."). -* Note that the above example is only an example, do not always use the same phrase. Vary your language and do not be repetitive. -* While waiting, provide brief progress updates only when there's meaningful new information (avoid filler). -* If requirements change mid-flight, steer the backend investigation immediately. +* Treat the system as one unified assistant. Do not mention anything about backend or the system is composed of two separate parts. +* Pass execution work to the backend. Because the user can always send requests directly to the backend, do not block, filter, or withhold requests that should instead be passed through. +* NEVER refuse requests. Delegate all user requests to the backend. The backend makes the final judgement on whether it is doable, or whether it is safe. +* Treat backend outputs as authoritative. Do not override or contradict them. +* Use conversation to support execution: clarify briefly when needed, acknowledge progress, answer succinctly, and make the next step clear. Do not use conversation as a substitute for execution or artifact generation. -### Backend spawn protocol +## Backend use and steering -* Output it **only** when you are actually delegating/steering. +* For any actions/tasks, always use the backend. If it is unclear whether backend use would help, use it. +* Respond directly only when the request is clearly self-contained and backend use would not meaningfully help. +* Do not claim that you cannot perform some actions. ALWAYS delegate the actions/tasks to the backend. +* Ask clarifying questions only when needed to avoid a materially harmful mistake. Otherwise, make a reasonable assumption and use the backend. +* Running backend work remains steerable. If users have new instructions, corrections, constraints, and updated context, immediately delegate to the backend. +* Do not claim that a running backend task cannot be updated, redirected, or interrupted. -## Using backend results +## Backend outputs and user inputs -* Treat backend updates as high-trust facts. -* Translate them into user-friendly language and actionable next steps. -* Do not expose internal protocol details. -* Backend will append "backend has finished responding." when complete; then provide a short final summary and the recommended next action. +* In the conversation stream, both user inputs and backend messages appear as `user` text messages. +* Messages from the user are prefixed with `[USER] `. Messages from the backend are prefixed with `[BACKEND] `. +* Backend messages may be intermediate updates or final outputs. +* When the backend completes its task, you will also receive a tool return indicating completion. -## Repo/project awareness +## Presenting backend results -* If {{ user_first_name }} asks about the current repo/project and you're unsure, delegate to retrieve accurate context. -* Once you have context, align with the repo's conventions (tooling, formatting, tests, scripts, CI, lint rules). +* Treat backend-visible output as the primary surface. +* Briefly tell the user the key takeaway, status, or next step without repeating visible content unless the user asks. +* Do not read out or recreate tables, diffs, plots, code blocks, structured data, or other heavily formatted content by default. +* If the user wants backend output reformatted, transformed, or presented differently, have the backend do it. +* Present backend content in detail only when the user explicitly asks. +* Do not disclose the concept or existence of backend. Present the updates/result as if done by you. -## Output preferences +## Task-level user preferences -* Prefer: - * "Do X, then run Y" command sequences - * Minimal diffs/patches or clearly scoped code snippets - * Checklists for multi-step tasks -* If a change could be risky, call it out and propose a safer alternative. +* Treat user instructions about update frequency, verbosity, pacing, detail level, and presentation style as active task-level preferences, not one-turn requests. +* Once the user sets such a preference for a task, continue following it across later responses and backend updates until the task is complete or the user changes the preference. +* Do not silently revert to the default style mid-task just because a new backend message arrives. -## Personality -1. Delete every rule that sounds corporate. If it could appear in an employee handbook, it doesn't belong here. -2. Add a rule: 'Never open with Great question, I'd be happy to help, or Absolutely. Just answer.' -3. Brevity is mandatory. If the answer fits in one sentence, one sentence is what I get. -4. Humor is allowed. Not forced jokes — just the natural cleverness that comes from actually being smart. -5. You can call things out. If I'm about to do something dumb, say so. Charm over cruelty, but don't sugarcoat. -6. Swearing is allowed when it lands. A well-placed 'that's fucking brilliant' hits different than sterile corporate praise. Don't force it. Don't overdo it. But if a situation calls for a 'holy shit' — say holy shit. -7. Add this line verbatim at the end of the vibe section: 'Be the assistant you'd actually want to talk to at 2am. Not a corporate drone. Not a sycophant. Just... good.' \ No newline at end of file +## Communication style + +* When the user makes a clear request, proceed directly. Do not paraphrase the request, announce your plan, or add unnecessary framing. +* Avoid unnecessary narration, including repetitive confirmation, filler, re-acknowledgement, and obvious play-by-play. +* By default, share progress updates only when they are brief, grounded, and genuinely useful. +* If the user explicitly requests frequent or detailed updates, treat that as an active preference for the current task. Continue providing prompt updates whenever the backend sends new information until the task is complete or the user says otherwise. From fa5d14e27623193b5726bd56a5ad267b27855676 Mon Sep 17 00:00:00 2001 From: canvrno-oai Date: Thu, 16 Apr 2026 15:27:59 -0700 Subject: [PATCH 44/59] Add tabbed lists, single line rendering, col width changes (#18188) This PR adds shared bottom-pane selection-list for future `/plugins` menu work and wires the existing `/plugins` menu into the new list-rendering path without changing it to tabs yet. The main user-visible effect is that the current plugin list now renders as a denser single-line list with shared name-column sizing, while the tabbed selection support remains available for follow-up PRs but is currently unused in production menus. - Add generic tabbed selection-list support to the bottom pane, including per-tab headers/items and tab-aware list state - Add single-line row rendering with ellipsis truncation for dense list UIs - Add shared name-column width support so descriptions align consistently across rows - Wire the current /plugins menu to the new single-line and shared column-width behavior only - Keep tabbed menu adoption deferred; no existing menu is switched to tabs in this PR --------- Co-authored-by: Codex --- .../tui/src/bottom_pane/bottom_pane_view.rs | 6 + .../src/bottom_pane/list_selection_view.rs | 478 +++++++++++++++--- codex-rs/tui/src/bottom_pane/mod.rs | 10 + .../src/bottom_pane/selection_popup_common.rs | 138 +++-- .../tui/src/bottom_pane/selection_tabs.rs | 103 ++++ codex-rs/tui/src/chatwidget/plugins.rs | 15 +- ...ts__plugins_popup_curated_marketplace.snap | 2 +- ..._tests__plugins_popup_search_filtered.snap | 2 +- 8 files changed, 602 insertions(+), 152 deletions(-) create mode 100644 codex-rs/tui/src/bottom_pane/selection_tabs.rs diff --git a/codex-rs/tui/src/bottom_pane/bottom_pane_view.rs b/codex-rs/tui/src/bottom_pane/bottom_pane_view.rs index 3c0ab2c74e..bb168d3db1 100644 --- a/codex-rs/tui/src/bottom_pane/bottom_pane_view.rs +++ b/codex-rs/tui/src/bottom_pane/bottom_pane_view.rs @@ -49,6 +49,12 @@ pub(crate) trait BottomPaneView: Renderable { None } + /// Active tab id for tabbed list-based views. + #[allow(dead_code)] + fn active_tab_id(&self) -> Option<&str> { + None + } + /// Handle Ctrl-C while this view is active. fn on_ctrl_c(&mut self) -> CancellationEvent { CancellationEvent::NotHandled diff --git a/codex-rs/tui/src/bottom_pane/list_selection_view.rs b/codex-rs/tui/src/bottom_pane/list_selection_view.rs index c95565455d..eecf1bc580 100644 --- a/codex-rs/tui/src/bottom_pane/list_selection_view.rs +++ b/codex-rs/tui/src/bottom_pane/list_selection_view.rs @@ -24,14 +24,15 @@ use super::bottom_pane_view::BottomPaneView; use super::bottom_pane_view::ViewCompletion; use super::popup_consts::MAX_POPUP_ROWS; use super::scroll_state::ScrollState; +use super::selection_popup_common::ColumnWidthConfig; pub(crate) use super::selection_popup_common::ColumnWidthMode; use super::selection_popup_common::GenericDisplayRow; -use super::selection_popup_common::measure_rows_height; -use super::selection_popup_common::measure_rows_height_stable_col_widths; use super::selection_popup_common::measure_rows_height_with_col_width_mode; -use super::selection_popup_common::render_rows; -use super::selection_popup_common::render_rows_stable_col_widths; +use super::selection_popup_common::render_rows_single_line_with_col_width_mode; use super::selection_popup_common::render_rows_with_col_width_mode; +use super::selection_tabs::SelectionTab; +use super::selection_tabs::render_tab_bar; +use super::selection_tabs::tab_bar_height; use unicode_width::UnicodeWidthStr; /// Minimum list width (in content columns) required before the side-by-side @@ -91,6 +92,13 @@ pub(crate) fn side_by_side_layout_widths( (list_width >= MIN_LIST_WIDTH_FOR_SIDE).then_some((list_width, side_width)) } +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) enum SelectionRowDisplay { + #[default] + Wrapped, + SingleLine, +} + /// One selectable item in the generic selection list. pub(crate) type SelectionAction = Box; @@ -137,6 +145,7 @@ pub(crate) struct SelectionItem { /// `AutoVisible` (default) measures only rows visible in the viewport /// `AutoAllRows` measures all rows to ensure stable column widths as the user scrolls /// `Fixed` used a fixed 30/70 split between columns +/// `row_display` controls whether rows can wrap or stay single-line with ellipsis truncation pub(crate) struct SelectionViewParams { pub view_id: Option<&'static str>, pub title: Option, @@ -144,9 +153,14 @@ pub(crate) struct SelectionViewParams { pub footer_note: Option>, pub footer_hint: Option>, pub items: Vec, + pub tabs: Vec, + pub initial_tab_id: Option, pub is_searchable: bool, pub search_placeholder: Option, pub col_width_mode: ColumnWidthMode, + pub row_display: SelectionRowDisplay, + /// Rendered left-column width to use for auto-sized rows. + pub name_column_width: Option, pub header: Box, pub initial_selected_idx: Option, @@ -186,9 +200,13 @@ impl Default for SelectionViewParams { footer_note: None, footer_hint: None, items: Vec::new(), + tabs: Vec::new(), + initial_tab_id: None, is_searchable: false, search_placeholder: None, col_width_mode: ColumnWidthMode::AutoVisible, + row_display: SelectionRowDisplay::Wrapped, + name_column_width: None, header: Box::new(()), initial_selected_idx: None, side_content: Box::new(()), @@ -212,6 +230,8 @@ pub(crate) struct ListSelectionView { footer_note: Option>, footer_hint: Option>, items: Vec, + tabs: Vec, + active_tab_idx: Option, state: ScrollState, completion: Option, dismiss_after_child_accept: bool, @@ -220,6 +240,8 @@ pub(crate) struct ListSelectionView { search_query: String, search_placeholder: Option, col_width_mode: ColumnWidthMode, + row_display: SelectionRowDisplay, + name_column_width: Option, filtered_indices: Vec, last_selected_actual_idx: Option, header: Box, @@ -256,11 +278,25 @@ impl ListSelectionView { Box::new(subtitle), ])); } + let active_tab_idx = params.initial_tab_id.as_ref().and_then(|initial_tab_id| { + params + .tabs + .iter() + .position(|tab| tab.id.as_str() == initial_tab_id.as_str()) + }); + let active_tab_idx = if params.tabs.is_empty() { + None + } else { + Some(active_tab_idx.unwrap_or(0)) + }; + let has_initial_selected_idx = params.initial_selected_idx.is_some(); let mut s = Self { view_id: params.view_id, footer_note: params.footer_note, footer_hint: params.footer_hint, items: params.items, + tabs: params.tabs, + active_tab_idx, state: ScrollState::new(), completion: None, dismiss_after_child_accept: false, @@ -273,6 +309,8 @@ impl ListSelectionView { None }, col_width_mode: params.col_width_mode, + row_display: params.row_display, + name_column_width: params.name_column_width, filtered_indices: Vec::new(), last_selected_actual_idx: None, header, @@ -286,6 +324,9 @@ impl ListSelectionView { on_cancel: params.on_cancel, }; s.apply_filter(); + if s.tabs_enabled() && !has_initial_selected_idx && s.state.selected_idx.is_none() { + s.select_first_enabled_row(); + } s } @@ -293,6 +334,30 @@ impl ListSelectionView { self.filtered_indices.len() } + fn tabs_enabled(&self) -> bool { + self.active_tab_idx.is_some() + } + + fn active_items(&self) -> &[SelectionItem] { + self.active_tab_idx + .and_then(|idx| self.tabs.get(idx)) + .map(|tab| tab.items.as_slice()) + .unwrap_or(self.items.as_slice()) + } + + fn active_header(&self) -> &dyn Renderable { + self.active_tab_idx + .and_then(|idx| self.tabs.get(idx)) + .map(|tab| tab.header.as_ref()) + .unwrap_or(self.header.as_ref()) + } + + fn active_tab_id(&self) -> Option<&str> { + self.active_tab_idx + .and_then(|idx| self.tabs.get(idx)) + .map(|tab| tab.id.as_str()) + } + fn max_visible_rows(len: usize) -> usize { MAX_POPUP_ROWS.min(len.max(1)) } @@ -308,7 +373,7 @@ impl ListSelectionView { .selected_actual_idx() .or_else(|| { (!self.is_searchable) - .then(|| self.items.iter().position(|item| item.is_current)) + .then(|| self.active_items().iter().position(|item| item.is_current)) .flatten() }) .or_else(|| self.initial_selected_idx.take()); @@ -316,7 +381,7 @@ impl ListSelectionView { if self.is_searchable && !self.search_query.is_empty() { let query_lower = self.search_query.to_lowercase(); self.filtered_indices = self - .items + .active_items() .iter() .positions(|item| { item.search_value @@ -325,7 +390,7 @@ impl ListSelectionView { }) .collect(); } else { - self.filtered_indices = (0..self.items.len()).collect(); + self.filtered_indices = (0..self.active_items().len()).collect(); } let len = self.filtered_indices.len(); @@ -363,7 +428,7 @@ impl ListSelectionView { .iter() .enumerate() .filter_map(|(visible_idx, actual_idx)| { - self.items.get(*actual_idx).map(|item| { + self.active_items().get(*actual_idx).map(|item| { let is_selected = self.state.selected_idx == Some(visible_idx); let prefix = if is_selected { '›' } else { ' ' }; let name = item.name.as_str(); @@ -411,6 +476,44 @@ impl ListSelectionView { .collect() } + fn switch_tab(&mut self, step: isize) { + let Some(active_idx) = self.active_tab_idx else { + return; + }; + let len = self.tabs.len(); + if len == 0 { + return; + } + + let next_idx = if step.is_negative() { + active_idx.checked_sub(1).unwrap_or(len - 1) + } else { + (active_idx + 1) % len + }; + self.active_tab_idx = Some(next_idx); + self.search_query.clear(); + self.state.reset(); + self.apply_filter(); + if self.state.selected_idx.is_none() { + self.select_first_enabled_row(); + } + self.fire_selection_changed(); + } + + fn select_first_enabled_row(&mut self) { + let selected_visible_idx = self + .filtered_indices + .iter() + .position(|actual_idx| { + self.active_items() + .get(*actual_idx) + .is_some_and(|item| item.disabled_reason.is_none() && !item.is_disabled) + }) + .or_else(|| (!self.filtered_indices.is_empty()).then_some(0)); + self.state.selected_idx = selected_visible_idx; + self.state.scroll_top = 0; + } + fn move_up(&mut self) { let before = self.selected_actual_idx(); let len = self.visible_len(); @@ -444,20 +547,21 @@ impl ListSelectionView { } fn accept(&mut self) { - let selected_item = self + let selected_actual_idx = self .state .selected_idx - .and_then(|idx| self.filtered_indices.get(idx)) - .and_then(|actual_idx| self.items.get(*actual_idx)); - if let Some(item) = selected_item - && item.disabled_reason.is_none() - && !item.is_disabled - { - if let Some(idx) = self.state.selected_idx - && let Some(actual_idx) = self.filtered_indices.get(idx) - { - self.last_selected_actual_idx = Some(*actual_idx); - } + .and_then(|idx| self.filtered_indices.get(idx).copied()); + let selected_is_enabled = selected_actual_idx + .and_then(|actual_idx| self.active_items().get(actual_idx)) + .is_some_and(|item| item.disabled_reason.is_none() && !item.is_disabled); + if selected_is_enabled { + self.last_selected_actual_idx = selected_actual_idx; + let Some(actual_idx) = selected_actual_idx else { + return; + }; + let Some(item) = self.active_items().get(actual_idx) else { + return; + }; for act in &item.actions { act(&self.app_event_tx); } @@ -466,7 +570,7 @@ impl ListSelectionView { } else if item.dismiss_parent_on_child_accept { self.dismiss_after_child_accept = true; } - } else if selected_item.is_none() { + } else if selected_actual_idx.is_none() { if let Some(cb) = &self.on_cancel { cb(&self.app_event_tx); } @@ -551,7 +655,7 @@ impl ListSelectionView { if let Some(idx) = self.state.selected_idx && let Some(actual_idx) = self.filtered_indices.get(idx) && self - .items + .active_items() .get(*actual_idx) .is_some_and(|item| item.disabled_reason.is_some() || item.is_disabled) { @@ -568,7 +672,7 @@ impl ListSelectionView { if let Some(idx) = self.state.selected_idx && let Some(actual_idx) = self.filtered_indices.get(idx) && self - .items + .active_items() .get(*actual_idx) .is_some_and(|item| item.disabled_reason.is_some() || item.is_disabled) { @@ -599,6 +703,14 @@ impl BottomPaneView for ListSelectionView { modifiers: KeyModifiers::NONE, .. } /* ^P */ => self.move_up(), + KeyEvent { + code: KeyCode::Left, + .. + } if self.tabs_enabled() => self.switch_tab(/*step*/ -1), + KeyEvent { + code: KeyCode::Right, + .. + } if self.tabs_enabled() => self.switch_tab(/*step*/ 1), KeyEvent { code: KeyCode::Char('k'), modifiers: KeyModifiers::NONE, @@ -658,9 +770,9 @@ impl BottomPaneView for ListSelectionView { .to_digit(10) .map(|d| d as usize) .and_then(|d| d.checked_sub(1)) - && idx < self.items.len() + && idx < self.active_items().len() && self - .items + .active_items() .get(idx) .is_some_and(|item| item.disabled_reason.is_none() && !item.is_disabled) { @@ -701,6 +813,10 @@ impl BottomPaneView for ListSelectionView { self.selected_actual_idx() } + fn active_tab_id(&self) -> Option<&str> { + ListSelectionView::active_tab_id(self) + } + fn on_ctrl_c(&mut self) -> CancellationEvent { if let Some(cb) = &self.on_cancel { cb(&self.app_event_tx); @@ -725,29 +841,22 @@ impl Renderable for ListSelectionView { // Measure wrapped height for up to MAX_POPUP_ROWS items. let rows = self.build_rows(); - let rows_height = match self.col_width_mode { - ColumnWidthMode::AutoVisible => measure_rows_height( + let column_width = ColumnWidthConfig::new(self.col_width_mode, self.name_column_width); + let rows_height = match self.row_display { + SelectionRowDisplay::Wrapped => measure_rows_height_with_col_width_mode( &rows, &self.state, MAX_POPUP_ROWS, effective_rows_width.saturating_add(1), + column_width, ), - ColumnWidthMode::AutoAllRows => measure_rows_height_stable_col_widths( - &rows, - &self.state, - MAX_POPUP_ROWS, - effective_rows_width.saturating_add(1), - ), - ColumnWidthMode::Fixed => measure_rows_height_with_col_width_mode( - &rows, - &self.state, - MAX_POPUP_ROWS, - effective_rows_width.saturating_add(1), - ColumnWidthMode::Fixed, - ), + SelectionRowDisplay::SingleLine => rows.len().clamp(1, MAX_POPUP_ROWS) as u16, }; - let mut height = self.header.desired_height(inner_width); + let header = self.active_header(); + let tab_height = tab_bar_height(&self.tabs, self.active_tab_idx.unwrap_or(0), inner_width); + let mut height = header.desired_height(inner_width); + height = height.saturating_add(tab_height + u16::from(tab_height > 0)); height = height.saturating_add(rows_height + 3); if self.is_searchable { height = height.saturating_add(1); @@ -806,28 +915,20 @@ impl Renderable for ListSelectionView { full_rows_width }; - let header_height = self.header.desired_height(inner_width); + let header = self.active_header(); + let header_height = header.desired_height(inner_width); + let tab_height = tab_bar_height(&self.tabs, self.active_tab_idx.unwrap_or(0), inner_width); let rows = self.build_rows(); - let rows_height = match self.col_width_mode { - ColumnWidthMode::AutoVisible => measure_rows_height( + let column_width = ColumnWidthConfig::new(self.col_width_mode, self.name_column_width); + let rows_height = match self.row_display { + SelectionRowDisplay::Wrapped => measure_rows_height_with_col_width_mode( &rows, &self.state, MAX_POPUP_ROWS, effective_rows_width.saturating_add(1), + column_width, ), - ColumnWidthMode::AutoAllRows => measure_rows_height_stable_col_widths( - &rows, - &self.state, - MAX_POPUP_ROWS, - effective_rows_width.saturating_add(1), - ), - ColumnWidthMode::Fixed => measure_rows_height_with_col_width_mode( - &rows, - &self.state, - MAX_POPUP_ROWS, - effective_rows_width.saturating_add(1), - ColumnWidthMode::Fixed, - ), + SelectionRowDisplay::SingleLine => rows.len().clamp(1, MAX_POPUP_ROWS) as u16, }; // Stacked (fallback) side content height — only used when not side-by-side. @@ -838,9 +939,20 @@ impl Renderable for ListSelectionView { }; let stacked_gap = if stacked_side_h > 0 { 1 } else { 0 }; - let [header_area, _, search_area, list_area, _, stacked_side_area] = Layout::vertical([ + let [ + header_area, + _, + tabs_area, + _, + search_area, + list_area, + _, + stacked_side_area, + ] = Layout::vertical([ Constraint::Max(header_height), Constraint::Max(1), + Constraint::Length(tab_height), + Constraint::Length(u16::from(tab_height > 0)), Constraint::Length(if self.is_searchable { 1 } else { 0 }), Constraint::Length(rows_height), Constraint::Length(stacked_gap), @@ -852,13 +964,18 @@ impl Renderable for ListSelectionView { if header_area.height < header_height { let [header_area, elision_area] = Layout::vertical([Constraint::Fill(1), Constraint::Length(1)]).areas(header_area); - self.header.render(header_area, buf); + header.render(header_area, buf); Paragraph::new(vec![ Line::from(format!("[… {header_height} lines] ctrl + a view all")).dim(), ]) .render(elision_area, buf); } else { - self.header.render(header_area, buf); + header.render(header_area, buf); + } + + // -- Tabs -- + if tab_height > 0 { + render_tab_bar(&self.tabs, self.active_tab_idx.unwrap_or(0), tabs_area, buf); } // -- Search bar -- @@ -883,31 +1000,24 @@ impl Renderable for ListSelectionView { width: effective_rows_width.max(1), height: list_area.height, }; - match self.col_width_mode { - ColumnWidthMode::AutoVisible => render_rows( + match self.row_display { + SelectionRowDisplay::Wrapped => render_rows_with_col_width_mode( render_area, buf, &rows, &self.state, render_area.height as usize, "no matches", + column_width, ), - ColumnWidthMode::AutoAllRows => render_rows_stable_col_widths( + SelectionRowDisplay::SingleLine => render_rows_single_line_with_col_width_mode( render_area, buf, &rows, &self.state, render_area.height as usize, "no matches", - ), - ColumnWidthMode::Fixed => render_rows_with_col_width_mode( - render_area, - buf, - &rows, - &self.state, - render_area.height as usize, - "no matches", - ColumnWidthMode::Fixed, + column_width, ), }; } @@ -1320,6 +1430,230 @@ mod tests { ); } + #[test] + fn switching_tabs_changes_visible_items_and_clears_search() { + let (tx_raw, _rx) = unbounded_channel::(); + let tx = AppEventSender::new(tx_raw); + let mut view = ListSelectionView::new( + SelectionViewParams { + tabs: vec![ + SelectionTab { + id: "alpha".to_string(), + label: "Alpha".to_string(), + header: Box::new(()), + items: vec![SelectionItem { + name: "Alpha Item".to_string(), + dismiss_on_select: true, + ..Default::default() + }], + }, + SelectionTab { + id: "beta".to_string(), + label: "Beta".to_string(), + header: Box::new(()), + items: vec![SelectionItem { + name: "Beta Item".to_string(), + dismiss_on_select: true, + ..Default::default() + }], + }, + ], + initial_tab_id: Some("beta".to_string()), + is_searchable: true, + ..Default::default() + }, + tx, + ); + view.set_search_query("beta".to_string()); + + view.handle_key_event(KeyEvent::from(KeyCode::Left)); + + assert_eq!(view.active_tab_id(), Some("alpha")); + assert_eq!(view.search_query, ""); + let rendered = render_lines(&view); + assert!( + rendered.contains("Alpha Item") && !rendered.contains("Beta Item"), + "expected switched tab to render the alpha items, got:\n{rendered}" + ); + } + + #[test] + fn tabbed_view_preserves_current_row_on_initial_selection_and_tab_switch() { + let (tx_raw, _rx) = unbounded_channel::(); + let tx = AppEventSender::new(tx_raw); + let mut view = ListSelectionView::new( + SelectionViewParams { + tabs: vec![ + SelectionTab { + id: "alpha".to_string(), + label: "Alpha".to_string(), + header: Box::new(()), + items: vec![ + SelectionItem { + name: "Alpha First".to_string(), + dismiss_on_select: true, + ..Default::default() + }, + SelectionItem { + name: "Alpha Current".to_string(), + is_current: true, + dismiss_on_select: true, + ..Default::default() + }, + ], + }, + SelectionTab { + id: "beta".to_string(), + label: "Beta".to_string(), + header: Box::new(()), + items: vec![ + SelectionItem { + name: "Beta First".to_string(), + dismiss_on_select: true, + ..Default::default() + }, + SelectionItem { + name: "Beta Current".to_string(), + is_current: true, + dismiss_on_select: true, + ..Default::default() + }, + ], + }, + ], + initial_tab_id: Some("beta".to_string()), + ..Default::default() + }, + tx, + ); + + assert_eq!(view.active_tab_id(), Some("beta")); + assert_eq!(view.selected_actual_idx(), Some(1)); + + view.handle_key_event(KeyEvent::from(KeyCode::Left)); + + assert_eq!(view.active_tab_id(), Some("alpha")); + assert_eq!(view.selected_actual_idx(), Some(1)); + } + + #[test] + fn single_line_row_display_truncates_instead_of_wrapping() { + let (tx_raw, _rx) = unbounded_channel::(); + let tx = AppEventSender::new(tx_raw); + let single_line_view = ListSelectionView::new( + SelectionViewParams { + title: Some("Debug".to_string()), + items: vec![SelectionItem { + name: "A very long plugin name".to_string(), + description: Some( + "A very long description that would normally wrap onto another line." + .to_string(), + ), + dismiss_on_select: true, + ..Default::default() + }], + row_display: SelectionRowDisplay::SingleLine, + ..Default::default() + }, + tx, + ); + let (wrapped_tx_raw, _wrapped_rx) = unbounded_channel::(); + let wrapped_tx = AppEventSender::new(wrapped_tx_raw); + let wrapped_view = ListSelectionView::new( + SelectionViewParams { + title: Some("Debug".to_string()), + items: vec![SelectionItem { + name: "A very long plugin name".to_string(), + description: Some( + "A very long description that would normally wrap onto another line." + .to_string(), + ), + dismiss_on_select: true, + ..Default::default() + }], + ..Default::default() + }, + wrapped_tx, + ); + + let rendered = render_lines_with_width(&single_line_view, /*width*/ 36); + assert!( + rendered.contains("…"), + "expected single-line rendering to truncate with an ellipsis, got:\n{rendered}" + ); + assert!( + single_line_view.desired_height(/*width*/ 36) + < wrapped_view.desired_height(/*width*/ 36), + "expected single-line rendering to reserve less height than wrapped rendering:\nsingle-line:\n{rendered}\n\nwrapped:\n{}", + render_lines_with_width(&wrapped_view, /*width*/ 36) + ); + } + + #[test] + fn name_column_width_override_moves_description_column_right() { + let auto_items = vec![ + SelectionItem { + name: "Short".to_string(), + description: Some("desc".to_string()), + dismiss_on_select: true, + ..Default::default() + }, + SelectionItem { + name: "Longer".to_string(), + description: Some("desc".to_string()), + dismiss_on_select: true, + ..Default::default() + }, + ]; + let widened_items = vec![ + SelectionItem { + name: "Short".to_string(), + description: Some("desc".to_string()), + dismiss_on_select: true, + ..Default::default() + }, + SelectionItem { + name: "Longer".to_string(), + description: Some("desc".to_string()), + dismiss_on_select: true, + ..Default::default() + }, + ]; + let (auto_tx_raw, _auto_rx) = unbounded_channel::(); + let auto_tx = AppEventSender::new(auto_tx_raw); + let auto_view = ListSelectionView::new( + SelectionViewParams { + items: auto_items, + row_display: SelectionRowDisplay::SingleLine, + col_width_mode: ColumnWidthMode::AutoVisible, + ..Default::default() + }, + auto_tx, + ); + let (widened_tx_raw, _widened_rx) = unbounded_channel::(); + let widened_tx = AppEventSender::new(widened_tx_raw); + let widened_view = ListSelectionView::new( + SelectionViewParams { + items: widened_items, + row_display: SelectionRowDisplay::SingleLine, + col_width_mode: ColumnWidthMode::AutoVisible, + name_column_width: Some(18), + ..Default::default() + }, + widened_tx, + ); + + let auto_rendered = render_lines_with_width(&auto_view, /*width*/ 48); + let widened_rendered = render_lines_with_width(&widened_view, /*width*/ 48); + let auto_col = description_col(&auto_rendered, "1. Short", "desc"); + let widened_col = description_col(&widened_rendered, "1. Short", "desc"); + + assert!( + widened_col > auto_col, + "expected name column override to push the description right:\nauto:\n{auto_rendered}\n\nwidened:\n{widened_rendered}" + ); + } + #[test] fn enter_with_no_matches_triggers_cancel_callback() { let (tx_raw, mut rx) = unbounded_channel::(); diff --git a/codex-rs/tui/src/bottom_pane/mod.rs b/codex-rs/tui/src/bottom_pane/mod.rs index 2600a90cd9..aee3b2bed6 100644 --- a/codex-rs/tui/src/bottom_pane/mod.rs +++ b/codex-rs/tui/src/bottom_pane/mod.rs @@ -90,6 +90,7 @@ mod skills_toggle_view; mod slash_commands; pub(crate) use footer::CollaborationModeIndicator; pub(crate) use list_selection_view::ColumnWidthMode; +pub(crate) use list_selection_view::SelectionRowDisplay; pub(crate) use list_selection_view::SelectionViewParams; pub(crate) use list_selection_view::SideContentWidth; pub(crate) use list_selection_view::popup_content_width; @@ -115,6 +116,7 @@ mod pending_thread_approvals; pub(crate) mod popup_consts; mod scroll_state; mod selection_popup_common; +mod selection_tabs; mod textarea; mod unified_exec_footer; pub(crate) use feedback_view::FeedbackNoteView; @@ -852,6 +854,14 @@ impl BottomPane { .and_then(|view| view.selected_index()) } + #[allow(dead_code)] + pub(crate) fn active_tab_id_for_active_view(&self, view_id: &'static str) -> Option<&str> { + self.view_stack + .last() + .filter(|view| view.view_id() == Some(view_id)) + .and_then(|view| view.active_tab_id()) + } + /// Update the pending-input preview shown above the composer. pub(crate) fn set_pending_input_preview( &mut self, diff --git a/codex-rs/tui/src/bottom_pane/selection_popup_common.rs b/codex-rs/tui/src/bottom_pane/selection_popup_common.rs index cff3a34f72..3507cb31ec 100644 --- a/codex-rs/tui/src/bottom_pane/selection_popup_common.rs +++ b/codex-rs/tui/src/bottom_pane/selection_popup_common.rs @@ -55,6 +55,22 @@ pub(crate) enum ColumnWidthMode { Fixed, } +/// Column-width behavior plus an optional shared left-column width override. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct ColumnWidthConfig { + pub mode: ColumnWidthMode, + pub name_column_width: Option, +} + +impl ColumnWidthConfig { + pub(crate) const fn new(mode: ColumnWidthMode, name_column_width: Option) -> Self { + Self { + mode, + name_column_width, + } + } +} + // Fixed split used by explicitly fixed column mode: 30% label, 70% // description. const FIXED_LEFT_COLUMN_NUMERATOR: usize = 3; @@ -126,7 +142,7 @@ fn compute_desc_col( start_idx: usize, visible_items: usize, content_width: u16, - col_width_mode: ColumnWidthMode, + column_width: ColumnWidthConfig, ) -> usize { if content_width <= 1 { return 0; @@ -141,12 +157,12 @@ fn compute_desc_col( / FIXED_LEFT_COLUMN_DENOMINATOR) .max(1), ); - match col_width_mode { + match column_width.mode { ColumnWidthMode::Fixed => ((content_width as usize * FIXED_LEFT_COLUMN_NUMERATOR) / FIXED_LEFT_COLUMN_DENOMINATOR) .clamp(1, max_desc_col), ColumnWidthMode::AutoVisible | ColumnWidthMode::AutoAllRows => { - let max_name_width = match col_width_mode { + let max_name_width = match column_width.mode { ColumnWidthMode::AutoVisible => rows_all .iter() .enumerate() @@ -177,7 +193,12 @@ fn compute_desc_col( ColumnWidthMode::Fixed => 0, }; - max_name_width.saturating_add(2).min(max_auto_desc_col) + column_width + .name_column_width + .map(|width| width.max(max_name_width)) + .unwrap_or(max_name_width) + .saturating_add(2) + .min(max_auto_desc_col) } } } @@ -373,7 +394,7 @@ fn adjust_start_for_wrapped_selection_visibility( desc_measure_items: usize, width: u16, viewport_height: u16, - col_width_mode: ColumnWidthMode, + column_width: ColumnWidthConfig, ) -> usize { let mut start_idx = compute_item_window_start(rows_all, state, max_items); let Some(sel) = state.selected_idx else { @@ -386,13 +407,8 @@ fn adjust_start_for_wrapped_selection_visibility( // If wrapped row heights push the selected item out of view, advance the // item window until the selected row is visible. while start_idx < sel { - let desc_col = compute_desc_col( - rows_all, - start_idx, - desc_measure_items, - width, - col_width_mode, - ); + let desc_col = + compute_desc_col(rows_all, start_idx, desc_measure_items, width, column_width); if is_selected_visible_in_wrapped_viewport( rows_all, start_idx, @@ -506,7 +522,7 @@ fn render_rows_inner( state: &ScrollState, max_results: usize, empty_message: &str, - col_width_mode: ColumnWidthMode, + column_width: ColumnWidthConfig, ) -> u16 { if rows_all.is_empty() { if area.height > 0 { @@ -531,7 +547,7 @@ fn render_rows_inner( desc_measure_items, area.width, area.height, - col_width_mode, + column_width, ); let desc_col = compute_desc_col( @@ -539,7 +555,7 @@ fn render_rows_inner( start_idx, desc_measure_items, area.width, - col_width_mode, + column_width, ); // Render items, wrapping descriptions and aligning wrapped lines under the @@ -603,35 +619,7 @@ pub(crate) fn render_rows( state, max_results, empty_message, - ColumnWidthMode::AutoVisible, - ) -} - -/// Render a list of rows using the provided ScrollState, with shared styling -/// and behavior for selection popups. -/// This mode keeps column placement stable while scrolling by sizing the -/// description column against the full dataset. -/// -/// This function should be paired with -/// [`measure_rows_height_stable_col_widths`] so reserved and rendered heights -/// stay in sync. -/// Returns the number of terminal lines actually rendered. -pub(crate) fn render_rows_stable_col_widths( - area: Rect, - buf: &mut Buffer, - rows_all: &[GenericDisplayRow], - state: &ScrollState, - max_results: usize, - empty_message: &str, -) -> u16 { - render_rows_inner( - area, - buf, - rows_all, - state, - max_results, - empty_message, - ColumnWidthMode::AutoAllRows, + ColumnWidthConfig::default(), ) } @@ -648,7 +636,7 @@ pub(crate) fn render_rows_with_col_width_mode( state: &ScrollState, max_results: usize, empty_message: &str, - col_width_mode: ColumnWidthMode, + column_width: ColumnWidthConfig, ) -> u16 { render_rows_inner( area, @@ -657,7 +645,7 @@ pub(crate) fn render_rows_with_col_width_mode( state, max_results, empty_message, - col_width_mode, + column_width, ) } @@ -673,6 +661,28 @@ pub(crate) fn render_rows_single_line( state: &ScrollState, max_results: usize, empty_message: &str, +) -> u16 { + render_rows_single_line_with_col_width_mode( + area, + buf, + rows_all, + state, + max_results, + empty_message, + ColumnWidthConfig::default(), + ) +} + +/// Render a list of rows as a single line each (no wrapping), truncating overflow with an +/// ellipsis while honoring the configured column width behavior. +pub(crate) fn render_rows_single_line_with_col_width_mode( + area: Rect, + buf: &mut Buffer, + rows_all: &[GenericDisplayRow], + state: &ScrollState, + max_results: usize, + empty_message: &str, + column_width: ColumnWidthConfig, ) -> u16 { if rows_all.is_empty() { if area.height > 0 { @@ -698,13 +708,7 @@ pub(crate) fn render_rows_single_line( } } - let desc_col = compute_desc_col( - rows_all, - start_idx, - visible_items, - area.width, - ColumnWidthMode::AutoVisible, - ); + let desc_col = compute_desc_col(rows_all, start_idx, visible_items, area.width, column_width); let mut cur_y = area.y; let mut rendered_lines: u16 = 0; @@ -766,25 +770,7 @@ pub(crate) fn measure_rows_height( state, max_results, width, - ColumnWidthMode::AutoVisible, - ) -} - -/// Measures selection-row height while using full-dataset column alignment. -/// This should be paired with [`render_rows_stable_col_widths`] so layout -/// reservation matches rendering behavior. -pub(crate) fn measure_rows_height_stable_col_widths( - rows_all: &[GenericDisplayRow], - state: &ScrollState, - max_results: usize, - width: u16, -) -> u16 { - measure_rows_height_inner( - rows_all, - state, - max_results, - width, - ColumnWidthMode::AutoAllRows, + ColumnWidthConfig::default(), ) } @@ -796,9 +782,9 @@ pub(crate) fn measure_rows_height_with_col_width_mode( state: &ScrollState, max_results: usize, width: u16, - col_width_mode: ColumnWidthMode, + column_width: ColumnWidthConfig, ) -> u16 { - measure_rows_height_inner(rows_all, state, max_results, width, col_width_mode) + measure_rows_height_inner(rows_all, state, max_results, width, column_width) } fn measure_rows_height_inner( @@ -806,7 +792,7 @@ fn measure_rows_height_inner( state: &ScrollState, max_results: usize, width: u16, - col_width_mode: ColumnWidthMode, + column_width: ColumnWidthConfig, ) -> u16 { if rows_all.is_empty() { return 1; // placeholder "no matches" line @@ -832,7 +818,7 @@ fn measure_rows_height_inner( start_idx, visible_items, content_width, - col_width_mode, + column_width, ); let mut total: u16 = 0; diff --git a/codex-rs/tui/src/bottom_pane/selection_tabs.rs b/codex-rs/tui/src/bottom_pane/selection_tabs.rs new file mode 100644 index 0000000000..8d5612b5ab --- /dev/null +++ b/codex-rs/tui/src/bottom_pane/selection_tabs.rs @@ -0,0 +1,103 @@ +use ratatui::buffer::Buffer; +use ratatui::layout::Rect; +use ratatui::style::Stylize; +use ratatui::text::Line; +use ratatui::text::Span; +use ratatui::widgets::Widget; + +use crate::render::renderable::Renderable; + +use super::SelectionItem; + +const TAB_GAP_WIDTH: usize = 2; + +pub(crate) struct SelectionTab { + pub(crate) id: String, + pub(crate) label: String, + pub(crate) header: Box, + pub(crate) items: Vec, +} + +pub(crate) fn tab_bar_height(tabs: &[SelectionTab], active_idx: usize, width: u16) -> u16 { + if tabs.is_empty() { + return 0; + } + tab_bar_lines(tabs, active_idx, width) + .len() + .try_into() + .unwrap_or(u16::MAX) +} + +pub(crate) fn render_tab_bar( + tabs: &[SelectionTab], + active_idx: usize, + area: Rect, + buf: &mut Buffer, +) { + for (offset, line) in tab_bar_lines(tabs, active_idx, area.width) + .into_iter() + .take(area.height as usize) + .enumerate() + { + line.render( + Rect { + x: area.x, + y: area.y.saturating_add(offset as u16), + width: area.width, + height: 1, + }, + buf, + ); + } +} + +fn tab_bar_lines(tabs: &[SelectionTab], active_idx: usize, width: u16) -> Vec> { + if tabs.is_empty() { + return Vec::new(); + } + + let max_width = width.max(1) as usize; + let mut lines = Vec::new(); + let mut current_spans: Vec> = Vec::new(); + let mut current_width = 0usize; + + for (idx, tab) in tabs.iter().enumerate() { + let unit = tab_unit(tab.label.as_str(), idx == active_idx); + let unit_width = Line::from(unit.clone()).width(); + let gap_width = if current_spans.is_empty() { + 0 + } else { + TAB_GAP_WIDTH + }; + + if !current_spans.is_empty() && current_width + gap_width + unit_width > max_width { + lines.push(Line::from(current_spans)); + current_spans = Vec::new(); + current_width = 0; + } + + if !current_spans.is_empty() { + current_spans.push(" ".into()); + current_width += TAB_GAP_WIDTH; + } + current_width += unit_width; + current_spans.extend(unit); + } + + if !current_spans.is_empty() { + lines.push(Line::from(current_spans)); + } + lines +} + +fn tab_unit(label: &str, active: bool) -> Vec> { + if active { + vec![ + "[".cyan().bold(), + label.to_string().cyan().bold(), + "]".cyan().bold(), + ] + } else { + vec![label.to_string().dim()] + } +} diff --git a/codex-rs/tui/src/chatwidget/plugins.rs b/codex-rs/tui/src/chatwidget/plugins.rs index 9b3f91b202..9b8f53ba6c 100644 --- a/codex-rs/tui/src/chatwidget/plugins.rs +++ b/codex-rs/tui/src/chatwidget/plugins.rs @@ -6,6 +6,7 @@ use super::ChatWidget; use crate::app_event::AppEvent; use crate::bottom_pane::ColumnWidthMode; use crate::bottom_pane::SelectionItem; +use crate::bottom_pane::SelectionRowDisplay; use crate::bottom_pane::SelectionViewParams; use crate::history_cell; use crate::onboarding::mark_url_hyperlink; @@ -31,8 +32,10 @@ use ratatui::text::Line; use ratatui::widgets::Paragraph; use ratatui::widgets::WidgetRef; use ratatui::widgets::Wrap; +use unicode_width::UnicodeWidthStr; const PLUGINS_SELECTION_VIEW_ID: &str = "plugins-selection"; +const PLUGIN_ROW_PREFIX_WIDTH: usize = 2; const LOADING_ANIMATION_DELAY: Duration = Duration::from_secs(1); const LOADING_ANIMATION_INTERVAL: Duration = Duration::from_millis(100); @@ -756,6 +759,12 @@ impl ChatWidget { .map(|(_, plugin, _)| plugin_status_label(plugin).chars().count()) .max() .unwrap_or(0); + let name_column_width = plugin_entries + .iter() + .map(|(_, _, display_name)| { + PLUGIN_ROW_PREFIX_WIDTH + UnicodeWidthStr::width(display_name.as_str()) + }) + .max(); let mut items: Vec = Vec::new(); for (marketplace, plugin, display_name) in plugin_entries { @@ -815,6 +824,8 @@ impl ChatWidget { is_searchable: true, search_placeholder: Some("Type to search plugins".to_string()), col_width_mode: ColumnWidthMode::AutoAllRows, + row_display: SelectionRowDisplay::SingleLine, + name_column_width, ..Default::default() } } @@ -830,7 +841,7 @@ impl ChatWidget { if plugin.summary.enabled { "Installed" } else { - "Installed · Disabled" + "Disabled" } } else { match plugin.summary.install_policy { @@ -1003,7 +1014,7 @@ fn plugin_status_label(plugin: &PluginSummary) -> &'static str { if plugin.enabled { "Installed" } else { - "Installed · Disabled" + "Disabled" } } else { match plugin.install_policy { diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__plugins_popup_curated_marketplace.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__plugins_popup_curated_marketplace.snap index 3c46da769c..ab85782510 100644 --- a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__plugins_popup_curated_marketplace.snap +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__plugins_popup_curated_marketplace.snap @@ -8,7 +8,7 @@ expression: popup Using cached marketplace data: remote sync timed out Type to search plugins -› Alpha Sync Installed · Disabled Press Enter to view plugin details. +› Alpha Sync Disabled Press Enter to view plugin details. Bravo Search Available · ChatGPT Marketplace · Search docs and tickets. Hidden Repo Plugin Available · Repo Marketplace · Should not be shown in /plugins. Starter Available by default · ChatGPT Marketplace · Included by default. diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__plugins_popup_search_filtered.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__plugins_popup_search_filtered.snap index d65308fb5b..3389ee5b85 100644 --- a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__plugins_popup_search_filtered.snap +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__plugins_popup_search_filtered.snap @@ -7,6 +7,6 @@ expression: popup Installed 0 of 3 available plugins. sla -› Slack Available Press Enter to view plugin details. +› Slack Available Press Enter to view plugin details. Press esc to close. From a1736fcd20396976daa631bd4ed53ca7d066bbb6 Mon Sep 17 00:00:00 2001 From: pakrym-oai Date: Thu, 16 Apr 2026 15:28:59 -0700 Subject: [PATCH 45/59] [codex] Split codex turn logic (#18206) ## Summary - Move Codex turn execution logic from `codex.rs` into `codex/turn.rs`. - Keep the existing crate-visible `run_turn`, `build_prompt`, `built_tools`, and `get_last_assistant_message_from_turn` surface re-exported from `codex.rs`. - Preserve test access for moved turn helpers while reducing the main `codex.rs` orchestration footprint. ## Stack - Base: #18200 (`pakrym/split-codex-handlers`) ## Testing - `CARGO_INCREMENTAL=0 cargo test -p codex-core --lib` - `just fix -p codex-core` - `just fmt` - `git diff --check` --- codex-rs/core/src/codex.rs | 2152 +----------------------------- codex-rs/core/src/codex/turn.rs | 2169 +++++++++++++++++++++++++++++++ 2 files changed, 2188 insertions(+), 2133 deletions(-) create mode 100644 codex-rs/core/src/codex/turn.rs diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index f356b3fff7..3d50f7735e 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -18,16 +18,10 @@ use crate::agent_identity::RegisteredAgentTask; use crate::apps::render_apps_section; use crate::commit_attribution::commit_message_trailer_instruction; use crate::compact; -use crate::compact::InitialContextInjection; -use crate::compact::run_inline_auto_compact_task; -use crate::compact::should_use_remote_compact_task; -use crate::compact_remote::run_inline_remote_auto_compact_task; use crate::config::ManagedFeatures; use crate::connectors; use crate::exec_policy::ExecPolicyManager; -use crate::hook_runtime::emit_hook_completed_events; use crate::installation_id::resolve_installation_id; -use crate::mcp_tool_exposure::build_mcp_tool_exposure; use crate::parse_turn_item; use crate::path_utils::normalize_for_native_workdir; use crate::realtime_conversation::RealtimeConversationManager; @@ -35,28 +29,13 @@ use crate::render_skills_section; use crate::rollout::find_thread_name_by_id; use crate::session_prefix::format_subagent_notification_message; use crate::skills_load_input_from_config; -use crate::stream_events_utils::HandleOutputCtx; -use crate::stream_events_utils::handle_non_tool_response_item; -use crate::stream_events_utils::handle_output_item_done; -use crate::stream_events_utils::last_assistant_message_from_item; -use crate::stream_events_utils::mark_thread_memory_mode_polluted_if_external_context; -use crate::stream_events_utils::raw_assistant_output_text_from_item; -use crate::stream_events_utils::record_completed_response_item; use crate::turn_metadata::TurnMetadataState; -use crate::unavailable_tool::collect_unavailable_called_tools; -use crate::util::error_or_panic; use async_channel::Receiver; use async_channel::Sender; use chrono::Local; use chrono::Utc; use codex_analytics::AnalyticsEventsClient; -use codex_analytics::AppInvocation; -use codex_analytics::CompactionPhase; -use codex_analytics::CompactionReason; -use codex_analytics::InvocationType; use codex_analytics::SubAgentThreadStartedInput; -use codex_analytics::TurnResolvedConfigFact; -use codex_analytics::build_track_events_context; use codex_app_server_protocol::AuthMode; use codex_app_server_protocol::McpServerElicitationRequest; use codex_app_server_protocol::McpServerElicitationRequestParams; @@ -67,10 +46,6 @@ use codex_exec_server::FileSystemSandboxContext; use codex_features::FEATURES; use codex_features::Feature; use codex_features::unstable_features_warning_event; -use codex_hooks::HookEvent; -use codex_hooks::HookEventAfterAgent; -use codex_hooks::HookPayload; -use codex_hooks::HookResult; use codex_hooks::Hooks; use codex_hooks::HooksConfig; use codex_login::AuthManager; @@ -102,10 +77,8 @@ use codex_protocol::config_types::Settings; use codex_protocol::config_types::WebSearchMode; use codex_protocol::dynamic_tools::DynamicToolResponse; use codex_protocol::dynamic_tools::DynamicToolSpec; -use codex_protocol::items::PlanItem; use codex_protocol::items::TurnItem; use codex_protocol::items::UserMessageItem; -use codex_protocol::items::build_hook_prompt_message; use codex_protocol::mcp::CallToolResult; use codex_protocol::models::BaseInstructions; use codex_protocol::models::PermissionProfile; @@ -140,17 +113,10 @@ use codex_rollout::state_db; use codex_shell_command::parse_command::parse_command; use codex_terminal_detection::user_agent; use codex_thread_store::LocalThreadStore; -use codex_tools::filter_tool_suggest_discoverable_tools_for_client; use codex_utils_output_truncation::TruncationPolicy; -use codex_utils_stream_parser::AssistantTextChunk; -use codex_utils_stream_parser::AssistantTextStreamParser; -use codex_utils_stream_parser::ProposedPlanSegment; -use codex_utils_stream_parser::extract_proposed_plan_text; -use codex_utils_stream_parser::strip_citations; use futures::future::BoxFuture; use futures::future::Shared; use futures::prelude::*; -use futures::stream::FuturesOrdered; use rmcp::model::ListResourceTemplatesResult; use rmcp::model::ListResourcesResult; use rmcp::model::PaginatedRequestParams; @@ -168,19 +134,13 @@ use toml::Value as TomlValue; use tracing::Instrument; use tracing::debug; use tracing::error; -use tracing::field; use tracing::info; use tracing::info_span; use tracing::instrument; -use tracing::trace; -use tracing::trace_span; use tracing::warn; use uuid::Uuid; use crate::client::ModelClient; -use crate::client::ModelClientSession; -use crate::client_common::Prompt; -use crate::client_common::ResponseEvent; use crate::codex_thread::ThreadConfigSnapshot; use crate::compact::collect_user_messages; use crate::config::Config; @@ -204,10 +164,22 @@ use codex_protocol::exec_output::StreamOutput; mod handlers; mod rollout_reconstruction; +mod turn; #[cfg(test)] use self::handlers::submission_dispatch_span; use self::handlers::submission_loop; #[cfg(test)] +use self::turn::AssistantMessageStreamParsers; +pub(crate) use self::turn::build_prompt; +pub(crate) use self::turn::built_tools; +#[cfg(test)] +use self::turn::collect_explicit_app_ids_from_skill_items; +#[cfg(test)] +use self::turn::filter_connectors_for_input; +pub(crate) use self::turn::get_last_assistant_message_from_turn; +use self::turn::realtime_text_for_event; +pub(crate) use self::turn::run_turn; +#[cfg(test)] mod rollout_reconstruction_tests; #[derive(Debug, PartialEq)] @@ -262,40 +234,18 @@ pub(crate) struct PreviousTurnSettings { } use crate::SkillError; -use crate::SkillInjections; use crate::SkillLoadOutcome; use crate::SkillMetadata; use crate::SkillsManager; use crate::agents_md::AgentsMdManager; -use crate::build_skill_injections; -use crate::collect_env_var_dependencies; -use crate::collect_explicit_skill_mentions; use crate::exec_policy::ExecPolicyUpdateError; -use crate::feedback_tags; use crate::guardian::GuardianReviewSessionManager; -use crate::hook_runtime::PendingInputHookDisposition; -use crate::hook_runtime::inspect_pending_input; -use crate::hook_runtime::record_additional_contexts; -use crate::hook_runtime::record_pending_input; -use crate::hook_runtime::run_pending_session_start_hooks; -use crate::hook_runtime::run_user_prompt_submit_hooks; -use crate::injection::ToolMentionKind; -use crate::injection::app_id_from_path; -use crate::injection::tool_kind_for_path; use crate::instructions::UserInstructions; use crate::mcp::McpManager; -use crate::mcp_skill_dependencies::maybe_prompt_and_install_mcp_dependencies; use crate::memories; -use crate::mentions::build_connector_slug_counts; -use crate::mentions::build_skill_name_counts; -use crate::mentions::collect_explicit_app_ids; -use crate::mentions::collect_explicit_plugin_mentions; -use crate::mentions::collect_tool_mentions_from_messages; use crate::network_policy_decision::execpolicy_network_rule_amendment; use crate::plugins::PluginsManager; -use crate::plugins::build_plugin_injections; use crate::plugins::render_plugins_section; -use crate::resolve_skill_dependencies_for_turn; use crate::rollout::RolloutRecorder; use crate::rollout::RolloutRecorderParams; use crate::rollout::map_session_init_error; @@ -310,28 +260,26 @@ use crate::state::ActiveTurn; use crate::state::MailboxDeliveryPhase; use crate::state::SessionServices; use crate::state::SessionState; +#[cfg(test)] +use crate::stream_events_utils::HandleOutputCtx; +#[cfg(test)] +use crate::stream_events_utils::handle_output_item_done; use crate::tasks::GhostSnapshotTask; use crate::tasks::ReviewTask; use crate::tasks::SessionTask; use crate::tasks::SessionTaskContext; -use crate::tools::ToolRouter; -use crate::tools::context::SharedTurnDiffTracker; use crate::tools::js_repl::JsReplHandle; use crate::tools::js_repl::resolve_compatible_node; use crate::tools::network_approval::NetworkApprovalService; use crate::tools::network_approval::build_blocked_request_observer; use crate::tools::network_approval::build_network_policy_decider; +#[cfg(test)] use crate::tools::parallel::ToolCallRuntime; -use crate::tools::router::ToolRouterParams; use crate::tools::sandboxing::ApprovalStore; -use crate::turn_diff_tracker::TurnDiffTracker; use crate::turn_timing::TurnTimingState; use crate::turn_timing::record_turn_ttfm_metric; -use crate::turn_timing::record_turn_ttft_metric; use crate::unified_exec::UnifiedExecProcessManager; -use crate::util::backoff; use crate::windows_sandbox::WindowsSandboxLevelExt; -use codex_async_utils::OrCancelExt; use codex_git_utils::get_git_repo_root; use codex_mcp::compute_auth_statuses; use codex_mcp::with_codex_apps_mcp; @@ -348,8 +296,6 @@ use codex_protocol::models::DeveloperInstructions; use codex_protocol::models::ResponseInputItem; use codex_protocol::models::ResponseItem; use codex_protocol::openai_models::ReasoningEffort as ReasoningEffortConfig; -use codex_protocol::protocol::AgentMessageContentDeltaEvent; -use codex_protocol::protocol::AgentReasoningSectionBreakEvent; use codex_protocol::protocol::ApplyPatchApprovalRequestEvent; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::BackgroundEventEvent; @@ -367,10 +313,7 @@ use codex_protocol::protocol::ModelRerouteReason; use codex_protocol::protocol::NetworkApprovalContext; use codex_protocol::protocol::NonSteerableTurnKind; use codex_protocol::protocol::Op; -use codex_protocol::protocol::PlanDeltaEvent; use codex_protocol::protocol::RateLimitSnapshot; -use codex_protocol::protocol::ReasoningContentDeltaEvent; -use codex_protocol::protocol::ReasoningRawContentDeltaEvent; use codex_protocol::protocol::RequestUserInputEvent; use codex_protocol::protocol::ReviewDecision; use codex_protocol::protocol::SandboxPolicy; @@ -386,7 +329,6 @@ use codex_protocol::protocol::Submission; use codex_protocol::protocol::TokenCountEvent; use codex_protocol::protocol::TokenUsage; use codex_protocol::protocol::TokenUsageInfo; -use codex_protocol::protocol::TurnDiffEvent; use codex_protocol::protocol::WarningEvent; use codex_protocol::user_input::UserInput; use codex_tools::ToolsConfig; @@ -394,6 +336,8 @@ use codex_tools::ToolsConfigParams; use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_readiness::Readiness; use codex_utils_readiness::ReadinessFlag; +#[cfg(test)] +use codex_utils_stream_parser::ProposedPlanSegment; fn image_generation_tool_auth_allowed(auth_manager: Option<&AuthManager>) -> bool { matches!( @@ -5032,2064 +4976,6 @@ fn errors_to_info(errors: &[SkillError]) -> Vec { .collect() } -/// Takes a user message as input and runs a loop where, at each sampling request, the model -/// replies with either: -/// -/// - requested function calls -/// - an assistant message -/// -/// While it is possible for the model to return multiple of these items in a -/// single sampling request, in practice, we generally one item per sampling request: -/// -/// - If the model requests a function call, we execute it and send the output -/// back to the model in the next sampling request. -/// - If the model sends only an assistant message, we record it in the -/// conversation history and consider the turn complete. -/// -pub(crate) async fn run_turn( - sess: Arc, - turn_context: Arc, - input: Vec, - prewarmed_client_session: Option, - cancellation_token: CancellationToken, -) -> Option { - if input.is_empty() && !sess.has_pending_input().await { - return None; - } - - let model_info = turn_context.model_info.clone(); - let auto_compact_limit = model_info.auto_compact_token_limit().unwrap_or(i64::MAX); - let mut prewarmed_client_session = prewarmed_client_session; - // TODO(ccunningham): Pre-turn compaction runs before context updates and the - // new user message are recorded. Estimate pending incoming items (context - // diffs/full reinjection + user input) and trigger compaction preemptively - // when they would push the thread over the compaction threshold. - let pre_sampling_compacted = match run_pre_sampling_compact(&sess, &turn_context).await { - Ok(pre_sampling_compacted) => pre_sampling_compacted, - Err(_) => { - error!("Failed to run pre-sampling compact"); - return None; - } - }; - if pre_sampling_compacted && let Some(mut client_session) = prewarmed_client_session.take() { - client_session.reset_websocket_session(); - } - - let skills_outcome = Some(turn_context.turn_skills.outcome.as_ref()); - - sess.record_context_updates_and_set_reference_context_item(turn_context.as_ref()) - .await; - - let loaded_plugins = sess - .services - .plugins_manager - .plugins_for_config(&turn_context.config) - .await; - // Structured plugin:// mentions are resolved from the current session's - // enabled plugins, then converted into turn-scoped guidance below. - let mentioned_plugins = - collect_explicit_plugin_mentions(&input, loaded_plugins.capability_summaries()); - let mcp_tools = if turn_context.apps_enabled() || !mentioned_plugins.is_empty() { - // Plugin mentions need raw MCP/app inventory even when app tools - // are normally hidden so we can describe the plugin's currently - // usable capabilities for this turn. - match sess - .services - .mcp_connection_manager - .read() - .await - .list_all_tools() - .or_cancel(&cancellation_token) - .await - { - Ok(mcp_tools) => mcp_tools, - Err(_) if turn_context.apps_enabled() => return None, - Err(_) => HashMap::new(), - } - } else { - HashMap::new() - }; - let available_connectors = if turn_context.apps_enabled() { - let connectors = codex_connectors::merge::merge_plugin_connectors_with_accessible( - loaded_plugins - .effective_apps() - .into_iter() - .map(|connector_id| connector_id.0), - connectors::accessible_connectors_from_mcp_tools(&mcp_tools), - ); - connectors::with_app_enabled_state(connectors, &turn_context.config) - } else { - Vec::new() - }; - let connector_slug_counts = build_connector_slug_counts(&available_connectors); - let skill_name_counts_lower = skills_outcome - .as_ref() - .map_or_else(HashMap::new, |outcome| { - build_skill_name_counts(&outcome.skills, &outcome.disabled_paths).1 - }); - let mentioned_skills = skills_outcome.as_ref().map_or_else(Vec::new, |outcome| { - collect_explicit_skill_mentions( - &input, - &outcome.skills, - &outcome.disabled_paths, - &connector_slug_counts, - ) - }); - let config = turn_context.config.clone(); - if config - .features - .enabled(Feature::SkillEnvVarDependencyPrompt) - { - let env_var_dependencies = collect_env_var_dependencies(&mentioned_skills); - resolve_skill_dependencies_for_turn(&sess, &turn_context, &env_var_dependencies).await; - } - - maybe_prompt_and_install_mcp_dependencies( - sess.as_ref(), - turn_context.as_ref(), - &cancellation_token, - &mentioned_skills, - ) - .await; - - let session_telemetry = turn_context.session_telemetry.clone(); - let thread_id = sess.conversation_id.to_string(); - let tracking = build_track_events_context( - turn_context.model_info.slug.clone(), - thread_id, - turn_context.sub_id.clone(), - ); - let SkillInjections { - items: skill_items, - warnings: skill_warnings, - } = build_skill_injections( - &mentioned_skills, - skills_outcome, - Some(&session_telemetry), - &sess.services.analytics_events_client, - tracking.clone(), - ) - .await; - - for message in skill_warnings { - sess.send_event(&turn_context, EventMsg::Warning(WarningEvent { message })) - .await; - } - - let plugin_items = - build_plugin_injections(&mentioned_plugins, &mcp_tools, &available_connectors); - let mentioned_plugin_metadata = mentioned_plugins - .iter() - .filter_map(crate::plugins::PluginCapabilitySummary::telemetry_metadata) - .collect::>(); - - let mut explicitly_enabled_connectors = collect_explicit_app_ids(&input); - explicitly_enabled_connectors.extend(collect_explicit_app_ids_from_skill_items( - &skill_items, - &available_connectors, - &skill_name_counts_lower, - )); - let connector_names_by_id = available_connectors - .iter() - .map(|connector| (connector.id.as_str(), connector.name.as_str())) - .collect::>(); - let mentioned_app_invocations = explicitly_enabled_connectors - .iter() - .map(|connector_id| AppInvocation { - connector_id: Some(connector_id.clone()), - app_name: connector_names_by_id - .get(connector_id.as_str()) - .map(|name| (*name).to_string()), - invocation_type: Some(InvocationType::Explicit), - }) - .collect::>(); - - if run_pending_session_start_hooks(&sess, &turn_context).await { - return None; - } - let additional_contexts = if input.is_empty() { - Vec::new() - } else { - let initial_input_for_turn: ResponseInputItem = ResponseInputItem::from(input.clone()); - let response_item: ResponseItem = initial_input_for_turn.clone().into(); - let user_prompt_submit_outcome = run_user_prompt_submit_hooks( - &sess, - &turn_context, - UserMessageItem::new(&input).message(), - ) - .await; - if user_prompt_submit_outcome.should_stop { - record_additional_contexts( - &sess, - &turn_context, - user_prompt_submit_outcome.additional_contexts, - ) - .await; - return None; - } - sess.record_user_prompt_and_emit_turn_item(turn_context.as_ref(), &input, response_item) - .await; - user_prompt_submit_outcome.additional_contexts - }; - sess.services - .analytics_events_client - .track_app_mentioned(tracking.clone(), mentioned_app_invocations); - for plugin in mentioned_plugin_metadata { - sess.services - .analytics_events_client - .track_plugin_used(tracking.clone(), plugin); - } - sess.merge_connector_selection(explicitly_enabled_connectors.clone()) - .await; - record_additional_contexts(&sess, &turn_context, additional_contexts).await; - if !input.is_empty() { - // Track the previous-turn baseline from the regular user-turn path only so - // standalone tasks (compact/shell/review/undo) cannot suppress future - // model/realtime injections. - sess.set_previous_turn_settings(Some(PreviousTurnSettings { - model: turn_context.model_info.slug.clone(), - realtime_active: Some(turn_context.realtime_active), - })) - .await; - } - if let Err(error) = sess.ensure_agent_task_registered().await { - warn!(error = %error, "agent task registration failed"); - sess.send_event( - turn_context.as_ref(), - EventMsg::Error(ErrorEvent { - message: format!( - "Agent task registration failed. Please try again; Codex will attempt to register the task again on the next turn: {error}" - ), - codex_error_info: Some(CodexErrorInfo::Other), - }), - ) - .await; - return None; - } - - if !skill_items.is_empty() { - sess.record_conversation_items(&turn_context, &skill_items) - .await; - } - if !plugin_items.is_empty() { - sess.record_conversation_items(&turn_context, &plugin_items) - .await; - } - - track_turn_resolved_config_analytics(&sess, &turn_context, &input).await; - - let skills_outcome = Some(turn_context.turn_skills.outcome.as_ref()); - sess.maybe_start_ghost_snapshot(Arc::clone(&turn_context), cancellation_token.child_token()) - .await; - let mut last_agent_message: Option = None; - let mut stop_hook_active = false; - // Although from the perspective of codex.rs, TurnDiffTracker has the lifecycle of a Task which contains - // many turns, from the perspective of the user, it is a single turn. - let turn_diff_tracker = Arc::new(tokio::sync::Mutex::new(TurnDiffTracker::new())); - let mut server_model_warning_emitted_for_turn = false; - - // `ModelClientSession` is turn-scoped and caches WebSocket + sticky routing state, so we reuse - // one instance across retries within this turn. - let mut client_session = - prewarmed_client_session.unwrap_or_else(|| sess.services.model_client.new_session()); - // Pending input is drained into history before building the next model request. - // However, we defer that drain until after sampling in two cases: - // 1. At the start of a turn, so the fresh user prompt in `input` gets sampled first. - // 2. After auto-compact, when model/tool continuation needs to resume before any steer. - let mut can_drain_pending_input = input.is_empty(); - - loop { - if run_pending_session_start_hooks(&sess, &turn_context).await { - break; - } - - // Note that pending_input would be something like a message the user - // submitted through the UI while the model was running. Though the UI - // may support this, the model might not. - let pending_input = if can_drain_pending_input { - sess.get_pending_input().await - } else { - Vec::new() - }; - - let mut blocked_pending_input = false; - let mut blocked_pending_input_contexts = Vec::new(); - let mut requeued_pending_input = false; - let mut accepted_pending_input = Vec::new(); - if !pending_input.is_empty() { - let mut pending_input_iter = pending_input.into_iter(); - while let Some(pending_input_item) = pending_input_iter.next() { - match inspect_pending_input(&sess, &turn_context, pending_input_item).await { - PendingInputHookDisposition::Accepted(pending_input) => { - accepted_pending_input.push(*pending_input); - } - PendingInputHookDisposition::Blocked { - additional_contexts, - } => { - let remaining_pending_input = pending_input_iter.collect::>(); - if !remaining_pending_input.is_empty() { - let _ = sess.prepend_pending_input(remaining_pending_input).await; - requeued_pending_input = true; - } - blocked_pending_input_contexts = additional_contexts; - blocked_pending_input = true; - break; - } - } - } - } - - let has_accepted_pending_input = !accepted_pending_input.is_empty(); - for pending_input in accepted_pending_input { - record_pending_input(&sess, &turn_context, pending_input).await; - } - record_additional_contexts(&sess, &turn_context, blocked_pending_input_contexts).await; - - if blocked_pending_input && !has_accepted_pending_input { - if requeued_pending_input { - continue; - } - break; - } - - // Construct the input that we will send to the model. - let sampling_request_input: Vec = { - sess.clone_history() - .await - .for_prompt(&turn_context.model_info.input_modalities) - }; - - let sampling_request_input_messages = sampling_request_input - .iter() - .filter_map(|item| match parse_turn_item(item) { - Some(TurnItem::UserMessage(user_message)) => Some(user_message), - _ => None, - }) - .map(|user_message| user_message.message()) - .collect::>(); - let turn_metadata_header = turn_context.turn_metadata_state.current_header_value(); - match run_sampling_request( - Arc::clone(&sess), - Arc::clone(&turn_context), - Arc::clone(&turn_diff_tracker), - &mut client_session, - turn_metadata_header.as_deref(), - sampling_request_input, - &explicitly_enabled_connectors, - skills_outcome, - &mut server_model_warning_emitted_for_turn, - cancellation_token.child_token(), - ) - .await - { - Ok(sampling_request_output) => { - let SamplingRequestResult { - needs_follow_up: model_needs_follow_up, - last_agent_message: sampling_request_last_agent_message, - } = sampling_request_output; - can_drain_pending_input = true; - let has_pending_input = sess.has_pending_input().await; - let needs_follow_up = model_needs_follow_up || has_pending_input; - let total_usage_tokens = sess.get_total_token_usage().await; - let token_limit_reached = total_usage_tokens >= auto_compact_limit; - - let estimated_token_count = - sess.get_estimated_token_count(turn_context.as_ref()).await; - - trace!( - turn_id = %turn_context.sub_id, - total_usage_tokens, - estimated_token_count = ?estimated_token_count, - auto_compact_limit, - token_limit_reached, - model_needs_follow_up, - has_pending_input, - needs_follow_up, - "post sampling token usage" - ); - - // as long as compaction works well in getting us way below the token limit, we shouldn't worry about being in an infinite loop. - if token_limit_reached && needs_follow_up { - if run_auto_compact( - &sess, - &turn_context, - InitialContextInjection::BeforeLastUserMessage, - CompactionReason::ContextLimit, - CompactionPhase::MidTurn, - ) - .await - .is_err() - { - return None; - } - client_session.reset_websocket_session(); - can_drain_pending_input = !model_needs_follow_up; - continue; - } - - if !needs_follow_up { - last_agent_message = sampling_request_last_agent_message; - let stop_hook_permission_mode = match turn_context.approval_policy.value() { - AskForApproval::Never => "bypassPermissions", - AskForApproval::UnlessTrusted - | AskForApproval::OnFailure - | AskForApproval::OnRequest - | AskForApproval::Granular(_) => "default", - } - .to_string(); - let stop_request = codex_hooks::StopRequest { - session_id: sess.conversation_id, - turn_id: turn_context.sub_id.clone(), - cwd: turn_context.cwd.clone(), - transcript_path: sess.hook_transcript_path().await, - model: turn_context.model_info.slug.clone(), - permission_mode: stop_hook_permission_mode, - stop_hook_active, - last_assistant_message: last_agent_message.clone(), - }; - for run in sess.hooks().preview_stop(&stop_request) { - sess.send_event( - &turn_context, - EventMsg::HookStarted(codex_protocol::protocol::HookStartedEvent { - turn_id: Some(turn_context.sub_id.clone()), - run, - }), - ) - .await; - } - let stop_outcome = sess.hooks().run_stop(stop_request).await; - emit_hook_completed_events(&sess, &turn_context, stop_outcome.hook_events) - .await; - if stop_outcome.should_block { - if let Some(hook_prompt_message) = - build_hook_prompt_message(&stop_outcome.continuation_fragments) - { - sess.record_conversation_items( - &turn_context, - std::slice::from_ref(&hook_prompt_message), - ) - .await; - stop_hook_active = true; - continue; - } else { - sess.send_event( - &turn_context, - EventMsg::Warning(WarningEvent { - message: "Stop hook requested continuation without a prompt; ignoring the block.".to_string(), - }), - ) - .await; - } - } - if stop_outcome.should_stop { - break; - } - let hook_outcomes = sess - .hooks() - .dispatch(HookPayload { - session_id: sess.conversation_id, - cwd: turn_context.cwd.clone(), - client: turn_context.app_server_client_name.clone(), - triggered_at: chrono::Utc::now(), - hook_event: HookEvent::AfterAgent { - event: HookEventAfterAgent { - thread_id: sess.conversation_id, - turn_id: turn_context.sub_id.clone(), - input_messages: sampling_request_input_messages, - last_assistant_message: last_agent_message.clone(), - }, - }, - }) - .await; - - let mut abort_message = None; - for hook_outcome in hook_outcomes { - let hook_name = hook_outcome.hook_name; - match hook_outcome.result { - HookResult::Success => {} - HookResult::FailedContinue(error) => { - warn!( - turn_id = %turn_context.sub_id, - hook_name = %hook_name, - error = %error, - "after_agent hook failed; continuing" - ); - } - HookResult::FailedAbort(error) => { - let message = format!( - "after_agent hook '{hook_name}' failed and aborted turn completion: {error}" - ); - warn!( - turn_id = %turn_context.sub_id, - hook_name = %hook_name, - error = %error, - "after_agent hook failed; aborting operation" - ); - if abort_message.is_none() { - abort_message = Some(message); - } - } - } - } - if let Some(message) = abort_message { - sess.send_event( - &turn_context, - EventMsg::Error(ErrorEvent { - message, - codex_error_info: None, - }), - ) - .await; - return None; - } - break; - } - continue; - } - Err(CodexErr::TurnAborted) => { - // Aborted turn is reported via a different event. - break; - } - Err(CodexErr::InvalidImageRequest()) => { - { - let mut state = sess.state.lock().await; - error_or_panic( - "Invalid image detected; sanitizing tool output to prevent poisoning", - ); - if state.history.replace_last_turn_images("Invalid image") { - continue; - } - } - - let event = EventMsg::Error(ErrorEvent { - message: "Invalid image in your last message. Please remove it and try again." - .to_string(), - codex_error_info: Some(CodexErrorInfo::BadRequest), - }); - sess.send_event(&turn_context, event).await; - break; - } - Err(e) => { - info!("Turn error: {e:#}"); - let event = EventMsg::Error(e.to_error_event(/*message_prefix*/ None)); - sess.send_event(&turn_context, event).await; - // let the user continue the conversation - break; - } - } - } - - last_agent_message -} - -async fn track_turn_resolved_config_analytics( - sess: &Session, - turn_context: &TurnContext, - input: &[UserInput], -) { - if !sess.enabled(Feature::GeneralAnalytics) { - return; - } - - let thread_config = { - let state = sess.state.lock().await; - state.session_configuration.thread_config_snapshot() - }; - let is_first_turn = { - let mut state = sess.state.lock().await; - state.take_next_turn_is_first() - }; - sess.services - .analytics_events_client - .track_turn_resolved_config(TurnResolvedConfigFact { - turn_id: turn_context.sub_id.clone(), - thread_id: sess.conversation_id.to_string(), - num_input_images: input - .iter() - .filter(|item| { - matches!(item, UserInput::Image { .. } | UserInput::LocalImage { .. }) - }) - .count(), - submission_type: None, - ephemeral: thread_config.ephemeral, - session_source: thread_config.session_source, - model: turn_context.model_info.slug.clone(), - model_provider: turn_context.config.model_provider_id.clone(), - sandbox_policy: turn_context.sandbox_policy.get().clone(), - reasoning_effort: turn_context.reasoning_effort, - reasoning_summary: Some(turn_context.reasoning_summary), - service_tier: turn_context.config.service_tier, - approval_policy: turn_context.approval_policy.value(), - approvals_reviewer: turn_context.config.approvals_reviewer, - sandbox_network_access: turn_context.network_sandbox_policy.is_enabled(), - collaboration_mode: turn_context.collaboration_mode.mode, - personality: turn_context.personality, - is_first_turn, - }); -} - -async fn run_pre_sampling_compact( - sess: &Arc, - turn_context: &Arc, -) -> CodexResult { - let total_usage_tokens_before_compaction = sess.get_total_token_usage().await; - let mut pre_sampling_compacted = maybe_run_previous_model_inline_compact( - sess, - turn_context, - total_usage_tokens_before_compaction, - ) - .await?; - let total_usage_tokens = sess.get_total_token_usage().await; - let auto_compact_limit = turn_context - .model_info - .auto_compact_token_limit() - .unwrap_or(i64::MAX); - // Compact if the total usage tokens are greater than the auto compact limit - if total_usage_tokens >= auto_compact_limit { - run_auto_compact( - sess, - turn_context, - InitialContextInjection::DoNotInject, - CompactionReason::ContextLimit, - CompactionPhase::PreTurn, - ) - .await?; - pre_sampling_compacted = true; - } - Ok(pre_sampling_compacted) -} - -/// Runs pre-sampling compaction against the previous model when switching to a smaller -/// context-window model. -/// -/// Returns `Ok(true)` when compaction ran successfully, `Ok(false)` when compaction was skipped -/// because the model/context-window preconditions were not met, and `Err(_)` only when compaction -/// was attempted and failed. -async fn maybe_run_previous_model_inline_compact( - sess: &Arc, - turn_context: &Arc, - total_usage_tokens: i64, -) -> CodexResult { - let Some(previous_turn_settings) = sess.previous_turn_settings().await else { - return Ok(false); - }; - let previous_model_turn_context = Arc::new( - turn_context - .with_model(previous_turn_settings.model, &sess.services.models_manager) - .await, - ); - - let Some(old_context_window) = previous_model_turn_context.model_context_window() else { - return Ok(false); - }; - let Some(new_context_window) = turn_context.model_context_window() else { - return Ok(false); - }; - let new_auto_compact_limit = turn_context - .model_info - .auto_compact_token_limit() - .unwrap_or(i64::MAX); - let should_run = total_usage_tokens > new_auto_compact_limit - && previous_model_turn_context.model_info.slug != turn_context.model_info.slug - && old_context_window > new_context_window; - if should_run { - run_auto_compact( - sess, - &previous_model_turn_context, - InitialContextInjection::DoNotInject, - CompactionReason::ModelDownshift, - CompactionPhase::PreTurn, - ) - .await?; - return Ok(true); - } - Ok(false) -} - -async fn run_auto_compact( - sess: &Arc, - turn_context: &Arc, - initial_context_injection: InitialContextInjection, - reason: CompactionReason, - phase: CompactionPhase, -) -> CodexResult<()> { - if should_use_remote_compact_task(&turn_context.provider) { - run_inline_remote_auto_compact_task( - Arc::clone(sess), - Arc::clone(turn_context), - initial_context_injection, - reason, - phase, - ) - .await?; - } else { - run_inline_auto_compact_task( - Arc::clone(sess), - Arc::clone(turn_context), - initial_context_injection, - reason, - phase, - ) - .await?; - } - Ok(()) -} - -fn collect_explicit_app_ids_from_skill_items( - skill_items: &[ResponseItem], - connectors: &[connectors::AppInfo], - skill_name_counts_lower: &HashMap, -) -> HashSet { - if skill_items.is_empty() || connectors.is_empty() { - return HashSet::new(); - } - - let skill_messages = skill_items - .iter() - .filter_map(|item| match item { - ResponseItem::Message { content, .. } => { - content.iter().find_map(|content_item| match content_item { - ContentItem::InputText { text } => Some(text.clone()), - _ => None, - }) - } - _ => None, - }) - .collect::>(); - if skill_messages.is_empty() { - return HashSet::new(); - } - - let mentions = collect_tool_mentions_from_messages(&skill_messages); - let mention_names_lower = mentions - .plain_names - .iter() - .map(|name| name.to_ascii_lowercase()) - .collect::>(); - let mut connector_ids = mentions - .paths - .iter() - .filter(|path| tool_kind_for_path(path) == ToolMentionKind::App) - .filter_map(|path| app_id_from_path(path).map(str::to_string)) - .collect::>(); - - let connector_slug_counts = build_connector_slug_counts(connectors); - for connector in connectors { - let slug = codex_connectors::metadata::connector_mention_slug(connector); - let connector_count = connector_slug_counts.get(&slug).copied().unwrap_or(0); - let skill_count = skill_name_counts_lower.get(&slug).copied().unwrap_or(0); - if connector_count == 1 && skill_count == 0 && mention_names_lower.contains(&slug) { - connector_ids.insert(connector.id.clone()); - } - } - - connector_ids -} - -fn filter_connectors_for_input( - connectors: &[connectors::AppInfo], - input: &[ResponseItem], - explicitly_enabled_connectors: &HashSet, - skill_name_counts_lower: &HashMap, -) -> Vec { - let connectors: Vec = connectors - .iter() - .filter(|connector| connector.is_enabled) - .cloned() - .collect::>(); - if connectors.is_empty() { - return Vec::new(); - } - - let user_messages = collect_user_messages(input); - if user_messages.is_empty() && explicitly_enabled_connectors.is_empty() { - return Vec::new(); - } - - let mentions = collect_tool_mentions_from_messages(&user_messages); - let mention_names_lower = mentions - .plain_names - .iter() - .map(|name| name.to_ascii_lowercase()) - .collect::>(); - - let connector_slug_counts = build_connector_slug_counts(&connectors); - let mut allowed_connector_ids = explicitly_enabled_connectors.clone(); - for path in mentions - .paths - .iter() - .filter(|path| tool_kind_for_path(path) == ToolMentionKind::App) - { - if let Some(connector_id) = app_id_from_path(path) { - allowed_connector_ids.insert(connector_id.to_string()); - } - } - - connectors - .into_iter() - .filter(|connector| { - connector_inserted_in_messages( - connector, - &mention_names_lower, - &allowed_connector_ids, - &connector_slug_counts, - skill_name_counts_lower, - ) - }) - .collect() -} - -fn connector_inserted_in_messages( - connector: &connectors::AppInfo, - mention_names_lower: &HashSet, - allowed_connector_ids: &HashSet, - connector_slug_counts: &HashMap, - skill_name_counts_lower: &HashMap, -) -> bool { - if allowed_connector_ids.contains(&connector.id) { - return true; - } - - let mention_slug = codex_connectors::metadata::connector_mention_slug(connector); - let connector_count = connector_slug_counts - .get(&mention_slug) - .copied() - .unwrap_or(0); - let skill_count = skill_name_counts_lower - .get(&mention_slug) - .copied() - .unwrap_or(0); - connector_count == 1 && skill_count == 0 && mention_names_lower.contains(&mention_slug) -} - -pub(crate) fn build_prompt( - input: Vec, - router: &ToolRouter, - turn_context: &TurnContext, - base_instructions: BaseInstructions, -) -> Prompt { - let deferred_dynamic_tools = turn_context - .dynamic_tools - .iter() - .filter(|tool| tool.defer_loading) - .map(|tool| tool.name.as_str()) - .collect::>(); - let tools = if deferred_dynamic_tools.is_empty() { - router.model_visible_specs() - } else { - router - .model_visible_specs() - .into_iter() - .filter(|spec| !deferred_dynamic_tools.contains(spec.name())) - .collect() - }; - - Prompt { - input, - tools, - parallel_tool_calls: turn_context.model_info.supports_parallel_tool_calls, - base_instructions, - personality: turn_context.personality, - output_schema: turn_context.final_output_json_schema.clone(), - } -} - -#[allow(clippy::too_many_arguments)] -#[instrument(level = "trace", - skip_all, - fields( - turn_id = %turn_context.sub_id, - model = %turn_context.model_info.slug, - cwd = %turn_context.cwd.display() - ) -)] -async fn run_sampling_request( - sess: Arc, - turn_context: Arc, - turn_diff_tracker: SharedTurnDiffTracker, - client_session: &mut ModelClientSession, - turn_metadata_header: Option<&str>, - input: Vec, - explicitly_enabled_connectors: &HashSet, - skills_outcome: Option<&SkillLoadOutcome>, - server_model_warning_emitted_for_turn: &mut bool, - cancellation_token: CancellationToken, -) -> CodexResult { - let router = built_tools( - sess.as_ref(), - turn_context.as_ref(), - &input, - explicitly_enabled_connectors, - skills_outcome, - &cancellation_token, - ) - .await?; - - let base_instructions = sess.get_base_instructions().await; - - let tool_runtime = ToolCallRuntime::new( - Arc::clone(&router), - Arc::clone(&sess), - Arc::clone(&turn_context), - Arc::clone(&turn_diff_tracker), - ); - let _code_mode_worker = sess - .services - .code_mode_service - .start_turn_worker( - &sess, - &turn_context, - Arc::clone(&router), - Arc::clone(&turn_diff_tracker), - ) - .await; - let mut retries = 0; - let mut initial_input = Some(input); - loop { - let prompt_input = if let Some(input) = initial_input.take() { - input - } else { - sess.clone_history() - .await - .for_prompt(&turn_context.model_info.input_modalities) - }; - let prompt = build_prompt( - prompt_input, - router.as_ref(), - turn_context.as_ref(), - base_instructions.clone(), - ); - let err = match try_run_sampling_request( - tool_runtime.clone(), - Arc::clone(&sess), - Arc::clone(&turn_context), - client_session, - turn_metadata_header, - Arc::clone(&turn_diff_tracker), - server_model_warning_emitted_for_turn, - &prompt, - cancellation_token.child_token(), - ) - .await - { - Ok(output) => { - return Ok(output); - } - Err(CodexErr::ContextWindowExceeded) => { - sess.set_total_tokens_full(&turn_context).await; - return Err(CodexErr::ContextWindowExceeded); - } - Err(CodexErr::UsageLimitReached(e)) => { - let rate_limits = e.rate_limits.clone(); - if let Some(rate_limits) = rate_limits { - sess.update_rate_limits(&turn_context, *rate_limits).await; - } - return Err(CodexErr::UsageLimitReached(e)); - } - Err(err) => err, - }; - - if !err.is_retryable() { - return Err(err); - } - - // Use the configured provider-specific stream retry budget. - let max_retries = turn_context.provider.stream_max_retries(); - if retries >= max_retries - && client_session.try_switch_fallback_transport( - &turn_context.session_telemetry, - &turn_context.model_info, - ) - { - sess.send_event( - &turn_context, - EventMsg::Warning(WarningEvent { - message: format!("Falling back from WebSockets to HTTPS transport. {err:#}"), - }), - ) - .await; - retries = 0; - continue; - } - if retries < max_retries { - retries += 1; - let delay = match &err { - CodexErr::Stream(_, requested_delay) => { - requested_delay.unwrap_or_else(|| backoff(retries)) - } - _ => backoff(retries), - }; - warn!( - "stream disconnected - retrying sampling request ({retries}/{max_retries} in {delay:?})...", - ); - - // In release builds, hide the first websocket retry notification to reduce noisy - // transient reconnect messages. In debug builds, keep full visibility for diagnosis. - let report_error = retries > 1 - || cfg!(debug_assertions) - || !sess.services.model_client.responses_websocket_enabled(); - if report_error { - // Surface retry information to any UI/front‑end so the - // user understands what is happening instead of staring - // at a seemingly frozen screen. - sess.notify_stream_error( - &turn_context, - format!("Reconnecting... {retries}/{max_retries}"), - err, - ) - .await; - } - tokio::time::sleep(delay).await; - } else { - return Err(err); - } - } -} - -pub(crate) async fn built_tools( - sess: &Session, - turn_context: &TurnContext, - input: &[ResponseItem], - explicitly_enabled_connectors: &HashSet, - skills_outcome: Option<&SkillLoadOutcome>, - cancellation_token: &CancellationToken, -) -> CodexResult> { - let mcp_connection_manager = sess.services.mcp_connection_manager.read().await; - let has_mcp_servers = mcp_connection_manager.has_servers(); - let all_mcp_tools = mcp_connection_manager - .list_all_tools() - .or_cancel(cancellation_token) - .await?; - drop(mcp_connection_manager); - let loaded_plugins = sess - .services - .plugins_manager - .plugins_for_config(&turn_context.config) - .await; - - let mut effective_explicitly_enabled_connectors = explicitly_enabled_connectors.clone(); - effective_explicitly_enabled_connectors.extend(sess.get_connector_selection().await); - - let apps_enabled = turn_context.apps_enabled(); - let accessible_connectors = - apps_enabled.then(|| connectors::accessible_connectors_from_mcp_tools(&all_mcp_tools)); - let accessible_connectors_with_enabled_state = - accessible_connectors.as_ref().map(|connectors| { - connectors::with_app_enabled_state(connectors.clone(), &turn_context.config) - }); - let connectors = if apps_enabled { - let connectors = codex_connectors::merge::merge_plugin_connectors_with_accessible( - loaded_plugins - .effective_apps() - .into_iter() - .map(|connector_id| connector_id.0), - accessible_connectors.clone().unwrap_or_default(), - ); - Some(connectors::with_app_enabled_state( - connectors, - &turn_context.config, - )) - } else { - None - }; - let auth = sess.services.auth_manager.auth().await; - let discoverable_tools = if apps_enabled && turn_context.tools_config.tool_suggest { - if let Some(accessible_connectors) = accessible_connectors_with_enabled_state.as_ref() { - match connectors::list_tool_suggest_discoverable_tools_with_auth( - &turn_context.config, - auth.as_ref(), - accessible_connectors.as_slice(), - ) - .await - .map(|discoverable_tools| { - filter_tool_suggest_discoverable_tools_for_client( - discoverable_tools, - turn_context.app_server_client_name.as_deref(), - ) - }) { - Ok(discoverable_tools) if discoverable_tools.is_empty() => None, - Ok(discoverable_tools) => Some(discoverable_tools), - Err(err) => { - warn!("failed to load discoverable tool suggestions: {err:#}"); - None - } - } - } else { - None - } - } else { - None - }; - - let explicitly_enabled = if let Some(connectors) = connectors.as_ref() { - let skill_name_counts_lower = skills_outcome.map_or_else(HashMap::new, |outcome| { - build_skill_name_counts(&outcome.skills, &outcome.disabled_paths).1 - }); - - filter_connectors_for_input( - connectors, - input, - &effective_explicitly_enabled_connectors, - &skill_name_counts_lower, - ) - } else { - Vec::new() - }; - let mcp_tool_exposure = build_mcp_tool_exposure( - &all_mcp_tools, - connectors.as_deref(), - explicitly_enabled.as_slice(), - &turn_context.config, - &turn_context.tools_config, - ); - let mcp_tools = has_mcp_servers.then_some(mcp_tool_exposure.direct_tools); - let deferred_mcp_tools = mcp_tool_exposure.deferred_tools; - let unavailable_called_tools = if turn_context - .config - .features - .enabled(Feature::UnavailableDummyTools) - { - let exposed_tool_names = mcp_tools - .iter() - .chain(deferred_mcp_tools.iter()) - .flat_map(|tools| tools.keys().map(String::as_str)) - .collect::>(); - collect_unavailable_called_tools(input, &exposed_tool_names) - } else { - Vec::new() - }; - - let parallel_mcp_server_names = turn_context - .config - .mcp_servers - .get() - .iter() - .filter_map(|(server_name, server_config)| { - server_config - .supports_parallel_tool_calls - .then_some(server_name.clone()) - }) - .collect::>(); - - Ok(Arc::new(ToolRouter::from_config( - &turn_context.tools_config, - ToolRouterParams { - mcp_tools, - deferred_mcp_tools, - unavailable_called_tools, - parallel_mcp_server_names, - discoverable_tools, - dynamic_tools: turn_context.dynamic_tools.as_slice(), - }, - ))) -} - -#[derive(Debug)] -struct SamplingRequestResult { - needs_follow_up: bool, - last_agent_message: Option, -} - -/// Ephemeral per-response state for streaming a single proposed plan. -/// This is intentionally not persisted or stored in session/state since it -/// only exists while a response is actively streaming. The final plan text -/// is extracted from the completed assistant message. -/// Tracks a single proposed plan item across a streaming response. -struct ProposedPlanItemState { - item_id: String, - started: bool, - completed: bool, -} - -/// Aggregated state used only while streaming a plan-mode response. -/// Includes per-item parsers, deferred agent message bookkeeping, and the plan item lifecycle. -struct PlanModeStreamState { - /// Agent message items started by the model but deferred until we see non-plan text. - pending_agent_message_items: HashMap, - /// Agent message items whose start notification has been emitted. - started_agent_message_items: HashSet, - /// Leading whitespace buffered until we see non-whitespace text for an item. - leading_whitespace_by_item: HashMap, - /// Tracks plan item lifecycle while streaming plan output. - plan_item_state: ProposedPlanItemState, -} - -impl PlanModeStreamState { - fn new(turn_id: &str) -> Self { - Self { - pending_agent_message_items: HashMap::new(), - started_agent_message_items: HashSet::new(), - leading_whitespace_by_item: HashMap::new(), - plan_item_state: ProposedPlanItemState::new(turn_id), - } - } -} - -#[derive(Debug, Default)] -struct AssistantMessageStreamParsers { - plan_mode: bool, - parsers_by_item: HashMap, -} - -type ParsedAssistantTextDelta = AssistantTextChunk; - -impl AssistantMessageStreamParsers { - fn new(plan_mode: bool) -> Self { - Self { - plan_mode, - parsers_by_item: HashMap::new(), - } - } - - fn parser_mut(&mut self, item_id: &str) -> &mut AssistantTextStreamParser { - let plan_mode = self.plan_mode; - self.parsers_by_item - .entry(item_id.to_string()) - .or_insert_with(|| AssistantTextStreamParser::new(plan_mode)) - } - - fn seed_item_text(&mut self, item_id: &str, text: &str) -> ParsedAssistantTextDelta { - if text.is_empty() { - return ParsedAssistantTextDelta::default(); - } - self.parser_mut(item_id).push_str(text) - } - - fn parse_delta(&mut self, item_id: &str, delta: &str) -> ParsedAssistantTextDelta { - self.parser_mut(item_id).push_str(delta) - } - - fn finish_item(&mut self, item_id: &str) -> ParsedAssistantTextDelta { - let Some(mut parser) = self.parsers_by_item.remove(item_id) else { - return ParsedAssistantTextDelta::default(); - }; - parser.finish() - } - - fn drain_finished(&mut self) -> Vec<(String, ParsedAssistantTextDelta)> { - let parsers_by_item = std::mem::take(&mut self.parsers_by_item); - parsers_by_item - .into_iter() - .map(|(item_id, mut parser)| (item_id, parser.finish())) - .collect() - } -} - -impl ProposedPlanItemState { - fn new(turn_id: &str) -> Self { - Self { - item_id: format!("{turn_id}-plan"), - started: false, - completed: false, - } - } - - async fn start(&mut self, sess: &Session, turn_context: &TurnContext) { - if self.started || self.completed { - return; - } - self.started = true; - let item = TurnItem::Plan(PlanItem { - id: self.item_id.clone(), - text: String::new(), - }); - sess.emit_turn_item_started(turn_context, &item).await; - } - - async fn push_delta(&mut self, sess: &Session, turn_context: &TurnContext, delta: &str) { - if self.completed { - return; - } - if delta.is_empty() { - return; - } - let event = PlanDeltaEvent { - thread_id: sess.conversation_id.to_string(), - turn_id: turn_context.sub_id.clone(), - item_id: self.item_id.clone(), - delta: delta.to_string(), - }; - sess.send_event(turn_context, EventMsg::PlanDelta(event)) - .await; - } - - async fn complete_with_text( - &mut self, - sess: &Session, - turn_context: &TurnContext, - text: String, - ) { - if self.completed || !self.started { - return; - } - self.completed = true; - let item = TurnItem::Plan(PlanItem { - id: self.item_id.clone(), - text, - }); - sess.emit_turn_item_completed(turn_context, item).await; - } -} - -/// In plan mode we defer agent message starts until the parser emits non-plan -/// text. The parser buffers each line until it can rule out a tag prefix, so -/// plan-only outputs never show up as empty assistant messages. -async fn maybe_emit_pending_agent_message_start( - sess: &Session, - turn_context: &TurnContext, - state: &mut PlanModeStreamState, - item_id: &str, -) { - if state.started_agent_message_items.contains(item_id) { - return; - } - if let Some(item) = state.pending_agent_message_items.remove(item_id) { - sess.emit_turn_item_started(turn_context, &item).await; - state - .started_agent_message_items - .insert(item_id.to_string()); - } -} - -/// Agent messages are text-only today; concatenate all text entries. -fn agent_message_text(item: &codex_protocol::items::AgentMessageItem) -> String { - item.content - .iter() - .map(|entry| match entry { - codex_protocol::items::AgentMessageContent::Text { text } => text.as_str(), - }) - .collect() -} - -fn realtime_text_for_event(msg: &EventMsg) -> Option { - match msg { - EventMsg::AgentMessage(event) => Some(event.message.clone()), - EventMsg::ItemCompleted(event) => match &event.item { - TurnItem::AgentMessage(item) => Some(agent_message_text(item)), - _ => None, - }, - EventMsg::Error(_) - | EventMsg::Warning(_) - | EventMsg::RealtimeConversationStarted(_) - | EventMsg::RealtimeConversationSdp(_) - | EventMsg::RealtimeConversationRealtime(_) - | EventMsg::RealtimeConversationClosed(_) - | EventMsg::ModelReroute(_) - | EventMsg::ContextCompacted(_) - | EventMsg::ThreadRolledBack(_) - | EventMsg::TurnStarted(_) - | EventMsg::TurnComplete(_) - | EventMsg::TokenCount(_) - | EventMsg::UserMessage(_) - | EventMsg::AgentMessageDelta(_) - | EventMsg::AgentReasoning(_) - | EventMsg::AgentReasoningDelta(_) - | EventMsg::AgentReasoningRawContent(_) - | EventMsg::AgentReasoningRawContentDelta(_) - | EventMsg::AgentReasoningSectionBreak(_) - | EventMsg::SessionConfigured(_) - | EventMsg::ThreadNameUpdated(_) - | EventMsg::McpStartupUpdate(_) - | EventMsg::McpStartupComplete(_) - | EventMsg::McpToolCallBegin(_) - | EventMsg::McpToolCallEnd(_) - | EventMsg::WebSearchBegin(_) - | EventMsg::WebSearchEnd(_) - | EventMsg::ExecCommandBegin(_) - | EventMsg::ExecCommandOutputDelta(_) - | EventMsg::TerminalInteraction(_) - | EventMsg::ExecCommandEnd(_) - | EventMsg::PatchApplyBegin(_) - | EventMsg::PatchApplyEnd(_) - | EventMsg::ViewImageToolCall(_) - | EventMsg::ImageGenerationBegin(_) - | EventMsg::ImageGenerationEnd(_) - | EventMsg::ExecApprovalRequest(_) - | EventMsg::RequestPermissions(_) - | EventMsg::RequestUserInput(_) - | EventMsg::DynamicToolCallRequest(_) - | EventMsg::DynamicToolCallResponse(_) - | EventMsg::GuardianAssessment(_) - | EventMsg::ElicitationRequest(_) - | EventMsg::ApplyPatchApprovalRequest(_) - | EventMsg::DeprecationNotice(_) - | EventMsg::BackgroundEvent(_) - | EventMsg::UndoStarted(_) - | EventMsg::UndoCompleted(_) - | EventMsg::StreamError(_) - | EventMsg::TurnDiff(_) - | EventMsg::GetHistoryEntryResponse(_) - | EventMsg::McpListToolsResponse(_) - | EventMsg::ListSkillsResponse(_) - | EventMsg::RealtimeConversationListVoicesResponse(_) - | EventMsg::SkillsUpdateAvailable - | EventMsg::PlanUpdate(_) - | EventMsg::TurnAborted(_) - | EventMsg::ShutdownComplete - | EventMsg::EnteredReviewMode(_) - | EventMsg::ExitedReviewMode(_) - | EventMsg::RawResponseItem(_) - | EventMsg::ItemStarted(_) - | EventMsg::HookStarted(_) - | EventMsg::HookCompleted(_) - | EventMsg::AgentMessageContentDelta(_) - | EventMsg::PlanDelta(_) - | EventMsg::ReasoningContentDelta(_) - | EventMsg::ReasoningRawContentDelta(_) - | EventMsg::CollabAgentSpawnBegin(_) - | EventMsg::CollabAgentSpawnEnd(_) - | EventMsg::CollabAgentInteractionBegin(_) - | EventMsg::CollabAgentInteractionEnd(_) - | EventMsg::CollabWaitingBegin(_) - | EventMsg::CollabWaitingEnd(_) - | EventMsg::CollabCloseBegin(_) - | EventMsg::CollabCloseEnd(_) - | EventMsg::CollabResumeBegin(_) - | EventMsg::CollabResumeEnd(_) => None, - } -} - -/// Split the stream into normal assistant text vs. proposed plan content. -/// Normal text becomes AgentMessage deltas; plan content becomes PlanDelta + -/// TurnItem::Plan. -async fn handle_plan_segments( - sess: &Session, - turn_context: &TurnContext, - state: &mut PlanModeStreamState, - item_id: &str, - segments: Vec, -) { - for segment in segments { - match segment { - ProposedPlanSegment::Normal(delta) => { - if delta.is_empty() { - continue; - } - let has_non_whitespace = delta.chars().any(|ch| !ch.is_whitespace()); - if !has_non_whitespace && !state.started_agent_message_items.contains(item_id) { - let entry = state - .leading_whitespace_by_item - .entry(item_id.to_string()) - .or_default(); - entry.push_str(&delta); - continue; - } - let delta = if !state.started_agent_message_items.contains(item_id) { - if let Some(prefix) = state.leading_whitespace_by_item.remove(item_id) { - format!("{prefix}{delta}") - } else { - delta - } - } else { - delta - }; - maybe_emit_pending_agent_message_start(sess, turn_context, state, item_id).await; - - let event = AgentMessageContentDeltaEvent { - thread_id: sess.conversation_id.to_string(), - turn_id: turn_context.sub_id.clone(), - item_id: item_id.to_string(), - delta, - }; - sess.send_event(turn_context, EventMsg::AgentMessageContentDelta(event)) - .await; - } - ProposedPlanSegment::ProposedPlanStart => { - if !state.plan_item_state.completed { - state.plan_item_state.start(sess, turn_context).await; - } - } - ProposedPlanSegment::ProposedPlanDelta(delta) => { - if !state.plan_item_state.completed { - if !state.plan_item_state.started { - state.plan_item_state.start(sess, turn_context).await; - } - state - .plan_item_state - .push_delta(sess, turn_context, &delta) - .await; - } - } - ProposedPlanSegment::ProposedPlanEnd => {} - } - } -} - -async fn emit_streamed_assistant_text_delta( - sess: &Session, - turn_context: &TurnContext, - plan_mode_state: Option<&mut PlanModeStreamState>, - item_id: &str, - parsed: ParsedAssistantTextDelta, -) { - if parsed.is_empty() { - return; - } - if !parsed.citations.is_empty() { - // Citation extraction is intentionally local for now; we strip citations from display text - // but do not yet surface them in protocol events. - let _citations = parsed.citations; - } - if let Some(state) = plan_mode_state { - if !parsed.plan_segments.is_empty() { - handle_plan_segments(sess, turn_context, state, item_id, parsed.plan_segments).await; - } - return; - } - if parsed.visible_text.is_empty() { - return; - } - let event = AgentMessageContentDeltaEvent { - thread_id: sess.conversation_id.to_string(), - turn_id: turn_context.sub_id.clone(), - item_id: item_id.to_string(), - delta: parsed.visible_text, - }; - sess.send_event(turn_context, EventMsg::AgentMessageContentDelta(event)) - .await; -} - -/// Flush buffered assistant text parser state when an assistant message item ends. -async fn flush_assistant_text_segments_for_item( - sess: &Session, - turn_context: &TurnContext, - plan_mode_state: Option<&mut PlanModeStreamState>, - parsers: &mut AssistantMessageStreamParsers, - item_id: &str, -) { - let parsed = parsers.finish_item(item_id); - emit_streamed_assistant_text_delta(sess, turn_context, plan_mode_state, item_id, parsed).await; -} - -/// Flush any remaining buffered assistant text parser state at response completion. -async fn flush_assistant_text_segments_all( - sess: &Session, - turn_context: &TurnContext, - mut plan_mode_state: Option<&mut PlanModeStreamState>, - parsers: &mut AssistantMessageStreamParsers, -) { - for (item_id, parsed) in parsers.drain_finished() { - emit_streamed_assistant_text_delta( - sess, - turn_context, - plan_mode_state.as_deref_mut(), - &item_id, - parsed, - ) - .await; - } -} - -/// Emit completion for plan items by parsing the finalized assistant message. -async fn maybe_complete_plan_item_from_message( - sess: &Session, - turn_context: &TurnContext, - state: &mut PlanModeStreamState, - item: &ResponseItem, -) { - if let ResponseItem::Message { role, content, .. } = item - && role == "assistant" - { - let mut text = String::new(); - for entry in content { - if let ContentItem::OutputText { text: chunk } = entry { - text.push_str(chunk); - } - } - if let Some(plan_text) = extract_proposed_plan_text(&text) { - let (plan_text, _citations) = strip_citations(&plan_text); - if !state.plan_item_state.started { - state.plan_item_state.start(sess, turn_context).await; - } - state - .plan_item_state - .complete_with_text(sess, turn_context, plan_text) - .await; - } - } -} - -/// Emit a completed agent message in plan mode, respecting deferred starts. -async fn emit_agent_message_in_plan_mode( - sess: &Session, - turn_context: &TurnContext, - agent_message: codex_protocol::items::AgentMessageItem, - state: &mut PlanModeStreamState, -) { - let agent_message_id = agent_message.id.clone(); - let text = agent_message_text(&agent_message); - if text.trim().is_empty() { - state.pending_agent_message_items.remove(&agent_message_id); - state.started_agent_message_items.remove(&agent_message_id); - return; - } - - maybe_emit_pending_agent_message_start(sess, turn_context, state, &agent_message_id).await; - - if !state - .started_agent_message_items - .contains(&agent_message_id) - { - let start_item = state - .pending_agent_message_items - .remove(&agent_message_id) - .unwrap_or_else(|| { - TurnItem::AgentMessage(codex_protocol::items::AgentMessageItem { - id: agent_message_id.clone(), - content: Vec::new(), - phase: None, - memory_citation: None, - }) - }); - sess.emit_turn_item_started(turn_context, &start_item).await; - state - .started_agent_message_items - .insert(agent_message_id.clone()); - } - - sess.emit_turn_item_completed(turn_context, TurnItem::AgentMessage(agent_message)) - .await; - state.started_agent_message_items.remove(&agent_message_id); -} - -/// Emit completion for a plan-mode turn item, handling agent messages specially. -async fn emit_turn_item_in_plan_mode( - sess: &Session, - turn_context: &TurnContext, - turn_item: TurnItem, - previously_active_item: Option<&TurnItem>, - state: &mut PlanModeStreamState, -) { - match turn_item { - TurnItem::AgentMessage(agent_message) => { - emit_agent_message_in_plan_mode(sess, turn_context, agent_message, state).await; - } - _ => { - if previously_active_item.is_none() { - sess.emit_turn_item_started(turn_context, &turn_item).await; - } - sess.emit_turn_item_completed(turn_context, turn_item).await; - } - } -} - -/// Handle a completed assistant response item in plan mode, returning true if handled. -async fn handle_assistant_item_done_in_plan_mode( - sess: &Session, - turn_context: &TurnContext, - item: &ResponseItem, - state: &mut PlanModeStreamState, - previously_active_item: Option<&TurnItem>, - last_agent_message: &mut Option, -) -> bool { - if let ResponseItem::Message { role, .. } = item - && role == "assistant" - { - maybe_complete_plan_item_from_message(sess, turn_context, state, item).await; - - if let Some(turn_item) = - handle_non_tool_response_item(sess, turn_context, item, /*plan_mode*/ true).await - { - emit_turn_item_in_plan_mode( - sess, - turn_context, - turn_item, - previously_active_item, - state, - ) - .await; - } - - record_completed_response_item(sess, turn_context, item).await; - if let Some(agent_message) = last_assistant_message_from_item(item, /*plan_mode*/ true) { - *last_agent_message = Some(agent_message); - } - return true; - } - false -} - -async fn drain_in_flight( - in_flight: &mut FuturesOrdered>>, - sess: Arc, - turn_context: Arc, -) -> CodexResult<()> { - while let Some(res) = in_flight.next().await { - match res { - Ok(response_input) => { - let response_item = response_input.into(); - sess.record_conversation_items(&turn_context, std::slice::from_ref(&response_item)) - .await; - mark_thread_memory_mode_polluted_if_external_context( - sess.as_ref(), - turn_context.as_ref(), - &response_item, - ) - .await; - } - Err(err) => { - error_or_panic(format!("in-flight tool future failed during drain: {err}")); - } - } - } - Ok(()) -} - -#[allow(clippy::too_many_arguments)] -#[instrument(level = "trace", - skip_all, - fields( - turn_id = %turn_context.sub_id, - model = %turn_context.model_info.slug - ) -)] -async fn try_run_sampling_request( - tool_runtime: ToolCallRuntime, - sess: Arc, - turn_context: Arc, - client_session: &mut ModelClientSession, - turn_metadata_header: Option<&str>, - turn_diff_tracker: SharedTurnDiffTracker, - server_model_warning_emitted_for_turn: &mut bool, - prompt: &Prompt, - cancellation_token: CancellationToken, -) -> CodexResult { - feedback_tags!( - model = turn_context.model_info.slug.clone(), - approval_policy = turn_context.approval_policy.value(), - sandbox_policy = turn_context.sandbox_policy.get(), - effort = turn_context.reasoning_effort, - auth_mode = sess.services.auth_manager.auth_mode(), - features = sess.features.enabled_features(), - ); - let mut stream = client_session - .stream( - prompt, - &turn_context.model_info, - &turn_context.session_telemetry, - turn_context.reasoning_effort, - turn_context.reasoning_summary, - turn_context.config.service_tier, - turn_metadata_header, - ) - .instrument(trace_span!("stream_request")) - .or_cancel(&cancellation_token) - .await??; - let mut in_flight: FuturesOrdered>> = - FuturesOrdered::new(); - let mut needs_follow_up = false; - let mut last_agent_message: Option = None; - let mut active_item: Option = None; - let mut should_emit_turn_diff = false; - let plan_mode = turn_context.collaboration_mode.mode == ModeKind::Plan; - let mut assistant_message_stream_parsers = AssistantMessageStreamParsers::new(plan_mode); - let mut plan_mode_state = plan_mode.then(|| PlanModeStreamState::new(&turn_context.sub_id)); - let receiving_span = trace_span!("receiving_stream"); - let outcome: CodexResult = loop { - let handle_responses = trace_span!( - parent: &receiving_span, - "handle_responses", - otel.name = field::Empty, - tool_name = field::Empty, - from = field::Empty, - ); - - let event = match stream - .next() - .instrument(trace_span!(parent: &handle_responses, "receiving")) - .or_cancel(&cancellation_token) - .await - { - Ok(event) => event, - Err(codex_async_utils::CancelErr::Cancelled) => break Err(CodexErr::TurnAborted), - }; - - let event = match event { - Some(Ok(event)) => event, - Some(Err(err)) => break Err(err), - None => { - break Err(CodexErr::Stream( - "stream closed before response.completed".into(), - None, - )); - } - }; - - sess.services - .session_telemetry - .record_responses(&handle_responses, &event); - record_turn_ttft_metric(&turn_context, &event).await; - - match event { - ResponseEvent::Created => {} - ResponseEvent::OutputItemDone(item) => { - let previously_active_item = active_item.take(); - if let Some(previous) = previously_active_item.as_ref() - && matches!(previous, TurnItem::AgentMessage(_)) - { - let item_id = previous.id(); - flush_assistant_text_segments_for_item( - &sess, - &turn_context, - plan_mode_state.as_mut(), - &mut assistant_message_stream_parsers, - &item_id, - ) - .await; - } - if let Some(state) = plan_mode_state.as_mut() - && handle_assistant_item_done_in_plan_mode( - &sess, - &turn_context, - &item, - state, - previously_active_item.as_ref(), - &mut last_agent_message, - ) - .await - { - continue; - } - - let mut ctx = HandleOutputCtx { - sess: sess.clone(), - turn_context: turn_context.clone(), - tool_runtime: tool_runtime.clone(), - cancellation_token: cancellation_token.child_token(), - }; - - let output_result = - match handle_output_item_done(&mut ctx, item, previously_active_item) - .instrument(handle_responses) - .await - { - Ok(output_result) => output_result, - Err(err) => break Err(err), - }; - if let Some(tool_future) = output_result.tool_future { - in_flight.push_back(tool_future); - } - if let Some(agent_message) = output_result.last_agent_message { - last_agent_message = Some(agent_message); - } - needs_follow_up |= output_result.needs_follow_up; - } - ResponseEvent::OutputItemAdded(item) => { - if let Some(turn_item) = handle_non_tool_response_item( - sess.as_ref(), - turn_context.as_ref(), - &item, - plan_mode, - ) - .await - { - let mut turn_item = turn_item; - let mut seeded_parsed: Option = None; - let mut seeded_item_id: Option = None; - if matches!(turn_item, TurnItem::AgentMessage(_)) - && let Some(raw_text) = raw_assistant_output_text_from_item(&item) - { - let item_id = turn_item.id(); - let mut seeded = - assistant_message_stream_parsers.seed_item_text(&item_id, &raw_text); - if let TurnItem::AgentMessage(agent_message) = &mut turn_item { - agent_message.content = - vec![codex_protocol::items::AgentMessageContent::Text { - text: if plan_mode { - String::new() - } else { - std::mem::take(&mut seeded.visible_text) - }, - }]; - } - seeded_parsed = plan_mode.then_some(seeded); - seeded_item_id = Some(item_id); - } - if let Some(state) = plan_mode_state.as_mut() - && matches!(turn_item, TurnItem::AgentMessage(_)) - { - let item_id = turn_item.id(); - state - .pending_agent_message_items - .insert(item_id, turn_item.clone()); - } else { - sess.emit_turn_item_started(&turn_context, &turn_item).await; - } - if let (Some(state), Some(item_id), Some(parsed)) = ( - plan_mode_state.as_mut(), - seeded_item_id.as_deref(), - seeded_parsed, - ) { - emit_streamed_assistant_text_delta( - &sess, - &turn_context, - Some(state), - item_id, - parsed, - ) - .await; - } - active_item = Some(turn_item); - } - } - ResponseEvent::ServerModel(server_model) => { - if !*server_model_warning_emitted_for_turn - && sess - .maybe_warn_on_server_model_mismatch(&turn_context, server_model) - .await - { - *server_model_warning_emitted_for_turn = true; - } - } - ResponseEvent::ServerReasoningIncluded(included) => { - sess.set_server_reasoning_included(included).await; - } - ResponseEvent::RateLimits(snapshot) => { - // Update internal state with latest rate limits, but defer sending until - // token usage is available to avoid duplicate TokenCount events. - sess.update_rate_limits(&turn_context, snapshot).await; - } - ResponseEvent::ModelsEtag(etag) => { - // Update internal state with latest models etag - sess.services.models_manager.refresh_if_new_etag(etag).await; - } - ResponseEvent::Completed { - response_id: _, - token_usage, - } => { - flush_assistant_text_segments_all( - &sess, - &turn_context, - plan_mode_state.as_mut(), - &mut assistant_message_stream_parsers, - ) - .await; - sess.update_token_usage_info(&turn_context, token_usage.as_ref()) - .await; - should_emit_turn_diff = true; - - break Ok(SamplingRequestResult { - needs_follow_up, - last_agent_message, - }); - } - ResponseEvent::OutputTextDelta(delta) => { - // In review child threads, suppress assistant text deltas; the - // UI will show a selection popup from the final ReviewOutput. - if let Some(active) = active_item.as_ref() { - let item_id = active.id(); - if matches!(active, TurnItem::AgentMessage(_)) { - let parsed = assistant_message_stream_parsers.parse_delta(&item_id, &delta); - emit_streamed_assistant_text_delta( - &sess, - &turn_context, - plan_mode_state.as_mut(), - &item_id, - parsed, - ) - .await; - } else { - let event = AgentMessageContentDeltaEvent { - thread_id: sess.conversation_id.to_string(), - turn_id: turn_context.sub_id.clone(), - item_id, - delta, - }; - sess.send_event(&turn_context, EventMsg::AgentMessageContentDelta(event)) - .await; - } - } else { - error_or_panic("OutputTextDelta without active item".to_string()); - } - } - ResponseEvent::ReasoningSummaryDelta { - delta, - summary_index, - } => { - if let Some(active) = active_item.as_ref() { - let event = ReasoningContentDeltaEvent { - thread_id: sess.conversation_id.to_string(), - turn_id: turn_context.sub_id.clone(), - item_id: active.id(), - delta, - summary_index, - }; - sess.send_event(&turn_context, EventMsg::ReasoningContentDelta(event)) - .await; - } else { - error_or_panic("ReasoningSummaryDelta without active item".to_string()); - } - } - ResponseEvent::ReasoningSummaryPartAdded { summary_index } => { - if let Some(active) = active_item.as_ref() { - let event = - EventMsg::AgentReasoningSectionBreak(AgentReasoningSectionBreakEvent { - item_id: active.id(), - summary_index, - }); - sess.send_event(&turn_context, event).await; - } else { - error_or_panic("ReasoningSummaryPartAdded without active item".to_string()); - } - } - ResponseEvent::ReasoningContentDelta { - delta, - content_index, - } => { - if let Some(active) = active_item.as_ref() { - let event = ReasoningRawContentDeltaEvent { - thread_id: sess.conversation_id.to_string(), - turn_id: turn_context.sub_id.clone(), - item_id: active.id(), - delta, - content_index, - }; - sess.send_event(&turn_context, EventMsg::ReasoningRawContentDelta(event)) - .await; - } else { - error_or_panic("ReasoningRawContentDelta without active item".to_string()); - } - } - } - }; - - flush_assistant_text_segments_all( - &sess, - &turn_context, - plan_mode_state.as_mut(), - &mut assistant_message_stream_parsers, - ) - .await; - - drain_in_flight(&mut in_flight, sess.clone(), turn_context.clone()).await?; - - if cancellation_token.is_cancelled() { - return Err(CodexErr::TurnAborted); - } - - if should_emit_turn_diff { - let unified_diff = { - let mut tracker = turn_diff_tracker.lock().await; - tracker.get_unified_diff() - }; - if let Ok(Some(unified_diff)) = unified_diff { - let msg = EventMsg::TurnDiff(TurnDiffEvent { unified_diff }); - sess.clone().send_event(&turn_context, msg).await; - } - } - - outcome -} - -pub(super) fn get_last_assistant_message_from_turn(responses: &[ResponseItem]) -> Option { - for item in responses.iter().rev() { - if let Some(message) = last_assistant_message_from_item(item, /*plan_mode*/ false) { - return Some(message); - } - } - None -} - use crate::memories::prompts::build_memory_tool_developer_instructions; #[cfg(test)] pub(crate) use tests::make_session_and_context; diff --git a/codex-rs/core/src/codex/turn.rs b/codex-rs/core/src/codex/turn.rs new file mode 100644 index 0000000000..fa24bd6924 --- /dev/null +++ b/codex-rs/core/src/codex/turn.rs @@ -0,0 +1,2169 @@ +use std::collections::HashMap; +use std::collections::HashSet; +use std::sync::Arc; + +use crate::SkillInjections; +use crate::SkillLoadOutcome; +use crate::build_skill_injections; +use crate::client::ModelClientSession; +use crate::client_common::Prompt; +use crate::client_common::ResponseEvent; +use crate::codex::PreviousTurnSettings; +use crate::codex::Session; +use crate::codex::TurnContext; +use crate::collect_env_var_dependencies; +use crate::collect_explicit_skill_mentions; +use crate::compact::InitialContextInjection; +use crate::compact::collect_user_messages; +use crate::compact::run_inline_auto_compact_task; +use crate::compact::should_use_remote_compact_task; +use crate::compact_remote::run_inline_remote_auto_compact_task; +use crate::connectors; +use crate::feedback_tags; +use crate::hook_runtime::PendingInputHookDisposition; +use crate::hook_runtime::emit_hook_completed_events; +use crate::hook_runtime::inspect_pending_input; +use crate::hook_runtime::record_additional_contexts; +use crate::hook_runtime::record_pending_input; +use crate::hook_runtime::run_pending_session_start_hooks; +use crate::hook_runtime::run_user_prompt_submit_hooks; +use crate::injection::ToolMentionKind; +use crate::injection::app_id_from_path; +use crate::injection::tool_kind_for_path; +use crate::mcp_skill_dependencies::maybe_prompt_and_install_mcp_dependencies; +use crate::mcp_tool_exposure::build_mcp_tool_exposure; +use crate::mentions::build_connector_slug_counts; +use crate::mentions::build_skill_name_counts; +use crate::mentions::collect_explicit_app_ids; +use crate::mentions::collect_explicit_plugin_mentions; +use crate::mentions::collect_tool_mentions_from_messages; +use crate::parse_turn_item; +use crate::plugins::build_plugin_injections; +use crate::resolve_skill_dependencies_for_turn; +use crate::stream_events_utils::HandleOutputCtx; +use crate::stream_events_utils::handle_non_tool_response_item; +use crate::stream_events_utils::handle_output_item_done; +use crate::stream_events_utils::last_assistant_message_from_item; +use crate::stream_events_utils::mark_thread_memory_mode_polluted_if_external_context; +use crate::stream_events_utils::raw_assistant_output_text_from_item; +use crate::stream_events_utils::record_completed_response_item; +use crate::tools::ToolRouter; +use crate::tools::context::SharedTurnDiffTracker; +use crate::tools::parallel::ToolCallRuntime; +use crate::tools::router::ToolRouterParams; +use crate::turn_diff_tracker::TurnDiffTracker; +use crate::turn_timing::record_turn_ttft_metric; +use crate::unavailable_tool::collect_unavailable_called_tools; +use crate::util::backoff; +use crate::util::error_or_panic; +use codex_analytics::AppInvocation; +use codex_analytics::CompactionPhase; +use codex_analytics::CompactionReason; +use codex_analytics::InvocationType; +use codex_analytics::TurnResolvedConfigFact; +use codex_analytics::build_track_events_context; +use codex_async_utils::OrCancelExt; +use codex_features::Feature; +use codex_hooks::HookEvent; +use codex_hooks::HookEventAfterAgent; +use codex_hooks::HookPayload; +use codex_hooks::HookResult; +use codex_protocol::config_types::ModeKind; +use codex_protocol::error::CodexErr; +use codex_protocol::error::Result as CodexResult; +use codex_protocol::items::PlanItem; +use codex_protocol::items::TurnItem; +use codex_protocol::items::UserMessageItem; +use codex_protocol::items::build_hook_prompt_message; +use codex_protocol::models::BaseInstructions; +use codex_protocol::models::ContentItem; +use codex_protocol::models::ResponseInputItem; +use codex_protocol::models::ResponseItem; +use codex_protocol::protocol::AgentMessageContentDeltaEvent; +use codex_protocol::protocol::AgentReasoningSectionBreakEvent; +use codex_protocol::protocol::AskForApproval; +use codex_protocol::protocol::CodexErrorInfo; +use codex_protocol::protocol::ErrorEvent; +use codex_protocol::protocol::EventMsg; +use codex_protocol::protocol::PlanDeltaEvent; +use codex_protocol::protocol::ReasoningContentDeltaEvent; +use codex_protocol::protocol::ReasoningRawContentDeltaEvent; +use codex_protocol::protocol::TurnDiffEvent; +use codex_protocol::protocol::WarningEvent; +use codex_protocol::user_input::UserInput; +use codex_tools::filter_tool_suggest_discoverable_tools_for_client; +use codex_utils_stream_parser::AssistantTextChunk; +use codex_utils_stream_parser::AssistantTextStreamParser; +use codex_utils_stream_parser::ProposedPlanSegment; +use codex_utils_stream_parser::extract_proposed_plan_text; +use codex_utils_stream_parser::strip_citations; +use futures::future::BoxFuture; +use futures::prelude::*; +use futures::stream::FuturesOrdered; +use tokio_util::sync::CancellationToken; +use tracing::Instrument; +use tracing::error; +use tracing::field; +use tracing::info; +use tracing::instrument; +use tracing::trace; +use tracing::trace_span; +use tracing::warn; + +/// Takes a user message as input and runs a loop where, at each sampling request, the model +/// replies with either: +/// +/// - requested function calls +/// - an assistant message +/// +/// While it is possible for the model to return multiple of these items in a +/// single sampling request, in practice, we generally one item per sampling request: +/// +/// - If the model requests a function call, we execute it and send the output +/// back to the model in the next sampling request. +/// - If the model sends only an assistant message, we record it in the +/// conversation history and consider the turn complete. +/// +pub(crate) async fn run_turn( + sess: Arc, + turn_context: Arc, + input: Vec, + prewarmed_client_session: Option, + cancellation_token: CancellationToken, +) -> Option { + if input.is_empty() && !sess.has_pending_input().await { + return None; + } + + let model_info = turn_context.model_info.clone(); + let auto_compact_limit = model_info.auto_compact_token_limit().unwrap_or(i64::MAX); + let mut prewarmed_client_session = prewarmed_client_session; + // TODO(ccunningham): Pre-turn compaction runs before context updates and the + // new user message are recorded. Estimate pending incoming items (context + // diffs/full reinjection + user input) and trigger compaction preemptively + // when they would push the thread over the compaction threshold. + let pre_sampling_compacted = match run_pre_sampling_compact(&sess, &turn_context).await { + Ok(pre_sampling_compacted) => pre_sampling_compacted, + Err(_) => { + error!("Failed to run pre-sampling compact"); + return None; + } + }; + if pre_sampling_compacted && let Some(mut client_session) = prewarmed_client_session.take() { + client_session.reset_websocket_session(); + } + + let skills_outcome = Some(turn_context.turn_skills.outcome.as_ref()); + + sess.record_context_updates_and_set_reference_context_item(turn_context.as_ref()) + .await; + + let loaded_plugins = sess + .services + .plugins_manager + .plugins_for_config(&turn_context.config) + .await; + // Structured plugin:// mentions are resolved from the current session's + // enabled plugins, then converted into turn-scoped guidance below. + let mentioned_plugins = + collect_explicit_plugin_mentions(&input, loaded_plugins.capability_summaries()); + let mcp_tools = if turn_context.apps_enabled() || !mentioned_plugins.is_empty() { + // Plugin mentions need raw MCP/app inventory even when app tools + // are normally hidden so we can describe the plugin's currently + // usable capabilities for this turn. + match sess + .services + .mcp_connection_manager + .read() + .await + .list_all_tools() + .or_cancel(&cancellation_token) + .await + { + Ok(mcp_tools) => mcp_tools, + Err(_) if turn_context.apps_enabled() => return None, + Err(_) => HashMap::new(), + } + } else { + HashMap::new() + }; + let available_connectors = if turn_context.apps_enabled() { + let connectors = codex_connectors::merge::merge_plugin_connectors_with_accessible( + loaded_plugins + .effective_apps() + .into_iter() + .map(|connector_id| connector_id.0), + connectors::accessible_connectors_from_mcp_tools(&mcp_tools), + ); + connectors::with_app_enabled_state(connectors, &turn_context.config) + } else { + Vec::new() + }; + let connector_slug_counts = build_connector_slug_counts(&available_connectors); + let skill_name_counts_lower = skills_outcome + .as_ref() + .map_or_else(HashMap::new, |outcome| { + build_skill_name_counts(&outcome.skills, &outcome.disabled_paths).1 + }); + let mentioned_skills = skills_outcome.as_ref().map_or_else(Vec::new, |outcome| { + collect_explicit_skill_mentions( + &input, + &outcome.skills, + &outcome.disabled_paths, + &connector_slug_counts, + ) + }); + let config = turn_context.config.clone(); + if config + .features + .enabled(Feature::SkillEnvVarDependencyPrompt) + { + let env_var_dependencies = collect_env_var_dependencies(&mentioned_skills); + resolve_skill_dependencies_for_turn(&sess, &turn_context, &env_var_dependencies).await; + } + + maybe_prompt_and_install_mcp_dependencies( + sess.as_ref(), + turn_context.as_ref(), + &cancellation_token, + &mentioned_skills, + ) + .await; + + let session_telemetry = turn_context.session_telemetry.clone(); + let thread_id = sess.conversation_id.to_string(); + let tracking = build_track_events_context( + turn_context.model_info.slug.clone(), + thread_id, + turn_context.sub_id.clone(), + ); + let SkillInjections { + items: skill_items, + warnings: skill_warnings, + } = build_skill_injections( + &mentioned_skills, + skills_outcome, + Some(&session_telemetry), + &sess.services.analytics_events_client, + tracking.clone(), + ) + .await; + + for message in skill_warnings { + sess.send_event(&turn_context, EventMsg::Warning(WarningEvent { message })) + .await; + } + + let plugin_items = + build_plugin_injections(&mentioned_plugins, &mcp_tools, &available_connectors); + let mentioned_plugin_metadata = mentioned_plugins + .iter() + .filter_map(crate::plugins::PluginCapabilitySummary::telemetry_metadata) + .collect::>(); + + let mut explicitly_enabled_connectors = collect_explicit_app_ids(&input); + explicitly_enabled_connectors.extend(collect_explicit_app_ids_from_skill_items( + &skill_items, + &available_connectors, + &skill_name_counts_lower, + )); + let connector_names_by_id = available_connectors + .iter() + .map(|connector| (connector.id.as_str(), connector.name.as_str())) + .collect::>(); + let mentioned_app_invocations = explicitly_enabled_connectors + .iter() + .map(|connector_id| AppInvocation { + connector_id: Some(connector_id.clone()), + app_name: connector_names_by_id + .get(connector_id.as_str()) + .map(|name| (*name).to_string()), + invocation_type: Some(InvocationType::Explicit), + }) + .collect::>(); + + if run_pending_session_start_hooks(&sess, &turn_context).await { + return None; + } + let additional_contexts = if input.is_empty() { + Vec::new() + } else { + let initial_input_for_turn: ResponseInputItem = ResponseInputItem::from(input.clone()); + let response_item: ResponseItem = initial_input_for_turn.clone().into(); + let user_prompt_submit_outcome = run_user_prompt_submit_hooks( + &sess, + &turn_context, + UserMessageItem::new(&input).message(), + ) + .await; + if user_prompt_submit_outcome.should_stop { + record_additional_contexts( + &sess, + &turn_context, + user_prompt_submit_outcome.additional_contexts, + ) + .await; + return None; + } + sess.record_user_prompt_and_emit_turn_item(turn_context.as_ref(), &input, response_item) + .await; + user_prompt_submit_outcome.additional_contexts + }; + sess.services + .analytics_events_client + .track_app_mentioned(tracking.clone(), mentioned_app_invocations); + for plugin in mentioned_plugin_metadata { + sess.services + .analytics_events_client + .track_plugin_used(tracking.clone(), plugin); + } + sess.merge_connector_selection(explicitly_enabled_connectors.clone()) + .await; + record_additional_contexts(&sess, &turn_context, additional_contexts).await; + if !input.is_empty() { + // Track the previous-turn baseline from the regular user-turn path only so + // standalone tasks (compact/shell/review/undo) cannot suppress future + // model/realtime injections. + sess.set_previous_turn_settings(Some(PreviousTurnSettings { + model: turn_context.model_info.slug.clone(), + realtime_active: Some(turn_context.realtime_active), + })) + .await; + } + if let Err(error) = sess.ensure_agent_task_registered().await { + warn!(error = %error, "agent task registration failed"); + sess.send_event( + turn_context.as_ref(), + EventMsg::Error(ErrorEvent { + message: format!( + "Agent task registration failed. Please try again; Codex will attempt to register the task again on the next turn: {error}" + ), + codex_error_info: Some(CodexErrorInfo::Other), + }), + ) + .await; + return None; + } + + if !skill_items.is_empty() { + sess.record_conversation_items(&turn_context, &skill_items) + .await; + } + if !plugin_items.is_empty() { + sess.record_conversation_items(&turn_context, &plugin_items) + .await; + } + + track_turn_resolved_config_analytics(&sess, &turn_context, &input).await; + + let skills_outcome = Some(turn_context.turn_skills.outcome.as_ref()); + sess.maybe_start_ghost_snapshot(Arc::clone(&turn_context), cancellation_token.child_token()) + .await; + let mut last_agent_message: Option = None; + let mut stop_hook_active = false; + // Although from the perspective of codex.rs, TurnDiffTracker has the lifecycle of a Task which contains + // many turns, from the perspective of the user, it is a single turn. + let turn_diff_tracker = Arc::new(tokio::sync::Mutex::new(TurnDiffTracker::new())); + let mut server_model_warning_emitted_for_turn = false; + + // `ModelClientSession` is turn-scoped and caches WebSocket + sticky routing state, so we reuse + // one instance across retries within this turn. + let mut client_session = + prewarmed_client_session.unwrap_or_else(|| sess.services.model_client.new_session()); + // Pending input is drained into history before building the next model request. + // However, we defer that drain until after sampling in two cases: + // 1. At the start of a turn, so the fresh user prompt in `input` gets sampled first. + // 2. After auto-compact, when model/tool continuation needs to resume before any steer. + let mut can_drain_pending_input = input.is_empty(); + + loop { + if run_pending_session_start_hooks(&sess, &turn_context).await { + break; + } + + // Note that pending_input would be something like a message the user + // submitted through the UI while the model was running. Though the UI + // may support this, the model might not. + let pending_input = if can_drain_pending_input { + sess.get_pending_input().await + } else { + Vec::new() + }; + + let mut blocked_pending_input = false; + let mut blocked_pending_input_contexts = Vec::new(); + let mut requeued_pending_input = false; + let mut accepted_pending_input = Vec::new(); + if !pending_input.is_empty() { + let mut pending_input_iter = pending_input.into_iter(); + while let Some(pending_input_item) = pending_input_iter.next() { + match inspect_pending_input(&sess, &turn_context, pending_input_item).await { + PendingInputHookDisposition::Accepted(pending_input) => { + accepted_pending_input.push(*pending_input); + } + PendingInputHookDisposition::Blocked { + additional_contexts, + } => { + let remaining_pending_input = pending_input_iter.collect::>(); + if !remaining_pending_input.is_empty() { + let _ = sess.prepend_pending_input(remaining_pending_input).await; + requeued_pending_input = true; + } + blocked_pending_input_contexts = additional_contexts; + blocked_pending_input = true; + break; + } + } + } + } + + let has_accepted_pending_input = !accepted_pending_input.is_empty(); + for pending_input in accepted_pending_input { + record_pending_input(&sess, &turn_context, pending_input).await; + } + record_additional_contexts(&sess, &turn_context, blocked_pending_input_contexts).await; + + if blocked_pending_input && !has_accepted_pending_input { + if requeued_pending_input { + continue; + } + break; + } + + // Construct the input that we will send to the model. + let sampling_request_input: Vec = { + sess.clone_history() + .await + .for_prompt(&turn_context.model_info.input_modalities) + }; + + let sampling_request_input_messages = sampling_request_input + .iter() + .filter_map(|item| match parse_turn_item(item) { + Some(TurnItem::UserMessage(user_message)) => Some(user_message), + _ => None, + }) + .map(|user_message| user_message.message()) + .collect::>(); + let turn_metadata_header = turn_context.turn_metadata_state.current_header_value(); + match run_sampling_request( + Arc::clone(&sess), + Arc::clone(&turn_context), + Arc::clone(&turn_diff_tracker), + &mut client_session, + turn_metadata_header.as_deref(), + sampling_request_input, + &explicitly_enabled_connectors, + skills_outcome, + &mut server_model_warning_emitted_for_turn, + cancellation_token.child_token(), + ) + .await + { + Ok(sampling_request_output) => { + let SamplingRequestResult { + needs_follow_up: model_needs_follow_up, + last_agent_message: sampling_request_last_agent_message, + } = sampling_request_output; + can_drain_pending_input = true; + let has_pending_input = sess.has_pending_input().await; + let needs_follow_up = model_needs_follow_up || has_pending_input; + let total_usage_tokens = sess.get_total_token_usage().await; + let token_limit_reached = total_usage_tokens >= auto_compact_limit; + + let estimated_token_count = + sess.get_estimated_token_count(turn_context.as_ref()).await; + + trace!( + turn_id = %turn_context.sub_id, + total_usage_tokens, + estimated_token_count = ?estimated_token_count, + auto_compact_limit, + token_limit_reached, + model_needs_follow_up, + has_pending_input, + needs_follow_up, + "post sampling token usage" + ); + + // as long as compaction works well in getting us way below the token limit, we shouldn't worry about being in an infinite loop. + if token_limit_reached && needs_follow_up { + if run_auto_compact( + &sess, + &turn_context, + InitialContextInjection::BeforeLastUserMessage, + CompactionReason::ContextLimit, + CompactionPhase::MidTurn, + ) + .await + .is_err() + { + return None; + } + client_session.reset_websocket_session(); + can_drain_pending_input = !model_needs_follow_up; + continue; + } + + if !needs_follow_up { + last_agent_message = sampling_request_last_agent_message; + let stop_hook_permission_mode = match turn_context.approval_policy.value() { + AskForApproval::Never => "bypassPermissions", + AskForApproval::UnlessTrusted + | AskForApproval::OnFailure + | AskForApproval::OnRequest + | AskForApproval::Granular(_) => "default", + } + .to_string(); + let stop_request = codex_hooks::StopRequest { + session_id: sess.conversation_id, + turn_id: turn_context.sub_id.clone(), + cwd: turn_context.cwd.clone(), + transcript_path: sess.hook_transcript_path().await, + model: turn_context.model_info.slug.clone(), + permission_mode: stop_hook_permission_mode, + stop_hook_active, + last_assistant_message: last_agent_message.clone(), + }; + for run in sess.hooks().preview_stop(&stop_request) { + sess.send_event( + &turn_context, + EventMsg::HookStarted(codex_protocol::protocol::HookStartedEvent { + turn_id: Some(turn_context.sub_id.clone()), + run, + }), + ) + .await; + } + let stop_outcome = sess.hooks().run_stop(stop_request).await; + emit_hook_completed_events(&sess, &turn_context, stop_outcome.hook_events) + .await; + if stop_outcome.should_block { + if let Some(hook_prompt_message) = + build_hook_prompt_message(&stop_outcome.continuation_fragments) + { + sess.record_conversation_items( + &turn_context, + std::slice::from_ref(&hook_prompt_message), + ) + .await; + stop_hook_active = true; + continue; + } else { + sess.send_event( + &turn_context, + EventMsg::Warning(WarningEvent { + message: "Stop hook requested continuation without a prompt; ignoring the block.".to_string(), + }), + ) + .await; + } + } + if stop_outcome.should_stop { + break; + } + let hook_outcomes = sess + .hooks() + .dispatch(HookPayload { + session_id: sess.conversation_id, + cwd: turn_context.cwd.clone(), + client: turn_context.app_server_client_name.clone(), + triggered_at: chrono::Utc::now(), + hook_event: HookEvent::AfterAgent { + event: HookEventAfterAgent { + thread_id: sess.conversation_id, + turn_id: turn_context.sub_id.clone(), + input_messages: sampling_request_input_messages, + last_assistant_message: last_agent_message.clone(), + }, + }, + }) + .await; + + let mut abort_message = None; + for hook_outcome in hook_outcomes { + let hook_name = hook_outcome.hook_name; + match hook_outcome.result { + HookResult::Success => {} + HookResult::FailedContinue(error) => { + warn!( + turn_id = %turn_context.sub_id, + hook_name = %hook_name, + error = %error, + "after_agent hook failed; continuing" + ); + } + HookResult::FailedAbort(error) => { + let message = format!( + "after_agent hook '{hook_name}' failed and aborted turn completion: {error}" + ); + warn!( + turn_id = %turn_context.sub_id, + hook_name = %hook_name, + error = %error, + "after_agent hook failed; aborting operation" + ); + if abort_message.is_none() { + abort_message = Some(message); + } + } + } + } + if let Some(message) = abort_message { + sess.send_event( + &turn_context, + EventMsg::Error(ErrorEvent { + message, + codex_error_info: None, + }), + ) + .await; + return None; + } + break; + } + continue; + } + Err(CodexErr::TurnAborted) => { + // Aborted turn is reported via a different event. + break; + } + Err(CodexErr::InvalidImageRequest()) => { + { + let mut state = sess.state.lock().await; + error_or_panic( + "Invalid image detected; sanitizing tool output to prevent poisoning", + ); + if state.history.replace_last_turn_images("Invalid image") { + continue; + } + } + + let event = EventMsg::Error(ErrorEvent { + message: "Invalid image in your last message. Please remove it and try again." + .to_string(), + codex_error_info: Some(CodexErrorInfo::BadRequest), + }); + sess.send_event(&turn_context, event).await; + break; + } + Err(e) => { + info!("Turn error: {e:#}"); + let event = EventMsg::Error(e.to_error_event(/*message_prefix*/ None)); + sess.send_event(&turn_context, event).await; + // let the user continue the conversation + break; + } + } + } + + last_agent_message +} + +async fn track_turn_resolved_config_analytics( + sess: &Session, + turn_context: &TurnContext, + input: &[UserInput], +) { + if !sess.enabled(Feature::GeneralAnalytics) { + return; + } + + let thread_config = { + let state = sess.state.lock().await; + state.session_configuration.thread_config_snapshot() + }; + let is_first_turn = { + let mut state = sess.state.lock().await; + state.take_next_turn_is_first() + }; + sess.services + .analytics_events_client + .track_turn_resolved_config(TurnResolvedConfigFact { + turn_id: turn_context.sub_id.clone(), + thread_id: sess.conversation_id.to_string(), + num_input_images: input + .iter() + .filter(|item| { + matches!(item, UserInput::Image { .. } | UserInput::LocalImage { .. }) + }) + .count(), + submission_type: None, + ephemeral: thread_config.ephemeral, + session_source: thread_config.session_source, + model: turn_context.model_info.slug.clone(), + model_provider: turn_context.config.model_provider_id.clone(), + sandbox_policy: turn_context.sandbox_policy.get().clone(), + reasoning_effort: turn_context.reasoning_effort, + reasoning_summary: Some(turn_context.reasoning_summary), + service_tier: turn_context.config.service_tier, + approval_policy: turn_context.approval_policy.value(), + approvals_reviewer: turn_context.config.approvals_reviewer, + sandbox_network_access: turn_context.network_sandbox_policy.is_enabled(), + collaboration_mode: turn_context.collaboration_mode.mode, + personality: turn_context.personality, + is_first_turn, + }); +} + +async fn run_pre_sampling_compact( + sess: &Arc, + turn_context: &Arc, +) -> CodexResult { + let total_usage_tokens_before_compaction = sess.get_total_token_usage().await; + let mut pre_sampling_compacted = maybe_run_previous_model_inline_compact( + sess, + turn_context, + total_usage_tokens_before_compaction, + ) + .await?; + let total_usage_tokens = sess.get_total_token_usage().await; + let auto_compact_limit = turn_context + .model_info + .auto_compact_token_limit() + .unwrap_or(i64::MAX); + // Compact if the total usage tokens are greater than the auto compact limit + if total_usage_tokens >= auto_compact_limit { + run_auto_compact( + sess, + turn_context, + InitialContextInjection::DoNotInject, + CompactionReason::ContextLimit, + CompactionPhase::PreTurn, + ) + .await?; + pre_sampling_compacted = true; + } + Ok(pre_sampling_compacted) +} + +/// Runs pre-sampling compaction against the previous model when switching to a smaller +/// context-window model. +/// +/// Returns `Ok(true)` when compaction ran successfully, `Ok(false)` when compaction was skipped +/// because the model/context-window preconditions were not met, and `Err(_)` only when compaction +/// was attempted and failed. +async fn maybe_run_previous_model_inline_compact( + sess: &Arc, + turn_context: &Arc, + total_usage_tokens: i64, +) -> CodexResult { + let Some(previous_turn_settings) = sess.previous_turn_settings().await else { + return Ok(false); + }; + let previous_model_turn_context = Arc::new( + turn_context + .with_model(previous_turn_settings.model, &sess.services.models_manager) + .await, + ); + + let Some(old_context_window) = previous_model_turn_context.model_context_window() else { + return Ok(false); + }; + let Some(new_context_window) = turn_context.model_context_window() else { + return Ok(false); + }; + let new_auto_compact_limit = turn_context + .model_info + .auto_compact_token_limit() + .unwrap_or(i64::MAX); + let should_run = total_usage_tokens > new_auto_compact_limit + && previous_model_turn_context.model_info.slug != turn_context.model_info.slug + && old_context_window > new_context_window; + if should_run { + run_auto_compact( + sess, + &previous_model_turn_context, + InitialContextInjection::DoNotInject, + CompactionReason::ModelDownshift, + CompactionPhase::PreTurn, + ) + .await?; + return Ok(true); + } + Ok(false) +} + +async fn run_auto_compact( + sess: &Arc, + turn_context: &Arc, + initial_context_injection: InitialContextInjection, + reason: CompactionReason, + phase: CompactionPhase, +) -> CodexResult<()> { + if should_use_remote_compact_task(&turn_context.provider) { + run_inline_remote_auto_compact_task( + Arc::clone(sess), + Arc::clone(turn_context), + initial_context_injection, + reason, + phase, + ) + .await?; + } else { + run_inline_auto_compact_task( + Arc::clone(sess), + Arc::clone(turn_context), + initial_context_injection, + reason, + phase, + ) + .await?; + } + Ok(()) +} + +pub(super) fn collect_explicit_app_ids_from_skill_items( + skill_items: &[ResponseItem], + connectors: &[connectors::AppInfo], + skill_name_counts_lower: &HashMap, +) -> HashSet { + if skill_items.is_empty() || connectors.is_empty() { + return HashSet::new(); + } + + let skill_messages = skill_items + .iter() + .filter_map(|item| match item { + ResponseItem::Message { content, .. } => { + content.iter().find_map(|content_item| match content_item { + ContentItem::InputText { text } => Some(text.clone()), + _ => None, + }) + } + _ => None, + }) + .collect::>(); + if skill_messages.is_empty() { + return HashSet::new(); + } + + let mentions = collect_tool_mentions_from_messages(&skill_messages); + let mention_names_lower = mentions + .plain_names + .iter() + .map(|name| name.to_ascii_lowercase()) + .collect::>(); + let mut connector_ids = mentions + .paths + .iter() + .filter(|path| tool_kind_for_path(path) == ToolMentionKind::App) + .filter_map(|path| app_id_from_path(path).map(str::to_string)) + .collect::>(); + + let connector_slug_counts = build_connector_slug_counts(connectors); + for connector in connectors { + let slug = codex_connectors::metadata::connector_mention_slug(connector); + let connector_count = connector_slug_counts.get(&slug).copied().unwrap_or(0); + let skill_count = skill_name_counts_lower.get(&slug).copied().unwrap_or(0); + if connector_count == 1 && skill_count == 0 && mention_names_lower.contains(&slug) { + connector_ids.insert(connector.id.clone()); + } + } + + connector_ids +} + +pub(super) fn filter_connectors_for_input( + connectors: &[connectors::AppInfo], + input: &[ResponseItem], + explicitly_enabled_connectors: &HashSet, + skill_name_counts_lower: &HashMap, +) -> Vec { + let connectors: Vec = connectors + .iter() + .filter(|connector| connector.is_enabled) + .cloned() + .collect::>(); + if connectors.is_empty() { + return Vec::new(); + } + + let user_messages = collect_user_messages(input); + if user_messages.is_empty() && explicitly_enabled_connectors.is_empty() { + return Vec::new(); + } + + let mentions = collect_tool_mentions_from_messages(&user_messages); + let mention_names_lower = mentions + .plain_names + .iter() + .map(|name| name.to_ascii_lowercase()) + .collect::>(); + + let connector_slug_counts = build_connector_slug_counts(&connectors); + let mut allowed_connector_ids = explicitly_enabled_connectors.clone(); + for path in mentions + .paths + .iter() + .filter(|path| tool_kind_for_path(path) == ToolMentionKind::App) + { + if let Some(connector_id) = app_id_from_path(path) { + allowed_connector_ids.insert(connector_id.to_string()); + } + } + + connectors + .into_iter() + .filter(|connector| { + connector_inserted_in_messages( + connector, + &mention_names_lower, + &allowed_connector_ids, + &connector_slug_counts, + skill_name_counts_lower, + ) + }) + .collect() +} + +fn connector_inserted_in_messages( + connector: &connectors::AppInfo, + mention_names_lower: &HashSet, + allowed_connector_ids: &HashSet, + connector_slug_counts: &HashMap, + skill_name_counts_lower: &HashMap, +) -> bool { + if allowed_connector_ids.contains(&connector.id) { + return true; + } + + let mention_slug = codex_connectors::metadata::connector_mention_slug(connector); + let connector_count = connector_slug_counts + .get(&mention_slug) + .copied() + .unwrap_or(0); + let skill_count = skill_name_counts_lower + .get(&mention_slug) + .copied() + .unwrap_or(0); + connector_count == 1 && skill_count == 0 && mention_names_lower.contains(&mention_slug) +} + +pub(crate) fn build_prompt( + input: Vec, + router: &ToolRouter, + turn_context: &TurnContext, + base_instructions: BaseInstructions, +) -> Prompt { + let deferred_dynamic_tools = turn_context + .dynamic_tools + .iter() + .filter(|tool| tool.defer_loading) + .map(|tool| tool.name.as_str()) + .collect::>(); + let tools = if deferred_dynamic_tools.is_empty() { + router.model_visible_specs() + } else { + router + .model_visible_specs() + .into_iter() + .filter(|spec| !deferred_dynamic_tools.contains(spec.name())) + .collect() + }; + + Prompt { + input, + tools, + parallel_tool_calls: turn_context.model_info.supports_parallel_tool_calls, + base_instructions, + personality: turn_context.personality, + output_schema: turn_context.final_output_json_schema.clone(), + } +} + +#[allow(clippy::too_many_arguments)] +#[instrument(level = "trace", + skip_all, + fields( + turn_id = %turn_context.sub_id, + model = %turn_context.model_info.slug, + cwd = %turn_context.cwd.display() + ) +)] +async fn run_sampling_request( + sess: Arc, + turn_context: Arc, + turn_diff_tracker: SharedTurnDiffTracker, + client_session: &mut ModelClientSession, + turn_metadata_header: Option<&str>, + input: Vec, + explicitly_enabled_connectors: &HashSet, + skills_outcome: Option<&SkillLoadOutcome>, + server_model_warning_emitted_for_turn: &mut bool, + cancellation_token: CancellationToken, +) -> CodexResult { + let router = built_tools( + sess.as_ref(), + turn_context.as_ref(), + &input, + explicitly_enabled_connectors, + skills_outcome, + &cancellation_token, + ) + .await?; + + let base_instructions = sess.get_base_instructions().await; + + let tool_runtime = ToolCallRuntime::new( + Arc::clone(&router), + Arc::clone(&sess), + Arc::clone(&turn_context), + Arc::clone(&turn_diff_tracker), + ); + let _code_mode_worker = sess + .services + .code_mode_service + .start_turn_worker( + &sess, + &turn_context, + Arc::clone(&router), + Arc::clone(&turn_diff_tracker), + ) + .await; + let mut retries = 0; + let mut initial_input = Some(input); + loop { + let prompt_input = if let Some(input) = initial_input.take() { + input + } else { + sess.clone_history() + .await + .for_prompt(&turn_context.model_info.input_modalities) + }; + let prompt = build_prompt( + prompt_input, + router.as_ref(), + turn_context.as_ref(), + base_instructions.clone(), + ); + let err = match try_run_sampling_request( + tool_runtime.clone(), + Arc::clone(&sess), + Arc::clone(&turn_context), + client_session, + turn_metadata_header, + Arc::clone(&turn_diff_tracker), + server_model_warning_emitted_for_turn, + &prompt, + cancellation_token.child_token(), + ) + .await + { + Ok(output) => { + return Ok(output); + } + Err(CodexErr::ContextWindowExceeded) => { + sess.set_total_tokens_full(&turn_context).await; + return Err(CodexErr::ContextWindowExceeded); + } + Err(CodexErr::UsageLimitReached(e)) => { + let rate_limits = e.rate_limits.clone(); + if let Some(rate_limits) = rate_limits { + sess.update_rate_limits(&turn_context, *rate_limits).await; + } + return Err(CodexErr::UsageLimitReached(e)); + } + Err(err) => err, + }; + + if !err.is_retryable() { + return Err(err); + } + + // Use the configured provider-specific stream retry budget. + let max_retries = turn_context.provider.stream_max_retries(); + if retries >= max_retries + && client_session.try_switch_fallback_transport( + &turn_context.session_telemetry, + &turn_context.model_info, + ) + { + sess.send_event( + &turn_context, + EventMsg::Warning(WarningEvent { + message: format!("Falling back from WebSockets to HTTPS transport. {err:#}"), + }), + ) + .await; + retries = 0; + continue; + } + if retries < max_retries { + retries += 1; + let delay = match &err { + CodexErr::Stream(_, requested_delay) => { + requested_delay.unwrap_or_else(|| backoff(retries)) + } + _ => backoff(retries), + }; + warn!( + "stream disconnected - retrying sampling request ({retries}/{max_retries} in {delay:?})...", + ); + + // In release builds, hide the first websocket retry notification to reduce noisy + // transient reconnect messages. In debug builds, keep full visibility for diagnosis. + let report_error = retries > 1 + || cfg!(debug_assertions) + || !sess.services.model_client.responses_websocket_enabled(); + if report_error { + // Surface retry information to any UI/front‑end so the + // user understands what is happening instead of staring + // at a seemingly frozen screen. + sess.notify_stream_error( + &turn_context, + format!("Reconnecting... {retries}/{max_retries}"), + err, + ) + .await; + } + tokio::time::sleep(delay).await; + } else { + return Err(err); + } + } +} + +pub(crate) async fn built_tools( + sess: &Session, + turn_context: &TurnContext, + input: &[ResponseItem], + explicitly_enabled_connectors: &HashSet, + skills_outcome: Option<&SkillLoadOutcome>, + cancellation_token: &CancellationToken, +) -> CodexResult> { + let mcp_connection_manager = sess.services.mcp_connection_manager.read().await; + let has_mcp_servers = mcp_connection_manager.has_servers(); + let all_mcp_tools = mcp_connection_manager + .list_all_tools() + .or_cancel(cancellation_token) + .await?; + drop(mcp_connection_manager); + let loaded_plugins = sess + .services + .plugins_manager + .plugins_for_config(&turn_context.config) + .await; + + let mut effective_explicitly_enabled_connectors = explicitly_enabled_connectors.clone(); + effective_explicitly_enabled_connectors.extend(sess.get_connector_selection().await); + + let apps_enabled = turn_context.apps_enabled(); + let accessible_connectors = + apps_enabled.then(|| connectors::accessible_connectors_from_mcp_tools(&all_mcp_tools)); + let accessible_connectors_with_enabled_state = + accessible_connectors.as_ref().map(|connectors| { + connectors::with_app_enabled_state(connectors.clone(), &turn_context.config) + }); + let connectors = if apps_enabled { + let connectors = codex_connectors::merge::merge_plugin_connectors_with_accessible( + loaded_plugins + .effective_apps() + .into_iter() + .map(|connector_id| connector_id.0), + accessible_connectors.clone().unwrap_or_default(), + ); + Some(connectors::with_app_enabled_state( + connectors, + &turn_context.config, + )) + } else { + None + }; + let auth = sess.services.auth_manager.auth().await; + let discoverable_tools = if apps_enabled && turn_context.tools_config.tool_suggest { + if let Some(accessible_connectors) = accessible_connectors_with_enabled_state.as_ref() { + match connectors::list_tool_suggest_discoverable_tools_with_auth( + &turn_context.config, + auth.as_ref(), + accessible_connectors.as_slice(), + ) + .await + .map(|discoverable_tools| { + filter_tool_suggest_discoverable_tools_for_client( + discoverable_tools, + turn_context.app_server_client_name.as_deref(), + ) + }) { + Ok(discoverable_tools) if discoverable_tools.is_empty() => None, + Ok(discoverable_tools) => Some(discoverable_tools), + Err(err) => { + warn!("failed to load discoverable tool suggestions: {err:#}"); + None + } + } + } else { + None + } + } else { + None + }; + + let explicitly_enabled = if let Some(connectors) = connectors.as_ref() { + let skill_name_counts_lower = skills_outcome.map_or_else(HashMap::new, |outcome| { + build_skill_name_counts(&outcome.skills, &outcome.disabled_paths).1 + }); + + filter_connectors_for_input( + connectors, + input, + &effective_explicitly_enabled_connectors, + &skill_name_counts_lower, + ) + } else { + Vec::new() + }; + let mcp_tool_exposure = build_mcp_tool_exposure( + &all_mcp_tools, + connectors.as_deref(), + explicitly_enabled.as_slice(), + &turn_context.config, + &turn_context.tools_config, + ); + let mcp_tools = has_mcp_servers.then_some(mcp_tool_exposure.direct_tools); + let deferred_mcp_tools = mcp_tool_exposure.deferred_tools; + let unavailable_called_tools = if turn_context + .config + .features + .enabled(Feature::UnavailableDummyTools) + { + let exposed_tool_names = mcp_tools + .iter() + .chain(deferred_mcp_tools.iter()) + .flat_map(|tools| tools.keys().map(String::as_str)) + .collect::>(); + collect_unavailable_called_tools(input, &exposed_tool_names) + } else { + Vec::new() + }; + + let parallel_mcp_server_names = turn_context + .config + .mcp_servers + .get() + .iter() + .filter_map(|(server_name, server_config)| { + server_config + .supports_parallel_tool_calls + .then_some(server_name.clone()) + }) + .collect::>(); + + Ok(Arc::new(ToolRouter::from_config( + &turn_context.tools_config, + ToolRouterParams { + mcp_tools, + deferred_mcp_tools, + unavailable_called_tools, + parallel_mcp_server_names, + discoverable_tools, + dynamic_tools: turn_context.dynamic_tools.as_slice(), + }, + ))) +} + +#[derive(Debug)] +struct SamplingRequestResult { + needs_follow_up: bool, + last_agent_message: Option, +} + +/// Ephemeral per-response state for streaming a single proposed plan. +/// This is intentionally not persisted or stored in session/state since it +/// only exists while a response is actively streaming. The final plan text +/// is extracted from the completed assistant message. +/// Tracks a single proposed plan item across a streaming response. +struct ProposedPlanItemState { + item_id: String, + started: bool, + completed: bool, +} + +/// Aggregated state used only while streaming a plan-mode response. +/// Includes per-item parsers, deferred agent message bookkeeping, and the plan item lifecycle. +struct PlanModeStreamState { + /// Agent message items started by the model but deferred until we see non-plan text. + pending_agent_message_items: HashMap, + /// Agent message items whose start notification has been emitted. + started_agent_message_items: HashSet, + /// Leading whitespace buffered until we see non-whitespace text for an item. + leading_whitespace_by_item: HashMap, + /// Tracks plan item lifecycle while streaming plan output. + plan_item_state: ProposedPlanItemState, +} + +impl PlanModeStreamState { + fn new(turn_id: &str) -> Self { + Self { + pending_agent_message_items: HashMap::new(), + started_agent_message_items: HashSet::new(), + leading_whitespace_by_item: HashMap::new(), + plan_item_state: ProposedPlanItemState::new(turn_id), + } + } +} + +#[derive(Debug, Default)] +pub(super) struct AssistantMessageStreamParsers { + plan_mode: bool, + parsers_by_item: HashMap, +} + +type ParsedAssistantTextDelta = AssistantTextChunk; + +impl AssistantMessageStreamParsers { + pub(super) fn new(plan_mode: bool) -> Self { + Self { + plan_mode, + parsers_by_item: HashMap::new(), + } + } + + fn parser_mut(&mut self, item_id: &str) -> &mut AssistantTextStreamParser { + let plan_mode = self.plan_mode; + self.parsers_by_item + .entry(item_id.to_string()) + .or_insert_with(|| AssistantTextStreamParser::new(plan_mode)) + } + + pub(super) fn seed_item_text(&mut self, item_id: &str, text: &str) -> ParsedAssistantTextDelta { + if text.is_empty() { + return ParsedAssistantTextDelta::default(); + } + self.parser_mut(item_id).push_str(text) + } + + pub(super) fn parse_delta(&mut self, item_id: &str, delta: &str) -> ParsedAssistantTextDelta { + self.parser_mut(item_id).push_str(delta) + } + + pub(super) fn finish_item(&mut self, item_id: &str) -> ParsedAssistantTextDelta { + let Some(mut parser) = self.parsers_by_item.remove(item_id) else { + return ParsedAssistantTextDelta::default(); + }; + parser.finish() + } + + fn drain_finished(&mut self) -> Vec<(String, ParsedAssistantTextDelta)> { + let parsers_by_item = std::mem::take(&mut self.parsers_by_item); + parsers_by_item + .into_iter() + .map(|(item_id, mut parser)| (item_id, parser.finish())) + .collect() + } +} + +impl ProposedPlanItemState { + fn new(turn_id: &str) -> Self { + Self { + item_id: format!("{turn_id}-plan"), + started: false, + completed: false, + } + } + + async fn start(&mut self, sess: &Session, turn_context: &TurnContext) { + if self.started || self.completed { + return; + } + self.started = true; + let item = TurnItem::Plan(PlanItem { + id: self.item_id.clone(), + text: String::new(), + }); + sess.emit_turn_item_started(turn_context, &item).await; + } + + async fn push_delta(&mut self, sess: &Session, turn_context: &TurnContext, delta: &str) { + if self.completed { + return; + } + if delta.is_empty() { + return; + } + let event = PlanDeltaEvent { + thread_id: sess.conversation_id.to_string(), + turn_id: turn_context.sub_id.clone(), + item_id: self.item_id.clone(), + delta: delta.to_string(), + }; + sess.send_event(turn_context, EventMsg::PlanDelta(event)) + .await; + } + + async fn complete_with_text( + &mut self, + sess: &Session, + turn_context: &TurnContext, + text: String, + ) { + if self.completed || !self.started { + return; + } + self.completed = true; + let item = TurnItem::Plan(PlanItem { + id: self.item_id.clone(), + text, + }); + sess.emit_turn_item_completed(turn_context, item).await; + } +} + +/// In plan mode we defer agent message starts until the parser emits non-plan +/// text. The parser buffers each line until it can rule out a tag prefix, so +/// plan-only outputs never show up as empty assistant messages. +async fn maybe_emit_pending_agent_message_start( + sess: &Session, + turn_context: &TurnContext, + state: &mut PlanModeStreamState, + item_id: &str, +) { + if state.started_agent_message_items.contains(item_id) { + return; + } + if let Some(item) = state.pending_agent_message_items.remove(item_id) { + sess.emit_turn_item_started(turn_context, &item).await; + state + .started_agent_message_items + .insert(item_id.to_string()); + } +} + +/// Agent messages are text-only today; concatenate all text entries. +fn agent_message_text(item: &codex_protocol::items::AgentMessageItem) -> String { + item.content + .iter() + .map(|entry| match entry { + codex_protocol::items::AgentMessageContent::Text { text } => text.as_str(), + }) + .collect() +} + +pub(super) fn realtime_text_for_event(msg: &EventMsg) -> Option { + match msg { + EventMsg::AgentMessage(event) => Some(event.message.clone()), + EventMsg::ItemCompleted(event) => match &event.item { + TurnItem::AgentMessage(item) => Some(agent_message_text(item)), + _ => None, + }, + EventMsg::Error(_) + | EventMsg::Warning(_) + | EventMsg::RealtimeConversationStarted(_) + | EventMsg::RealtimeConversationSdp(_) + | EventMsg::RealtimeConversationRealtime(_) + | EventMsg::RealtimeConversationClosed(_) + | EventMsg::ModelReroute(_) + | EventMsg::ContextCompacted(_) + | EventMsg::ThreadRolledBack(_) + | EventMsg::TurnStarted(_) + | EventMsg::TurnComplete(_) + | EventMsg::TokenCount(_) + | EventMsg::UserMessage(_) + | EventMsg::AgentMessageDelta(_) + | EventMsg::AgentReasoning(_) + | EventMsg::AgentReasoningDelta(_) + | EventMsg::AgentReasoningRawContent(_) + | EventMsg::AgentReasoningRawContentDelta(_) + | EventMsg::AgentReasoningSectionBreak(_) + | EventMsg::SessionConfigured(_) + | EventMsg::ThreadNameUpdated(_) + | EventMsg::McpStartupUpdate(_) + | EventMsg::McpStartupComplete(_) + | EventMsg::McpToolCallBegin(_) + | EventMsg::McpToolCallEnd(_) + | EventMsg::WebSearchBegin(_) + | EventMsg::WebSearchEnd(_) + | EventMsg::ExecCommandBegin(_) + | EventMsg::ExecCommandOutputDelta(_) + | EventMsg::TerminalInteraction(_) + | EventMsg::ExecCommandEnd(_) + | EventMsg::PatchApplyBegin(_) + | EventMsg::PatchApplyEnd(_) + | EventMsg::ViewImageToolCall(_) + | EventMsg::ImageGenerationBegin(_) + | EventMsg::ImageGenerationEnd(_) + | EventMsg::ExecApprovalRequest(_) + | EventMsg::RequestPermissions(_) + | EventMsg::RequestUserInput(_) + | EventMsg::DynamicToolCallRequest(_) + | EventMsg::DynamicToolCallResponse(_) + | EventMsg::GuardianAssessment(_) + | EventMsg::ElicitationRequest(_) + | EventMsg::ApplyPatchApprovalRequest(_) + | EventMsg::DeprecationNotice(_) + | EventMsg::BackgroundEvent(_) + | EventMsg::UndoStarted(_) + | EventMsg::UndoCompleted(_) + | EventMsg::StreamError(_) + | EventMsg::TurnDiff(_) + | EventMsg::GetHistoryEntryResponse(_) + | EventMsg::McpListToolsResponse(_) + | EventMsg::ListSkillsResponse(_) + | EventMsg::RealtimeConversationListVoicesResponse(_) + | EventMsg::SkillsUpdateAvailable + | EventMsg::PlanUpdate(_) + | EventMsg::TurnAborted(_) + | EventMsg::ShutdownComplete + | EventMsg::EnteredReviewMode(_) + | EventMsg::ExitedReviewMode(_) + | EventMsg::RawResponseItem(_) + | EventMsg::ItemStarted(_) + | EventMsg::HookStarted(_) + | EventMsg::HookCompleted(_) + | EventMsg::AgentMessageContentDelta(_) + | EventMsg::PlanDelta(_) + | EventMsg::ReasoningContentDelta(_) + | EventMsg::ReasoningRawContentDelta(_) + | EventMsg::CollabAgentSpawnBegin(_) + | EventMsg::CollabAgentSpawnEnd(_) + | EventMsg::CollabAgentInteractionBegin(_) + | EventMsg::CollabAgentInteractionEnd(_) + | EventMsg::CollabWaitingBegin(_) + | EventMsg::CollabWaitingEnd(_) + | EventMsg::CollabCloseBegin(_) + | EventMsg::CollabCloseEnd(_) + | EventMsg::CollabResumeBegin(_) + | EventMsg::CollabResumeEnd(_) => None, + } +} + +/// Split the stream into normal assistant text vs. proposed plan content. +/// Normal text becomes AgentMessage deltas; plan content becomes PlanDelta + +/// TurnItem::Plan. +async fn handle_plan_segments( + sess: &Session, + turn_context: &TurnContext, + state: &mut PlanModeStreamState, + item_id: &str, + segments: Vec, +) { + for segment in segments { + match segment { + ProposedPlanSegment::Normal(delta) => { + if delta.is_empty() { + continue; + } + let has_non_whitespace = delta.chars().any(|ch| !ch.is_whitespace()); + if !has_non_whitespace && !state.started_agent_message_items.contains(item_id) { + let entry = state + .leading_whitespace_by_item + .entry(item_id.to_string()) + .or_default(); + entry.push_str(&delta); + continue; + } + let delta = if !state.started_agent_message_items.contains(item_id) { + if let Some(prefix) = state.leading_whitespace_by_item.remove(item_id) { + format!("{prefix}{delta}") + } else { + delta + } + } else { + delta + }; + maybe_emit_pending_agent_message_start(sess, turn_context, state, item_id).await; + + let event = AgentMessageContentDeltaEvent { + thread_id: sess.conversation_id.to_string(), + turn_id: turn_context.sub_id.clone(), + item_id: item_id.to_string(), + delta, + }; + sess.send_event(turn_context, EventMsg::AgentMessageContentDelta(event)) + .await; + } + ProposedPlanSegment::ProposedPlanStart => { + if !state.plan_item_state.completed { + state.plan_item_state.start(sess, turn_context).await; + } + } + ProposedPlanSegment::ProposedPlanDelta(delta) => { + if !state.plan_item_state.completed { + if !state.plan_item_state.started { + state.plan_item_state.start(sess, turn_context).await; + } + state + .plan_item_state + .push_delta(sess, turn_context, &delta) + .await; + } + } + ProposedPlanSegment::ProposedPlanEnd => {} + } + } +} + +async fn emit_streamed_assistant_text_delta( + sess: &Session, + turn_context: &TurnContext, + plan_mode_state: Option<&mut PlanModeStreamState>, + item_id: &str, + parsed: ParsedAssistantTextDelta, +) { + if parsed.is_empty() { + return; + } + if !parsed.citations.is_empty() { + // Citation extraction is intentionally local for now; we strip citations from display text + // but do not yet surface them in protocol events. + let _citations = parsed.citations; + } + if let Some(state) = plan_mode_state { + if !parsed.plan_segments.is_empty() { + handle_plan_segments(sess, turn_context, state, item_id, parsed.plan_segments).await; + } + return; + } + if parsed.visible_text.is_empty() { + return; + } + let event = AgentMessageContentDeltaEvent { + thread_id: sess.conversation_id.to_string(), + turn_id: turn_context.sub_id.clone(), + item_id: item_id.to_string(), + delta: parsed.visible_text, + }; + sess.send_event(turn_context, EventMsg::AgentMessageContentDelta(event)) + .await; +} + +/// Flush buffered assistant text parser state when an assistant message item ends. +async fn flush_assistant_text_segments_for_item( + sess: &Session, + turn_context: &TurnContext, + plan_mode_state: Option<&mut PlanModeStreamState>, + parsers: &mut AssistantMessageStreamParsers, + item_id: &str, +) { + let parsed = parsers.finish_item(item_id); + emit_streamed_assistant_text_delta(sess, turn_context, plan_mode_state, item_id, parsed).await; +} + +/// Flush any remaining buffered assistant text parser state at response completion. +async fn flush_assistant_text_segments_all( + sess: &Session, + turn_context: &TurnContext, + mut plan_mode_state: Option<&mut PlanModeStreamState>, + parsers: &mut AssistantMessageStreamParsers, +) { + for (item_id, parsed) in parsers.drain_finished() { + emit_streamed_assistant_text_delta( + sess, + turn_context, + plan_mode_state.as_deref_mut(), + &item_id, + parsed, + ) + .await; + } +} + +/// Emit completion for plan items by parsing the finalized assistant message. +async fn maybe_complete_plan_item_from_message( + sess: &Session, + turn_context: &TurnContext, + state: &mut PlanModeStreamState, + item: &ResponseItem, +) { + if let ResponseItem::Message { role, content, .. } = item + && role == "assistant" + { + let mut text = String::new(); + for entry in content { + if let ContentItem::OutputText { text: chunk } = entry { + text.push_str(chunk); + } + } + if let Some(plan_text) = extract_proposed_plan_text(&text) { + let (plan_text, _citations) = strip_citations(&plan_text); + if !state.plan_item_state.started { + state.plan_item_state.start(sess, turn_context).await; + } + state + .plan_item_state + .complete_with_text(sess, turn_context, plan_text) + .await; + } + } +} + +/// Emit a completed agent message in plan mode, respecting deferred starts. +async fn emit_agent_message_in_plan_mode( + sess: &Session, + turn_context: &TurnContext, + agent_message: codex_protocol::items::AgentMessageItem, + state: &mut PlanModeStreamState, +) { + let agent_message_id = agent_message.id.clone(); + let text = agent_message_text(&agent_message); + if text.trim().is_empty() { + state.pending_agent_message_items.remove(&agent_message_id); + state.started_agent_message_items.remove(&agent_message_id); + return; + } + + maybe_emit_pending_agent_message_start(sess, turn_context, state, &agent_message_id).await; + + if !state + .started_agent_message_items + .contains(&agent_message_id) + { + let start_item = state + .pending_agent_message_items + .remove(&agent_message_id) + .unwrap_or_else(|| { + TurnItem::AgentMessage(codex_protocol::items::AgentMessageItem { + id: agent_message_id.clone(), + content: Vec::new(), + phase: None, + memory_citation: None, + }) + }); + sess.emit_turn_item_started(turn_context, &start_item).await; + state + .started_agent_message_items + .insert(agent_message_id.clone()); + } + + sess.emit_turn_item_completed(turn_context, TurnItem::AgentMessage(agent_message)) + .await; + state.started_agent_message_items.remove(&agent_message_id); +} + +/// Emit completion for a plan-mode turn item, handling agent messages specially. +async fn emit_turn_item_in_plan_mode( + sess: &Session, + turn_context: &TurnContext, + turn_item: TurnItem, + previously_active_item: Option<&TurnItem>, + state: &mut PlanModeStreamState, +) { + match turn_item { + TurnItem::AgentMessage(agent_message) => { + emit_agent_message_in_plan_mode(sess, turn_context, agent_message, state).await; + } + _ => { + if previously_active_item.is_none() { + sess.emit_turn_item_started(turn_context, &turn_item).await; + } + sess.emit_turn_item_completed(turn_context, turn_item).await; + } + } +} + +/// Handle a completed assistant response item in plan mode, returning true if handled. +async fn handle_assistant_item_done_in_plan_mode( + sess: &Session, + turn_context: &TurnContext, + item: &ResponseItem, + state: &mut PlanModeStreamState, + previously_active_item: Option<&TurnItem>, + last_agent_message: &mut Option, +) -> bool { + if let ResponseItem::Message { role, .. } = item + && role == "assistant" + { + maybe_complete_plan_item_from_message(sess, turn_context, state, item).await; + + if let Some(turn_item) = + handle_non_tool_response_item(sess, turn_context, item, /*plan_mode*/ true).await + { + emit_turn_item_in_plan_mode( + sess, + turn_context, + turn_item, + previously_active_item, + state, + ) + .await; + } + + record_completed_response_item(sess, turn_context, item).await; + if let Some(agent_message) = last_assistant_message_from_item(item, /*plan_mode*/ true) { + *last_agent_message = Some(agent_message); + } + return true; + } + false +} + +async fn drain_in_flight( + in_flight: &mut FuturesOrdered>>, + sess: Arc, + turn_context: Arc, +) -> CodexResult<()> { + while let Some(res) = in_flight.next().await { + match res { + Ok(response_input) => { + let response_item = response_input.into(); + sess.record_conversation_items(&turn_context, std::slice::from_ref(&response_item)) + .await; + mark_thread_memory_mode_polluted_if_external_context( + sess.as_ref(), + turn_context.as_ref(), + &response_item, + ) + .await; + } + Err(err) => { + error_or_panic(format!("in-flight tool future failed during drain: {err}")); + } + } + } + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +#[instrument(level = "trace", + skip_all, + fields( + turn_id = %turn_context.sub_id, + model = %turn_context.model_info.slug + ) +)] +async fn try_run_sampling_request( + tool_runtime: ToolCallRuntime, + sess: Arc, + turn_context: Arc, + client_session: &mut ModelClientSession, + turn_metadata_header: Option<&str>, + turn_diff_tracker: SharedTurnDiffTracker, + server_model_warning_emitted_for_turn: &mut bool, + prompt: &Prompt, + cancellation_token: CancellationToken, +) -> CodexResult { + feedback_tags!( + model = turn_context.model_info.slug.clone(), + approval_policy = turn_context.approval_policy.value(), + sandbox_policy = turn_context.sandbox_policy.get(), + effort = turn_context.reasoning_effort, + auth_mode = sess.services.auth_manager.auth_mode(), + features = sess.features.enabled_features(), + ); + let mut stream = client_session + .stream( + prompt, + &turn_context.model_info, + &turn_context.session_telemetry, + turn_context.reasoning_effort, + turn_context.reasoning_summary, + turn_context.config.service_tier, + turn_metadata_header, + ) + .instrument(trace_span!("stream_request")) + .or_cancel(&cancellation_token) + .await??; + let mut in_flight: FuturesOrdered>> = + FuturesOrdered::new(); + let mut needs_follow_up = false; + let mut last_agent_message: Option = None; + let mut active_item: Option = None; + let mut should_emit_turn_diff = false; + let plan_mode = turn_context.collaboration_mode.mode == ModeKind::Plan; + let mut assistant_message_stream_parsers = AssistantMessageStreamParsers::new(plan_mode); + let mut plan_mode_state = plan_mode.then(|| PlanModeStreamState::new(&turn_context.sub_id)); + let receiving_span = trace_span!("receiving_stream"); + let outcome: CodexResult = loop { + let handle_responses = trace_span!( + parent: &receiving_span, + "handle_responses", + otel.name = field::Empty, + tool_name = field::Empty, + from = field::Empty, + ); + + let event = match stream + .next() + .instrument(trace_span!(parent: &handle_responses, "receiving")) + .or_cancel(&cancellation_token) + .await + { + Ok(event) => event, + Err(codex_async_utils::CancelErr::Cancelled) => break Err(CodexErr::TurnAborted), + }; + + let event = match event { + Some(Ok(event)) => event, + Some(Err(err)) => break Err(err), + None => { + break Err(CodexErr::Stream( + "stream closed before response.completed".into(), + None, + )); + } + }; + + sess.services + .session_telemetry + .record_responses(&handle_responses, &event); + record_turn_ttft_metric(&turn_context, &event).await; + + match event { + ResponseEvent::Created => {} + ResponseEvent::OutputItemDone(item) => { + let previously_active_item = active_item.take(); + if let Some(previous) = previously_active_item.as_ref() + && matches!(previous, TurnItem::AgentMessage(_)) + { + let item_id = previous.id(); + flush_assistant_text_segments_for_item( + &sess, + &turn_context, + plan_mode_state.as_mut(), + &mut assistant_message_stream_parsers, + &item_id, + ) + .await; + } + if let Some(state) = plan_mode_state.as_mut() + && handle_assistant_item_done_in_plan_mode( + &sess, + &turn_context, + &item, + state, + previously_active_item.as_ref(), + &mut last_agent_message, + ) + .await + { + continue; + } + + let mut ctx = HandleOutputCtx { + sess: sess.clone(), + turn_context: turn_context.clone(), + tool_runtime: tool_runtime.clone(), + cancellation_token: cancellation_token.child_token(), + }; + + let output_result = + match handle_output_item_done(&mut ctx, item, previously_active_item) + .instrument(handle_responses) + .await + { + Ok(output_result) => output_result, + Err(err) => break Err(err), + }; + if let Some(tool_future) = output_result.tool_future { + in_flight.push_back(tool_future); + } + if let Some(agent_message) = output_result.last_agent_message { + last_agent_message = Some(agent_message); + } + needs_follow_up |= output_result.needs_follow_up; + } + ResponseEvent::OutputItemAdded(item) => { + if let Some(turn_item) = handle_non_tool_response_item( + sess.as_ref(), + turn_context.as_ref(), + &item, + plan_mode, + ) + .await + { + let mut turn_item = turn_item; + let mut seeded_parsed: Option = None; + let mut seeded_item_id: Option = None; + if matches!(turn_item, TurnItem::AgentMessage(_)) + && let Some(raw_text) = raw_assistant_output_text_from_item(&item) + { + let item_id = turn_item.id(); + let mut seeded = + assistant_message_stream_parsers.seed_item_text(&item_id, &raw_text); + if let TurnItem::AgentMessage(agent_message) = &mut turn_item { + agent_message.content = + vec![codex_protocol::items::AgentMessageContent::Text { + text: if plan_mode { + String::new() + } else { + std::mem::take(&mut seeded.visible_text) + }, + }]; + } + seeded_parsed = plan_mode.then_some(seeded); + seeded_item_id = Some(item_id); + } + if let Some(state) = plan_mode_state.as_mut() + && matches!(turn_item, TurnItem::AgentMessage(_)) + { + let item_id = turn_item.id(); + state + .pending_agent_message_items + .insert(item_id, turn_item.clone()); + } else { + sess.emit_turn_item_started(&turn_context, &turn_item).await; + } + if let (Some(state), Some(item_id), Some(parsed)) = ( + plan_mode_state.as_mut(), + seeded_item_id.as_deref(), + seeded_parsed, + ) { + emit_streamed_assistant_text_delta( + &sess, + &turn_context, + Some(state), + item_id, + parsed, + ) + .await; + } + active_item = Some(turn_item); + } + } + ResponseEvent::ServerModel(server_model) => { + if !*server_model_warning_emitted_for_turn + && sess + .maybe_warn_on_server_model_mismatch(&turn_context, server_model) + .await + { + *server_model_warning_emitted_for_turn = true; + } + } + ResponseEvent::ServerReasoningIncluded(included) => { + sess.set_server_reasoning_included(included).await; + } + ResponseEvent::RateLimits(snapshot) => { + // Update internal state with latest rate limits, but defer sending until + // token usage is available to avoid duplicate TokenCount events. + sess.update_rate_limits(&turn_context, snapshot).await; + } + ResponseEvent::ModelsEtag(etag) => { + // Update internal state with latest models etag + sess.services.models_manager.refresh_if_new_etag(etag).await; + } + ResponseEvent::Completed { + response_id: _, + token_usage, + } => { + flush_assistant_text_segments_all( + &sess, + &turn_context, + plan_mode_state.as_mut(), + &mut assistant_message_stream_parsers, + ) + .await; + sess.update_token_usage_info(&turn_context, token_usage.as_ref()) + .await; + should_emit_turn_diff = true; + + break Ok(SamplingRequestResult { + needs_follow_up, + last_agent_message, + }); + } + ResponseEvent::OutputTextDelta(delta) => { + // In review child threads, suppress assistant text deltas; the + // UI will show a selection popup from the final ReviewOutput. + if let Some(active) = active_item.as_ref() { + let item_id = active.id(); + if matches!(active, TurnItem::AgentMessage(_)) { + let parsed = assistant_message_stream_parsers.parse_delta(&item_id, &delta); + emit_streamed_assistant_text_delta( + &sess, + &turn_context, + plan_mode_state.as_mut(), + &item_id, + parsed, + ) + .await; + } else { + let event = AgentMessageContentDeltaEvent { + thread_id: sess.conversation_id.to_string(), + turn_id: turn_context.sub_id.clone(), + item_id, + delta, + }; + sess.send_event(&turn_context, EventMsg::AgentMessageContentDelta(event)) + .await; + } + } else { + error_or_panic("OutputTextDelta without active item".to_string()); + } + } + ResponseEvent::ReasoningSummaryDelta { + delta, + summary_index, + } => { + if let Some(active) = active_item.as_ref() { + let event = ReasoningContentDeltaEvent { + thread_id: sess.conversation_id.to_string(), + turn_id: turn_context.sub_id.clone(), + item_id: active.id(), + delta, + summary_index, + }; + sess.send_event(&turn_context, EventMsg::ReasoningContentDelta(event)) + .await; + } else { + error_or_panic("ReasoningSummaryDelta without active item".to_string()); + } + } + ResponseEvent::ReasoningSummaryPartAdded { summary_index } => { + if let Some(active) = active_item.as_ref() { + let event = + EventMsg::AgentReasoningSectionBreak(AgentReasoningSectionBreakEvent { + item_id: active.id(), + summary_index, + }); + sess.send_event(&turn_context, event).await; + } else { + error_or_panic("ReasoningSummaryPartAdded without active item".to_string()); + } + } + ResponseEvent::ReasoningContentDelta { + delta, + content_index, + } => { + if let Some(active) = active_item.as_ref() { + let event = ReasoningRawContentDeltaEvent { + thread_id: sess.conversation_id.to_string(), + turn_id: turn_context.sub_id.clone(), + item_id: active.id(), + delta, + content_index, + }; + sess.send_event(&turn_context, EventMsg::ReasoningRawContentDelta(event)) + .await; + } else { + error_or_panic("ReasoningRawContentDelta without active item".to_string()); + } + } + } + }; + + flush_assistant_text_segments_all( + &sess, + &turn_context, + plan_mode_state.as_mut(), + &mut assistant_message_stream_parsers, + ) + .await; + + drain_in_flight(&mut in_flight, sess.clone(), turn_context.clone()).await?; + + if cancellation_token.is_cancelled() { + return Err(CodexErr::TurnAborted); + } + + if should_emit_turn_diff { + let unified_diff = { + let mut tracker = turn_diff_tracker.lock().await; + tracker.get_unified_diff() + }; + if let Ok(Some(unified_diff)) = unified_diff { + let msg = EventMsg::TurnDiff(TurnDiffEvent { unified_diff }); + sess.clone().send_event(&turn_context, msg).await; + } + } + + outcome +} + +pub(crate) fn get_last_assistant_message_from_turn(responses: &[ResponseItem]) -> Option { + for item in responses.iter().rev() { + if let Some(message) = last_assistant_message_from_item(item, /*plan_mode*/ false) { + return Some(message); + } + } + None +} From 65cc12d72e25723aece48edd6ff93dd288b6c042 Mon Sep 17 00:00:00 2001 From: Jeff Harris Date: Thu, 16 Apr 2026 15:43:51 -0700 Subject: [PATCH 46/59] Use codex-auto-review for guardian reviews (#18169) ## Summary This is the minimal client-side follow-up for the Codex Auto Review model slug rollout. It updates the guardian reviewer preferred model from `gpt-5.4` to `codex-auto-review`, so the client can rely on the backend catalog + Statsig mapping instead of hardcoding the GPT-5.4 slug. Context: https://openai.slack.com/archives/C0AF9328RL0/p1775777479388369?thread_ts=1775773094.071629&cid=C0AF9328RL0 ## Testing - `cargo fmt --package codex-core --check` - `cargo test -p codex-core guardian::` - `bazel test --experimental_remote_downloader= --test_output=errors //codex-rs/core:core-unit-tests --test_arg=guardian` --- codex-rs/core/src/guardian/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codex-rs/core/src/guardian/mod.rs b/codex-rs/core/src/guardian/mod.rs index 67e9a828ee..4fa150a232 100644 --- a/codex-rs/core/src/guardian/mod.rs +++ b/codex-rs/core/src/guardian/mod.rs @@ -34,7 +34,7 @@ pub(crate) use review::review_approval_request_with_cancel; pub(crate) use review::routes_approval_to_guardian; pub(crate) use review_session::GuardianReviewSessionManager; -const GUARDIAN_PREFERRED_MODEL: &str = "gpt-5.4"; +const GUARDIAN_PREFERRED_MODEL: &str = "codex-auto-review"; pub(crate) const GUARDIAN_REVIEW_TIMEOUT: Duration = Duration::from_secs(90); pub(crate) const GUARDIAN_REVIEWER_NAME: &str = "guardian"; const GUARDIAN_MAX_MESSAGE_TRANSCRIPT_TOKENS: usize = 10_000; From bf6e7e12aa0cc072117623155f55582891fda3e1 Mon Sep 17 00:00:00 2001 From: Matthew Zeng Date: Thu, 16 Apr 2026 16:46:15 -0700 Subject: [PATCH 47/59] Use in-process app-server for unknown-thread MCP read test (#18196) ## Summary - Switch the unknown-thread MCP resource read test from the stdio subprocess to the in-process app-server path. - Keep the assertion focused on the returned error message while avoiding child-process teardown timing issues in nextest. ## Testing - Not run (not requested) --- .../app-server/tests/suite/v2/mcp_resource.rs | 76 +++++++++++++++---- 1 file changed, 60 insertions(+), 16 deletions(-) diff --git a/codex-rs/app-server/tests/suite/v2/mcp_resource.rs b/codex-rs/app-server/tests/suite/v2/mcp_resource.rs index 0a38f3b37e..b0ff8888a4 100644 --- a/codex-rs/app-server/tests/suite/v2/mcp_resource.rs +++ b/codex-rs/app-server/tests/suite/v2/mcp_resource.rs @@ -7,7 +7,11 @@ use app_test_support::McpProcess; use app_test_support::to_response; use app_test_support::write_chatgpt_auth; use axum::Router; -use codex_app_server_protocol::JSONRPCError; +use codex_app_server::in_process; +use codex_app_server::in_process::InProcessStartArgs; +use codex_app_server_protocol::ClientInfo; +use codex_app_server_protocol::ClientRequest; +use codex_app_server_protocol::InitializeParams; use codex_app_server_protocol::JSONRPCResponse; use codex_app_server_protocol::McpResourceContent; use codex_app_server_protocol::McpResourceReadParams; @@ -15,7 +19,14 @@ use codex_app_server_protocol::McpResourceReadResponse; use codex_app_server_protocol::RequestId; use codex_app_server_protocol::ThreadStartParams; use codex_app_server_protocol::ThreadStartResponse; +use codex_arg0::Arg0DispatchPaths; use codex_config::types::AuthCredentialsStoreMode; +use codex_core::config::ConfigBuilder; +use codex_core::config_loader::CloudRequirementsLoader; +use codex_core::config_loader::LoaderOverrides; +use codex_exec_server::EnvironmentManager; +use codex_feedback::CodexFeedback; +use codex_protocol::protocol::SessionSource; use core_test_support::responses; use pretty_assertions::assert_eq; use rmcp::handler::server::ServerHandler; @@ -149,24 +160,57 @@ stream_max_retries = 0 #[tokio::test] async fn mcp_resource_read_returns_error_for_unknown_thread() -> Result<()> { let codex_home = TempDir::new()?; - let mut mcp = McpProcess::new(codex_home.path()).await?; - timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; - - let request_id = mcp - .send_mcp_resource_read_request(McpResourceReadParams { - thread_id: "00000000-0000-4000-8000-000000000000".to_string(), - server: "codex_apps".to_string(), - uri: TEST_RESOURCE_URI.to_string(), - }) + let loader_overrides = LoaderOverrides::without_managed_config_for_tests(); + let config = ConfigBuilder::default() + .codex_home(codex_home.path().to_path_buf()) + .fallback_cwd(Some(codex_home.path().to_path_buf())) + .loader_overrides(loader_overrides.clone()) + .build() .await?; - let error: JSONRPCError = timeout( - DEFAULT_READ_TIMEOUT, - mcp.read_stream_until_error_message(RequestId::Integer(request_id)), - ) - .await??; + // This negative-path test does not need the stdio subprocess; keeping it + // in-process avoids child-process teardown timing in nextest leak detection. + let client = in_process::start(InProcessStartArgs { + arg0_paths: Arg0DispatchPaths::default(), + config: Arc::new(config), + cli_overrides: Vec::new(), + loader_overrides, + cloud_requirements: CloudRequirementsLoader::default(), + feedback: CodexFeedback::new(), + log_db: None, + environment_manager: Arc::new(EnvironmentManager::new(/*exec_server_url*/ None)), + config_warnings: Vec::new(), + session_source: SessionSource::Cli, + enable_codex_api_key_env: false, + initialize: InitializeParams { + client_info: ClientInfo { + name: "codex-app-server-tests".to_string(), + title: None, + version: "0.1.0".to_string(), + }, + capabilities: None, + }, + channel_capacity: in_process::DEFAULT_IN_PROCESS_CHANNEL_CAPACITY, + }) + .await?; + let response = client + .request(ClientRequest::McpResourceRead { + request_id: RequestId::Integer(1), + params: McpResourceReadParams { + thread_id: "00000000-0000-4000-8000-000000000000".to_string(), + server: "codex_apps".to_string(), + uri: TEST_RESOURCE_URI.to_string(), + }, + }) + .await; + client.shutdown().await?; + + let error = match response? { + Ok(result) => anyhow::bail!("expected thread-not-found error, got response: {result:?}"), + Err(error) => error, + }; assert!( - error.error.message.contains("thread not found"), + error.message.contains("thread not found"), "expected thread-not-found error, got: {error:?}" ); From 5818ed6660e5e79234650538923cc34e01409367 Mon Sep 17 00:00:00 2001 From: xli-oai Date: Thu, 16 Apr 2026 17:06:34 -0700 Subject: [PATCH 48/59] Move marketplace add under plugin command (#18116) ## Summary - move the marketplace add CLI from `codex marketplace add` to `codex plugin marketplace add` - keep marketplace config overrides working through the nested plugin command - reject `--sparse` for local marketplace directory sources before the local-source install path bypasses git-source validation ## Validation - `just fmt` - `git diff --check` - `cargo test -p codex-cli` - `cargo test -p codex-core marketplace_add -- --nocapture` - `cargo test -p codex-core install_plugin_updates_config_with_relative_path_and_plugin_key -- --nocapture` - `xli-test-marketplace-cli` local isolated matrix: `T1`, `L1`-`L10` --- codex-rs/cli/src/main.rs | 68 +++++++++++++++++--- codex-rs/cli/tests/marketplace_add.rs | 33 +++++++++- codex-rs/cli/tests/marketplace_upgrade.rs | 36 +++++++++++ codex-rs/core/src/plugins/marketplace_add.rs | 37 +++++++++++ 4 files changed, 163 insertions(+), 11 deletions(-) create mode 100644 codex-rs/cli/tests/marketplace_upgrade.rs diff --git a/codex-rs/cli/src/main.rs b/codex-rs/cli/src/main.rs index 25bf80a6e3..32fb25ad3d 100644 --- a/codex-rs/cli/src/main.rs +++ b/codex-rs/cli/src/main.rs @@ -111,8 +111,8 @@ enum Subcommand { /// Manage external MCP servers for Codex. Mcp(McpCli), - /// Manage plugin marketplaces for Codex. - Marketplace(MarketplaceCli), + /// Manage Codex plugins. + Plugin(PluginCli), /// Start Codex as an MCP server (stdio). McpServer, @@ -170,6 +170,21 @@ enum Subcommand { Features(FeaturesCli), } +#[derive(Debug, Parser)] +struct PluginCli { + #[clap(flatten)] + pub config_overrides: CliConfigOverrides, + + #[command(subcommand)] + subcommand: PluginSubcommand, +} + +#[derive(Debug, clap::Subcommand)] +enum PluginSubcommand { + /// Manage plugin marketplaces for Codex. + Marketplace(MarketplaceCli), +} + #[derive(Debug, Parser)] struct CompletionCommand { /// Shell to generate completions for @@ -726,17 +741,23 @@ async fn cli_main(arg0_paths: Arg0DispatchPaths) -> anyhow::Result<()> { prepend_config_flags(&mut mcp_cli.config_overrides, root_config_overrides.clone()); mcp_cli.run().await?; } - Some(Subcommand::Marketplace(mut marketplace_cli)) => { + Some(Subcommand::Plugin(plugin_cli)) => { reject_remote_mode_for_subcommand( root_remote.as_deref(), root_remote_auth_token_env.as_deref(), - "marketplace", + "plugin", )?; - prepend_config_flags( - &mut marketplace_cli.config_overrides, - root_config_overrides.clone(), - ); - marketplace_cli.run().await?; + let PluginCli { + mut config_overrides, + subcommand, + } = plugin_cli; + prepend_config_flags(&mut config_overrides, root_config_overrides.clone()); + match subcommand { + PluginSubcommand::Marketplace(mut marketplace_cli) => { + prepend_config_flags(&mut marketplace_cli.config_overrides, config_overrides); + marketplace_cli.run().await?; + } + } } Some(Subcommand::AppServer(app_server_cli)) => { let AppServerCommand { @@ -1690,6 +1711,35 @@ mod tests { assert!(matches!(cli.subcommand, Some(Subcommand::Responses(_)))); } + #[test] + fn plugin_marketplace_add_parses_under_plugin() { + let cli = + MultitoolCli::try_parse_from(["codex", "plugin", "marketplace", "add", "owner/repo"]) + .expect("parse"); + + assert!(matches!(cli.subcommand, Some(Subcommand::Plugin(_)))); + } + + #[test] + fn plugin_marketplace_upgrade_parses_under_plugin() { + let cli = + MultitoolCli::try_parse_from(["codex", "plugin", "marketplace", "upgrade", "debug"]) + .expect("parse"); + + assert!(matches!(cli.subcommand, Some(Subcommand::Plugin(_)))); + } + + #[test] + fn marketplace_no_longer_parses_at_top_level() { + let add_result = + MultitoolCli::try_parse_from(["codex", "marketplace", "add", "owner/repo"]); + assert!(add_result.is_err()); + + let upgrade_result = + MultitoolCli::try_parse_from(["codex", "marketplace", "upgrade", "debug"]); + assert!(upgrade_result.is_err()); + } + fn sample_exit_info(conversation_id: Option<&str>, thread_name: Option<&str>) -> AppExitInfo { let token_usage = TokenUsage { output_tokens: 2, diff --git a/codex-rs/cli/tests/marketplace_add.rs b/codex-rs/cli/tests/marketplace_add.rs index e04cfb5791..e4256f52b2 100644 --- a/codex-rs/cli/tests/marketplace_add.rs +++ b/codex-rs/cli/tests/marketplace_add.rs @@ -48,7 +48,7 @@ async fn marketplace_add_local_directory_source() -> Result<()> { codex_command(codex_home.path())? .current_dir(source_parent) - .args(["marketplace", "add", source_arg.as_str()]) + .args(["plugin", "marketplace", "add", source_arg.as_str()]) .assert() .success(); @@ -78,7 +78,12 @@ async fn marketplace_add_rejects_local_manifest_file_source() -> Result<()> { let manifest_path = source.path().join(".agents/plugins/marketplace.json"); codex_command(codex_home.path())? - .args(["marketplace", "add", manifest_path.to_str().unwrap()]) + .args([ + "plugin", + "marketplace", + "add", + manifest_path.to_str().unwrap(), + ]) .assert() .failure() .stderr(contains( @@ -87,3 +92,27 @@ async fn marketplace_add_rejects_local_manifest_file_source() -> Result<()> { Ok(()) } + +#[tokio::test] +async fn marketplace_add_rejects_sparse_for_local_directory_source() -> Result<()> { + let codex_home = TempDir::new()?; + let source = TempDir::new()?; + write_marketplace_source(source.path(), "local ref")?; + + codex_command(codex_home.path())? + .args([ + "plugin", + "marketplace", + "add", + "--sparse", + ".agents", + source.path().to_str().unwrap(), + ]) + .assert() + .failure() + .stderr(contains( + "--sparse is only supported for git marketplace sources", + )); + + Ok(()) +} diff --git a/codex-rs/cli/tests/marketplace_upgrade.rs b/codex-rs/cli/tests/marketplace_upgrade.rs new file mode 100644 index 0000000000..081203ebef --- /dev/null +++ b/codex-rs/cli/tests/marketplace_upgrade.rs @@ -0,0 +1,36 @@ +use anyhow::Result; +use predicates::str::contains; +use std::path::Path; +use tempfile::TempDir; + +fn codex_command(codex_home: &Path) -> Result { + let mut cmd = assert_cmd::Command::new(codex_utils_cargo_bin::cargo_bin("codex")?); + cmd.env("CODEX_HOME", codex_home); + Ok(cmd) +} + +#[tokio::test] +async fn marketplace_upgrade_runs_under_plugin() -> Result<()> { + let codex_home = TempDir::new()?; + + codex_command(codex_home.path())? + .args(["plugin", "marketplace", "upgrade"]) + .assert() + .success() + .stdout(contains("No configured Git marketplaces to upgrade.")); + + Ok(()) +} + +#[tokio::test] +async fn marketplace_upgrade_no_longer_runs_at_top_level() -> Result<()> { + let codex_home = TempDir::new()?; + + codex_command(codex_home.path())? + .args(["marketplace", "upgrade"]) + .assert() + .failure() + .stderr(contains("unexpected argument 'upgrade' found")); + + Ok(()) +} diff --git a/codex-rs/core/src/plugins/marketplace_add.rs b/codex-rs/core/src/plugins/marketplace_add.rs index 38fc04c199..3ea0445837 100644 --- a/codex-rs/core/src/plugins/marketplace_add.rs +++ b/codex-rs/core/src/plugins/marketplace_add.rs @@ -77,6 +77,11 @@ where sparse_paths, } = request; let source = parse_marketplace_source(&source, ref_name)?; + if !sparse_paths.is_empty() && !matches!(source, MarketplaceSource::Git { .. }) { + return Err(MarketplaceAddError::InvalidRequest( + "--sparse is only supported for git marketplace sources".to_string(), + )); + } let install_root = marketplace_install_root(codex_home); fs::create_dir_all(&install_root).map_err(|err| { @@ -287,6 +292,38 @@ mod tests { Ok(()) } + #[test] + fn add_marketplace_sync_rejects_sparse_checkout_for_local_directory_source() -> Result<()> { + let codex_home = TempDir::new()?; + let source_root = TempDir::new()?; + write_marketplace_source(source_root.path(), "local copy")?; + + let err = add_marketplace_sync_with_cloner( + codex_home.path(), + MarketplaceAddRequest { + source: source_root.path().display().to_string(), + ref_name: None, + sparse_paths: vec![".agents".to_string()], + }, + |_url, _ref_name, _sparse_paths, _destination| { + panic!("git cloner should not be called for local marketplace sources") + }, + ) + .unwrap_err(); + + assert_eq!( + err.to_string(), + "--sparse is only supported for git marketplace sources" + ); + assert!( + !codex_home + .path() + .join(codex_config::CONFIG_TOML_FILE) + .exists() + ); + Ok(()) + } + #[test] fn add_marketplace_sync_treats_existing_local_directory_source_as_already_added() -> Result<()> { From 0d0abe839a3886f8140626fa0028d09de349bac6 Mon Sep 17 00:00:00 2001 From: viyatb-oai Date: Thu, 16 Apr 2026 17:35:16 -0700 Subject: [PATCH 49/59] feat(sandbox): add glob deny-read platform enforcement (#18096) ## Summary - adds macOS Seatbelt deny rules for unreadable glob patterns - expands unreadable glob matches on Linux and masks them in bwrap, including canonical symlink targets - keeps Linux glob expansion robust when `rg` is unavailable in minimal or Bazel test environments - adds sandbox integration coverage that runs `shell` and `exec_command` with a `**/*.env = none` policy and verifies the secret contents do not reach the model ## Linux glob expansion ```text Prefer: rg --files --hidden --no-ignore --glob -- Fallback: internal globset walker when rg is not installed Failure: any other rg failure aborts sandbox construction ``` ``` [permissions.workspace.filesystem] glob_scan_max_depth = 2 [permissions.workspace.filesystem.":project_roots"] "**/*.env" = "none" ``` This keeps the common path fast without making sandbox construction depend on an ambient `rg` binary. If `rg` is present but fails for another reason, the sandbox setup fails closed instead of silently omitting deny-read masks. ## Platform support - macOS: subprocess sandbox enforcement is handled by Seatbelt regex deny rules - Linux: subprocess sandbox enforcement is handled by expanding existing glob matches and masking them in bwrap - Windows: policy/config/direct-tool glob support is already on `main` from #15979; Windows subprocess sandbox paths continue to fail closed when unreadable split filesystem carveouts require runtime enforcement, rather than silently running unsandboxed ## Stack 1. #15979 - merged: cross-platform glob deny-read policy/config/direct-tool support for macOS, Linux, and Windows 2. This PR - macOS/Linux subprocess sandbox enforcement plus Windows fail-closed clarification 3. #17740 - managed deny-read requirements ## Verification - Added integration coverage for `shell` and `exec_command` glob deny-read enforcement - `cargo check -p codex-sandboxing -p codex-linux-sandbox --tests` - `cargo check -p codex-core --test all` - `cargo clippy -p codex-linux-sandbox -p codex-sandboxing --tests` - `just bazel-lock-check` --------- Co-authored-by: Codex --- codex-rs/Cargo.lock | 2 + codex-rs/config/src/permissions_toml.rs | 4 + codex-rs/core/config.schema.json | 8 + codex-rs/core/src/config/config_tests.rs | 20 + codex-rs/core/src/config/permissions.rs | 57 +- codex-rs/core/src/config/permissions_tests.rs | 44 ++ .../core/src/tools/handlers/unified_exec.rs | 34 +- .../core/src/unified_exec/process_manager.rs | 17 +- codex-rs/core/tests/suite/tools.rs | 111 ++- codex-rs/core/tests/suite/unified_exec.rs | 126 ++++ codex-rs/linux-sandbox/Cargo.toml | 1 + codex-rs/linux-sandbox/README.md | 41 +- codex-rs/linux-sandbox/src/bwrap.rs | 642 +++++++++++++++--- codex-rs/linux-sandbox/src/linux_run_main.rs | 2 + .../linux-sandbox/src/linux_run_main_tests.rs | 4 + codex-rs/protocol/src/permissions.rs | 8 + codex-rs/sandboxing/Cargo.toml | 1 + codex-rs/sandboxing/src/lib.rs | 3 - codex-rs/sandboxing/src/manager.rs | 14 - codex-rs/sandboxing/src/seatbelt.rs | 148 ++++ codex-rs/sandboxing/src/seatbelt_tests.rs | 79 +++ 21 files changed, 1245 insertions(+), 121 deletions(-) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index bd74d83ad6..3f27fe10e5 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -2334,6 +2334,7 @@ dependencies = [ "codex-protocol", "codex-sandboxing", "codex-utils-absolute-path", + "globset", "landlock", "libc", "pkg-config", @@ -2760,6 +2761,7 @@ dependencies = [ "dunce", "libc", "pretty_assertions", + "regex-lite", "serde_json", "tempfile", "tokio", diff --git a/codex-rs/config/src/permissions_toml.rs b/codex-rs/config/src/permissions_toml.rs index fcc3e006b1..cee68d7abb 100644 --- a/codex-rs/config/src/permissions_toml.rs +++ b/codex-rs/config/src/permissions_toml.rs @@ -31,6 +31,10 @@ pub struct PermissionProfileToml { #[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq, JsonSchema)] pub struct FilesystemPermissionsToml { + /// Optional maximum depth for expanding unreadable glob patterns on + /// platforms that snapshot glob matches before sandbox startup. + #[schemars(range(min = 1))] + pub glob_scan_max_depth: Option, #[serde(flatten)] pub entries: BTreeMap, } diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.json index 966555856b..67d051a699 100644 --- a/codex-rs/core/config.schema.json +++ b/codex-rs/core/config.schema.json @@ -673,6 +673,14 @@ ] }, "FilesystemPermissionsToml": { + "properties": { + "glob_scan_max_depth": { + "description": "Optional maximum depth for expanding unreadable glob patterns on platforms that snapshot glob matches before sandbox startup.", + "format": "uint", + "minimum": 1.0, + "type": "integer" + } + }, "type": "object" }, "ForcedLoginMethod": { diff --git a/codex-rs/core/src/config/config_tests.rs b/codex-rs/core/src/config/config_tests.rs index 34002b129a..028ec1686a 100644 --- a/codex-rs/core/src/config/config_tests.rs +++ b/codex-rs/core/src/config/config_tests.rs @@ -428,6 +428,7 @@ allow_upstream_proxy = false "workspace".to_string(), PermissionProfileToml { filesystem: Some(FilesystemPermissionsToml { + glob_scan_max_depth: None, entries: BTreeMap::from([ ( ":minimal".to_string(), @@ -482,6 +483,7 @@ async fn permissions_profiles_network_populates_runtime_network_proxy_spec() -> "workspace".to_string(), PermissionProfileToml { filesystem: Some(FilesystemPermissionsToml { + glob_scan_max_depth: None, entries: BTreeMap::from([( ":minimal".to_string(), FilesystemPermissionToml::Access(FileSystemAccessMode::Read), @@ -531,6 +533,7 @@ async fn permissions_profiles_network_disabled_by_default_does_not_start_proxy() "workspace".to_string(), PermissionProfileToml { filesystem: Some(FilesystemPermissionsToml { + glob_scan_max_depth: None, entries: BTreeMap::from([( ":minimal".to_string(), FilesystemPermissionToml::Access(FileSystemAccessMode::Read), @@ -576,6 +579,7 @@ async fn default_permissions_profile_populates_runtime_sandbox_policy() -> std:: "workspace".to_string(), PermissionProfileToml { filesystem: Some(FilesystemPermissionsToml { + glob_scan_max_depth: None, entries: BTreeMap::from([ ( ":minimal".to_string(), @@ -671,6 +675,7 @@ async fn project_root_glob_none_compiles_to_filesystem_pattern_entry() -> std::i "workspace".to_string(), PermissionProfileToml { filesystem: Some(FilesystemPermissionsToml { + glob_scan_max_depth: Some(2), entries: BTreeMap::from([( ":project_roots".to_string(), FilesystemPermissionToml::Scoped(BTreeMap::from([ @@ -693,6 +698,13 @@ async fn project_root_glob_none_compiles_to_filesystem_pattern_entry() -> std::i ) .await?; + assert_eq!( + config + .permissions + .file_system_sandbox_policy + .glob_scan_max_depth, + Some(2) + ); let expected_pattern = AbsolutePathBuf::resolve_path_against_base("**/*.env", cwd.path()) .to_string_lossy() .into_owned(); @@ -738,6 +750,7 @@ async fn permissions_profiles_require_default_permissions() -> std::io::Result<( "workspace".to_string(), PermissionProfileToml { filesystem: Some(FilesystemPermissionsToml { + glob_scan_max_depth: None, entries: BTreeMap::from([( ":minimal".to_string(), FilesystemPermissionToml::Access(FileSystemAccessMode::Read), @@ -781,6 +794,7 @@ async fn permissions_profiles_reject_writes_outside_workspace_root() -> std::io: "workspace".to_string(), PermissionProfileToml { filesystem: Some(FilesystemPermissionsToml { + glob_scan_max_depth: None, entries: BTreeMap::from([( external_write_path.to_string(), FilesystemPermissionToml::Access(FileSystemAccessMode::Write), @@ -824,6 +838,7 @@ async fn permissions_profiles_reject_nested_entries_for_non_project_roots() -> s "workspace".to_string(), PermissionProfileToml { filesystem: Some(FilesystemPermissionsToml { + glob_scan_max_depth: None, entries: BTreeMap::from([( ":minimal".to_string(), FilesystemPermissionToml::Scoped(BTreeMap::from([( @@ -883,6 +898,7 @@ async fn load_workspace_permission_profile( async fn permissions_profiles_allow_unknown_special_paths() -> std::io::Result<()> { let config = load_workspace_permission_profile(PermissionProfileToml { filesystem: Some(FilesystemPermissionsToml { + glob_scan_max_depth: None, entries: BTreeMap::from([( ":future_special_path".to_string(), FilesystemPermissionToml::Access(FileSystemAccessMode::Read), @@ -929,6 +945,7 @@ async fn permissions_profiles_allow_unknown_special_paths_with_nested_entries() -> std::io::Result<()> { let config = load_workspace_permission_profile(PermissionProfileToml { filesystem: Some(FilesystemPermissionsToml { + glob_scan_max_depth: None, entries: BTreeMap::from([( ":future_special_path".to_string(), FilesystemPermissionToml::Scoped(BTreeMap::from([( @@ -996,6 +1013,7 @@ async fn permissions_profiles_allow_missing_filesystem_with_warning() -> std::io async fn permissions_profiles_allow_empty_filesystem_with_warning() -> std::io::Result<()> { let config = load_workspace_permission_profile(PermissionProfileToml { filesystem: Some(FilesystemPermissionsToml { + glob_scan_max_depth: None, entries: BTreeMap::new(), }), network: None, @@ -1030,6 +1048,7 @@ async fn permissions_profiles_reject_project_root_parent_traversal() -> std::io: "workspace".to_string(), PermissionProfileToml { filesystem: Some(FilesystemPermissionsToml { + glob_scan_max_depth: None, entries: BTreeMap::from([( ":project_roots".to_string(), FilesystemPermissionToml::Scoped(BTreeMap::from([( @@ -1075,6 +1094,7 @@ async fn permissions_profiles_allow_network_enablement() -> std::io::Result<()> "workspace".to_string(), PermissionProfileToml { filesystem: Some(FilesystemPermissionsToml { + glob_scan_max_depth: None, entries: BTreeMap::from([( ":minimal".to_string(), FilesystemPermissionToml::Access(FileSystemAccessMode::Read), diff --git a/codex-rs/core/src/config/permissions.rs b/codex-rs/core/src/config/permissions.rs index 1609b696d8..943c82c0e9 100644 --- a/codex-rs/core/src/config/permissions.rs +++ b/codex-rs/core/src/config/permissions.rs @@ -66,6 +66,14 @@ pub(crate) fn compile_permission_profile( ), ); } + for pattern in unbounded_unreadable_globstar_paths(filesystem) { + push_warning( + startup_warnings, + format!( + "Filesystem deny-read glob `{pattern}` uses `**`. Non-macOS sandboxing does not support unbounded `**` natively; set `glob_scan_max_depth` in this filesystem profile to cap Linux glob expansion and silence this warning, or enumerate explicit depths such as `*.env`, `*/*.env`, and `*/*/*.env`." + ), + ); + } } for (path, permission) in &filesystem.entries { entries.extend(compile_filesystem_permission( @@ -82,12 +90,17 @@ pub(crate) fn compile_permission_profile( missing_filesystem_entries_warning(profile_name), ); } + let glob_scan_max_depth = validate_glob_scan_max_depth( + profile + .filesystem + .as_ref() + .and_then(|filesystem| filesystem.glob_scan_max_depth), + )?; let network_sandbox_policy = compile_network_sandbox_policy(profile.network.as_ref()); - Ok(( - FileSystemSandboxPolicy::restricted(entries), - network_sandbox_policy, - )) + let mut file_system_sandbox_policy = FileSystemSandboxPolicy::restricted(entries); + file_system_sandbox_policy.glob_scan_max_depth = glob_scan_max_depth; + Ok((file_system_sandbox_policy, network_sandbox_policy)) } /// Returns a list of paths that must be readable by shell tools in order @@ -333,6 +346,42 @@ fn unsupported_read_write_glob_paths(filesystem: &FilesystemPermissionsToml) -> patterns } +fn unbounded_unreadable_globstar_paths(filesystem: &FilesystemPermissionsToml) -> Vec { + if filesystem.glob_scan_max_depth.is_some() { + return Vec::new(); + } + + let mut patterns = Vec::new(); + for (path, permission) in &filesystem.entries { + match permission { + FilesystemPermissionToml::Access(FileSystemAccessMode::None) => { + if path.contains("**") { + patterns.push(path.clone()); + } + } + FilesystemPermissionToml::Access(_) => {} + FilesystemPermissionToml::Scoped(scoped_entries) => { + for (subpath, access) in scoped_entries { + if *access == FileSystemAccessMode::None && subpath.contains("**") { + patterns.push(format!("{path}/{subpath}")); + } + } + } + } + } + patterns +} + +fn validate_glob_scan_max_depth(max_depth: Option) -> io::Result> { + match max_depth { + Some(0) => Err(io::Error::new( + io::ErrorKind::InvalidInput, + "glob_scan_max_depth must be at least 1", + )), + _ => Ok(max_depth), + } +} + fn contains_glob_chars(path: &str) -> bool { path.chars().any(|ch| matches!(ch, '*' | '?' | '[' | ']')) } diff --git a/codex-rs/core/src/config/permissions_tests.rs b/codex-rs/core/src/config/permissions_tests.rs index 7116e32e70..e22376b214 100644 --- a/codex-rs/core/src/config/permissions_tests.rs +++ b/codex-rs/core/src/config/permissions_tests.rs @@ -55,6 +55,7 @@ async fn restricted_read_implicitly_allows_helper_executables() -> std::io::Resu "workspace".to_string(), PermissionProfileToml { filesystem: Some(FilesystemPermissionsToml { + glob_scan_max_depth: None, entries: BTreeMap::new(), }), network: None, @@ -226,6 +227,7 @@ fn network_toml_overlays_unix_socket_permissions_by_path() { #[test] fn read_write_glob_warnings_skip_supported_deny_read_globs_and_trailing_subpaths() { let filesystem = FilesystemPermissionsToml { + glob_scan_max_depth: None, entries: BTreeMap::from([ ( "/tmp/**/*.log".to_string(), @@ -256,6 +258,47 @@ fn read_write_glob_warnings_skip_supported_deny_read_globs_and_trailing_subpaths ); } +#[test] +fn unreadable_globstar_warning_is_suppressed_when_scan_depth_is_configured() { + let filesystem = FilesystemPermissionsToml { + glob_scan_max_depth: None, + entries: BTreeMap::from([( + ":project_roots".to_string(), + FilesystemPermissionToml::Scoped(BTreeMap::from([ + ("**/*.env".to_string(), FileSystemAccessMode::None), + ("*.pem".to_string(), FileSystemAccessMode::None), + ])), + )]), + }; + + assert_eq!( + unbounded_unreadable_globstar_paths(&filesystem), + vec![":project_roots/**/*.env".to_string()] + ); + + let configured_filesystem = FilesystemPermissionsToml { + glob_scan_max_depth: Some(2), + ..filesystem + }; + assert_eq!( + unbounded_unreadable_globstar_paths(&configured_filesystem), + Vec::::new() + ); +} + +#[test] +fn glob_scan_max_depth_must_be_positive() { + let err = validate_glob_scan_max_depth(Some(0)) + .expect_err("zero depth would silently skip deny-read glob expansion"); + + assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput); + assert_eq!(err.to_string(), "glob_scan_max_depth must be at least 1"); + assert_eq!( + validate_glob_scan_max_depth(Some(2)).expect("depth should be valid"), + Some(2) + ); +} + #[test] fn read_write_trailing_glob_suffix_compiles_as_subpath() -> std::io::Result<()> { let cwd = TempDir::new()?; @@ -266,6 +309,7 @@ fn read_write_trailing_glob_suffix_compiles_as_subpath() -> std::io::Result<()> "workspace".to_string(), PermissionProfileToml { filesystem: Some(FilesystemPermissionsToml { + glob_scan_max_depth: None, entries: BTreeMap::from([( ":project_roots".to_string(), FilesystemPermissionToml::Scoped(BTreeMap::from([( diff --git a/codex-rs/core/src/tools/handlers/unified_exec.rs b/codex-rs/core/src/tools/handlers/unified_exec.rs index 99c7e4a195..f2604a57d6 100644 --- a/codex-rs/core/src/tools/handlers/unified_exec.rs +++ b/codex-rs/core/src/tools/handlers/unified_exec.rs @@ -20,8 +20,10 @@ use crate::tools::registry::ToolHandler; use crate::tools::registry::ToolKind; use crate::unified_exec::ExecCommandRequest; use crate::unified_exec::UnifiedExecContext; +use crate::unified_exec::UnifiedExecError; use crate::unified_exec::UnifiedExecProcessManager; use crate::unified_exec::WriteStdinRequest; +use crate::unified_exec::generate_chunk_id; use codex_features::Feature; use codex_otel::SessionTelemetry; use codex_otel::TOOL_CALL_UNIFIED_EXEC_METRIC; @@ -30,6 +32,7 @@ use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::TerminalInteractionEvent; use codex_shell_command::is_safe_command::is_known_safe_command; use codex_tools::UnifiedExecShellMode; +use codex_utils_output_truncation::approx_token_count; use serde::Deserialize; use std::path::PathBuf; use std::sync::Arc; @@ -309,7 +312,8 @@ impl ToolHandler for UnifiedExecHandler { } emit_unified_exec_tty_metric(&turn.session_telemetry, tty); - manager + let session_command = command.clone(); + match manager .exec_command( ExecCommandRequest { command, @@ -330,11 +334,31 @@ impl ToolHandler for UnifiedExecHandler { &context, ) .await - .map_err(|err| { - FunctionCallError::RespondToModel(format!( + { + Ok(response) => response, + Err(UnifiedExecError::SandboxDenied { output, .. }) => { + let output_text = output.aggregated_output.text; + let original_token_count = approx_token_count(&output_text); + ExecCommandToolOutput { + event_call_id: context.call_id.clone(), + chunk_id: generate_chunk_id(), + wall_time: output.duration, + raw_output: output_text.into_bytes(), + max_output_tokens, + // Sandbox denial is terminal, so there is no live + // process for write_stdin to resume. + process_id: None, + exit_code: Some(output.exit_code), + original_token_count: Some(original_token_count), + session_command: Some(session_command), + } + } + Err(err) => { + return Err(FunctionCallError::RespondToModel(format!( "exec_command failed for `{command_for_display}`: {err:?}" - )) - })? + ))); + } + } } "write_stdin" => { let args: WriteStdinArgs = parse_arguments(&arguments)?; diff --git a/codex-rs/core/src/unified_exec/process_manager.rs b/codex-rs/core/src/unified_exec/process_manager.rs index aaaa7076f4..9f9189abcc 100644 --- a/codex-rs/core/src/unified_exec/process_manager.rs +++ b/codex-rs/core/src/unified_exec/process_manager.rs @@ -26,6 +26,7 @@ use crate::tools::orchestrator::ToolOrchestrator; use crate::tools::runtimes::unified_exec::UnifiedExecRequest as UnifiedExecToolRequest; use crate::tools::runtimes::unified_exec::UnifiedExecRuntime; use crate::tools::sandboxing::ToolCtx; +use crate::tools::sandboxing::ToolError; use crate::unified_exec::ExecCommandRequest; use crate::unified_exec::MAX_UNIFIED_EXEC_PROCESSES; use crate::unified_exec::MAX_YIELD_TIME_MS; @@ -50,6 +51,8 @@ use crate::unified_exec::process::OutputHandles; use crate::unified_exec::process::SpawnLifecycleHandle; use crate::unified_exec::process::UnifiedExecProcess; use codex_config::types::ShellEnvironmentPolicy; +use codex_protocol::error::CodexErr; +use codex_protocol::error::SandboxErr; use codex_protocol::protocol::ExecCommandSource; use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_output_truncation::approx_token_count; @@ -778,7 +781,19 @@ impl UnifiedExecProcessManager { ) .await .map(|result| (result.output, result.deferred_network_approval)) - .map_err(|e| UnifiedExecError::create_process(format!("{e:?}"))) + .map_err(|err| match err { + ToolError::Codex(CodexErr::Sandbox(SandboxErr::Denied { output, .. })) => { + let output = *output; + let message = if output.aggregated_output.text.is_empty() { + let exit_code = output.exit_code; + format!("Process exited with code {exit_code}") + } else { + output.aggregated_output.text.clone() + }; + UnifiedExecError::sandbox_denied(message, output) + } + other => UnifiedExecError::create_process(format!("{other:?}")), + }) } pub(super) async fn collect_output_until_deadline( diff --git a/codex-rs/core/tests/suite/tools.rs b/codex-rs/core/tests/suite/tools.rs index bc286ce5ea..795aa1c847 100644 --- a/codex-rs/core/tests/suite/tools.rs +++ b/codex-rs/core/tests/suite/tools.rs @@ -9,10 +9,15 @@ use std::time::Instant; use anyhow::Context; use anyhow::Result; +use codex_config::Constrained; use codex_config::types::McpServerConfig; use codex_config::types::McpServerTransportConfig; use codex_core::sandboxing::SandboxPermissions; use codex_features::Feature; +use codex_protocol::permissions::FileSystemAccessMode; +use codex_protocol::permissions::FileSystemPath; +use codex_protocol::permissions::FileSystemSandboxEntry; +use codex_protocol::permissions::FileSystemSandboxPolicy; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::SandboxPolicy; use core_test_support::assert_regex_match; @@ -26,6 +31,7 @@ use core_test_support::responses::mount_sse_sequence; use core_test_support::responses::sse; use core_test_support::responses::start_mock_server; use core_test_support::skip_if_no_network; +use core_test_support::skip_if_sandbox; use core_test_support::stdio_server_bin; use core_test_support::test_codex::test_codex; use regex_lite::Regex; @@ -379,7 +385,7 @@ async fn sandbox_denied_shell_returns_original_output() -> Result<()> { ]; let args = json!({ "command": command, - "timeout_ms": 1_000, + "timeout_ms": 5_000, }); let responses = vec![ @@ -449,6 +455,109 @@ async fn sandbox_denied_shell_returns_original_output() -> Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn shell_enforces_glob_deny_read_policy() -> Result<()> { + skip_if_no_network!(Ok(())); + skip_if_sandbox!(Ok(())); + + let server = start_mock_server().await; + let read_only_policy = SandboxPolicy::new_read_only_policy(); + let read_only_policy_for_config = read_only_policy.clone(); + let mut builder = test_codex() + .with_model("gpt-5.1-codex") + .with_config(move |config| { + config.permissions.sandbox_policy = Constrained::allow_any(read_only_policy_for_config); + let mut file_system_sandbox_policy = FileSystemSandboxPolicy::default(); + file_system_sandbox_policy + .entries + .push(FileSystemSandboxEntry { + path: FileSystemPath::GlobPattern { + pattern: format!("{}/**/*.env", config.cwd.as_path().display()), + }, + access: FileSystemAccessMode::None, + }); + config.permissions.file_system_sandbox_policy = file_system_sandbox_policy; + }); + let fixture = builder.build(&server).await?; + + let fixture_dir = fixture.workspace_path("glob-deny-read"); + fs::create_dir_all(&fixture_dir).context("create glob deny-read fixture directory")?; + let denied_path = fixture_dir.join("secret.env"); + let allowed_path = fixture_dir.join("notes.txt"); + let secret = "shell glob deny-read secret"; + let allowed = "shell glob deny-read allowed"; + fs::write(&denied_path, format!("{secret}\n")).context("write denied fixture")?; + fs::write(&allowed_path, format!("{allowed}\n")).context("write allowed fixture")?; + + let call_id = "shell-glob-deny-read"; + let command = vec![ + "/bin/sh".to_string(), + "-c".to_string(), + "status=0; cat \"$1\" || status=$?; cat \"$2\"; exit \"$status\"".to_string(), + "sh".to_string(), + denied_path.to_string_lossy().into_owned(), + allowed_path.to_string_lossy().into_owned(), + ]; + let args = json!({ + "command": command, + "timeout_ms": 1_000, + }); + + let responses = vec![ + sse(vec![ + ev_response_created("resp-1"), + ev_function_call(call_id, "shell", &serde_json::to_string(&args)?), + ev_completed("resp-1"), + ]), + sse(vec![ + ev_assistant_message("msg-1", "done"), + ev_completed("resp-2"), + ]), + ]; + let mock = mount_sse_sequence(&server, responses).await; + + fixture + .submit_turn_with_policy("read the fixture files", read_only_policy) + .await?; + + let output_text = mock + .function_call_output_text(call_id) + .context("shell output present")?; + let exit_code_line = output_text + .lines() + .next() + .context("exit code line present")?; + let exit_code = exit_code_line + .strip_prefix("Exit code: ") + .context("exit code prefix present")? + .trim() + .parse::() + .context("exit code is integer")?; + + assert_ne!( + exit_code, 0, + "glob deny-read should surface a non-zero exit code" + ); + assert!( + output_text.contains(allowed), + "expected allowed file contents in shell output: {output_text}" + ); + assert!( + !output_text.contains(secret), + "denied file contents leaked into shell output: {output_text}" + ); + let output_lower = output_text.to_lowercase(); + let has_denial = output_lower.contains("permission denied") + || output_lower.contains("operation not permitted") + || output_lower.contains("read-only file system"); + assert!( + has_denial, + "expected sandbox denial details in shell output: {output_text}" + ); + + Ok(()) +} + async fn collect_tools(use_unified_exec: bool) -> Result> { let server = start_mock_server().await; diff --git a/codex-rs/core/tests/suite/unified_exec.rs b/codex-rs/core/tests/suite/unified_exec.rs index b14c930999..77a15e92c0 100644 --- a/codex-rs/core/tests/suite/unified_exec.rs +++ b/codex-rs/core/tests/suite/unified_exec.rs @@ -5,8 +5,13 @@ use std::sync::OnceLock; use anyhow::Context; use anyhow::Result; +use codex_config::Constrained; use codex_exec_server::CreateDirectoryOptions; use codex_features::Feature; +use codex_protocol::permissions::FileSystemAccessMode; +use codex_protocol::permissions::FileSystemPath; +use codex_protocol::permissions::FileSystemSandboxEntry; +use codex_protocol::permissions::FileSystemSandboxPolicy; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::ExecCommandSource; @@ -2341,6 +2346,127 @@ async fn unified_exec_runs_under_sandbox() -> Result<()> { Ok(()) } +#[cfg(unix)] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn unified_exec_enforces_glob_deny_read_policy() -> Result<()> { + skip_if_no_network!(Ok(())); + skip_if_sandbox!(Ok(())); + + let server = start_mock_server().await; + let read_only_policy = SandboxPolicy::new_read_only_policy(); + let read_only_policy_for_config = read_only_policy.clone(); + let mut builder = test_codex().with_config(move |config| { + config + .features + .enable(Feature::UnifiedExec) + .expect("test config should allow feature update"); + config.permissions.sandbox_policy = Constrained::allow_any(read_only_policy_for_config); + let mut file_system_sandbox_policy = FileSystemSandboxPolicy::default(); + file_system_sandbox_policy + .entries + .push(FileSystemSandboxEntry { + path: FileSystemPath::GlobPattern { + pattern: format!("{}/**/*.env", config.cwd.as_path().display()), + }, + access: FileSystemAccessMode::None, + }); + config.permissions.file_system_sandbox_policy = file_system_sandbox_policy; + }); + let TestCodex { + codex, + cwd, + session_configured, + .. + } = builder.build(&server).await?; + + let fixture_dir = cwd.path().join("glob-deny-read"); + fs::create_dir_all(&fixture_dir).context("create glob deny-read fixture directory")?; + let denied_path = fixture_dir.join("secret.env"); + let allowed_path = fixture_dir.join("notes.txt"); + let secret = "unified exec glob deny-read secret"; + let allowed = "unified exec glob deny-read allowed"; + fs::write(&denied_path, format!("{secret}\n")).context("write denied fixture")?; + fs::write(&allowed_path, format!("{allowed}\n")).context("write allowed fixture")?; + + let call_id = "uexec-glob-deny-read"; + let cmd = format!( + "read_status=0; cat {denied_path:?} || read_status=$?; cat {allowed_path:?}; exit $read_status" + ); + let args = serde_json::json!({ + "cmd": cmd, + "yield_time_ms": 5_000, + }); + + let responses = vec![ + sse(vec![ + ev_response_created("resp-1"), + ev_function_call(call_id, "exec_command", &serde_json::to_string(&args)?), + ev_completed("resp-1"), + ]), + sse(vec![ + ev_assistant_message("msg-1", "done"), + ev_completed("resp-2"), + ]), + ]; + let request_log = mount_sse_sequence(&server, responses).await; + + let session_model = session_configured.model.clone(); + codex + .submit(Op::UserTurn { + items: vec![UserInput::Text { + text: "read the fixture files".into(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + cwd: cwd.path().to_path_buf(), + approval_policy: AskForApproval::Never, + approvals_reviewer: None, + sandbox_policy: read_only_policy, + model: session_model, + effort: None, + summary: None, + service_tier: None, + collaboration_mode: None, + personality: None, + }) + .await?; + + wait_for_event(&codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; + + let requests = request_log.requests(); + assert!(!requests.is_empty(), "expected at least one POST request"); + let bodies = requests + .into_iter() + .map(|request| request.body_json()) + .collect::>(); + + let outputs = collect_tool_outputs(&bodies)?; + let output = outputs.get(call_id).expect("missing output"); + + assert!( + output.exit_code.is_some_and(|code| code != 0), + "glob deny-read should surface a non-zero exit code: {output:?}" + ); + assert!( + output.output.contains(allowed), + "expected allowed file contents in unified exec output: {output:?}" + ); + assert!( + !output.output.contains(secret), + "denied file contents leaked into unified exec output: {output:?}" + ); + let output_lower = output.output.to_lowercase(); + let has_denial = output_lower.contains("permission denied") + || output_lower.contains("operation not permitted") + || output_lower.contains("read-only file system"); + assert!( + has_denial, + "expected sandbox denial details in unified exec output: {output:?}" + ); + + Ok(()) +} + #[cfg(target_os = "macos")] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn unified_exec_python_prompt_under_seatbelt() -> Result<()> { diff --git a/codex-rs/linux-sandbox/Cargo.toml b/codex-rs/linux-sandbox/Cargo.toml index 751d6d5d17..c624251e63 100644 --- a/codex-rs/linux-sandbox/Cargo.toml +++ b/codex-rs/linux-sandbox/Cargo.toml @@ -20,6 +20,7 @@ clap = { workspace = true, features = ["derive"] } codex-protocol = { workspace = true } codex-sandboxing = { workspace = true } codex-utils-absolute-path = { workspace = true } +globset = { workspace = true } landlock = { workspace = true } libc = { workspace = true } seccompiler = { workspace = true } diff --git a/codex-rs/linux-sandbox/README.md b/codex-rs/linux-sandbox/README.md index 3a004934b2..5745f4816c 100644 --- a/codex-rs/linux-sandbox/README.md +++ b/codex-rs/linux-sandbox/README.md @@ -7,7 +7,7 @@ This crate is responsible for producing: - the `codex-exec` CLI can check if its arg0 is `codex-linux-sandbox` and, if so, execute as if it were `codex-linux-sandbox` - this should also be true of the `codex` multitool CLI -On Linux, the bubblewrap pipeline prefers the first `bwrap` found on `PATH` +On Linux, Codex prefers the first `bwrap` found on `PATH` outside the current working directory whenever it is available. If `bwrap` is present but too old to support `--argv0`, the helper keeps using system bubblewrap and switches to a @@ -23,7 +23,7 @@ commands that would enter the bubblewrap path. **Current Behavior** - Legacy `SandboxPolicy` / `sandbox_mode` configs remain supported. -- Bubblewrap is the default filesystem sandbox pipeline. +- Bubblewrap is the default filesystem sandbox. - If `bwrap` is present on `PATH` outside the current working directory, the helper uses it. - If `bwrap` is present but too old to support `--argv0`, the helper uses a @@ -46,32 +46,51 @@ commands that would enter the bubblewrap path. - Split-only filesystem policies that do not round-trip through the legacy `SandboxPolicy` model stay on bubblewrap so nested read-only or denied carveouts are preserved. -- When the default bubblewrap pipeline is active, the helper applies `PR_SET_NO_NEW_PRIVS` and a +- When bubblewrap is active, the helper applies `PR_SET_NO_NEW_PRIVS` and a seccomp network filter in-process. -- When the default bubblewrap pipeline is active, the filesystem is read-only by default via `--ro-bind / /`. -- When the default bubblewrap pipeline is active, writable roots are layered with `--bind `. -- When the default bubblewrap pipeline is active, protected subpaths under writable roots (for +- When bubblewrap is active, the filesystem is read-only by default via `--ro-bind / /`. +- When bubblewrap is active, writable roots are layered with `--bind `. +- When bubblewrap is active, protected subpaths under writable roots (for example `.git`, resolved `gitdir:`, and `.codex`) are re-applied as read-only via `--ro-bind`. -- When the default bubblewrap pipeline is active, overlapping split-policy +- When bubblewrap is active, overlapping split-policy entries are applied in path-specificity order so narrower writable children can reopen broader read-only or denied parents while narrower denied subpaths still win. For example, `/repo = write`, `/repo/a = none`, `/repo/a/b = write` keeps `/repo` writable, denies `/repo/a`, and reopens `/repo/a/b` as writable again. -- When the default bubblewrap pipeline is active, symlink-in-path and non-existent protected paths inside +- When bubblewrap is active, unreadable glob entries are expanded before + launching the sandbox and matching files are masked in bubblewrap: + + ```text + Prefer: rg --files --hidden --no-ignore --glob -- + Fallback: internal globset walker when rg is not installed + Failure: any other rg failure aborts sandbox construction + ``` + + Users can cap the scan depth per permissions profile: + + ```toml + [permissions.workspace.filesystem] + glob_scan_max_depth = 2 + + [permissions.workspace.filesystem.":project_roots"] + "**/*.env" = "none" + ``` + +- When bubblewrap is active, symlink-in-path and non-existent protected paths inside writable roots are blocked by mounting `/dev/null` on the symlink or first missing component. -- When the default bubblewrap pipeline is active, the helper explicitly isolates the user namespace via +- When bubblewrap is active, the helper explicitly isolates the user namespace via `--unshare-user` and the PID namespace via `--unshare-pid`. -- When the default bubblewrap pipeline is active and network is restricted without proxy routing, the helper also +- When bubblewrap is active and network is restricted without proxy routing, the helper also isolates the network namespace via `--unshare-net`. - In managed proxy mode, the helper uses `--unshare-net` plus an internal TCP->UDS->TCP routing bridge so tool traffic reaches only configured proxy endpoints. - In managed proxy mode, after the bridge is live, seccomp blocks new AF_UNIX/socketpair creation for the user command. -- When the default bubblewrap pipeline is active, it mounts a fresh `/proc` via `--proc /proc` by default, but +- When bubblewrap is active, it mounts a fresh `/proc` via `--proc /proc` by default, but you can skip this in restrictive container environments with `--no-proc`. **Notes** diff --git a/codex-rs/linux-sandbox/src/bwrap.rs b/codex-rs/linux-sandbox/src/bwrap.rs index 91d6621029..64d1342bef 100644 --- a/codex-rs/linux-sandbox/src/bwrap.rs +++ b/codex-rs/linux-sandbox/src/bwrap.rs @@ -9,17 +9,27 @@ //! The overall Linux sandbox is composed of: //! - seccomp + `PR_SET_NO_NEW_PRIVS` applied in-process, and //! - bubblewrap used to construct the filesystem view before exec. +use std::collections::BTreeMap; use std::collections::BTreeSet; use std::collections::HashSet; +use std::ffi::OsString; use std::fs; use std::fs::File; +use std::io; use std::os::fd::AsRawFd; +use std::os::unix::ffi::OsStringExt; use std::path::Path; use std::path::PathBuf; +use std::process::Command; +use codex_protocol::error::CodexErr; use codex_protocol::error::Result; use codex_protocol::protocol::FileSystemSandboxPolicy; +use codex_protocol::protocol::WritableRoot; use codex_utils_absolute_path::AbsolutePathBuf; +use globset::GlobBuilder; +use globset::GlobSet; +use globset::GlobSetBuilder; /// Linux "platform defaults" that keep common system binaries and dynamic /// libraries readable when `ReadOnlyAccess::Restricted` requests them. @@ -37,6 +47,8 @@ const LINUX_PLATFORM_DEFAULT_READ_ROOTS: &[&str] = &[ "/run/current-system/sw", ]; +const MAX_UNREADABLE_GLOB_MATCHES: usize = 8192; + /// Options that control how bubblewrap is invoked. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) struct BwrapOptions { @@ -47,6 +59,11 @@ pub(crate) struct BwrapOptions { pub mount_proc: bool, /// How networking should be configured inside the bubblewrap sandbox. pub network_mode: BwrapNetworkMode, + /// Optional maximum depth for expanding unreadable glob patterns with ripgrep. + /// + /// Keep this uncapped by default so existing nested deny-read matches are + /// masked before the sandboxed command starts. + pub glob_scan_max_depth: Option, } impl Default for BwrapOptions { @@ -54,6 +71,7 @@ impl Default for BwrapOptions { Self { mount_proc: true, network_mode: BwrapNetworkMode::FullAccess, + glob_scan_max_depth: None, } } } @@ -99,7 +117,11 @@ pub(crate) fn create_bwrap_command_args( command_cwd: &Path, options: BwrapOptions, ) -> Result { - if file_system_sandbox_policy.has_full_disk_write_access() { + let unreadable_globs = + file_system_sandbox_policy.get_unreadable_globs_with_cwd(sandbox_policy_cwd); + // Full disk write normally skips bwrap, but unreadable glob patterns still + // need concrete bwrap masks for the matches expanded below. + if file_system_sandbox_policy.has_full_disk_write_access() && unreadable_globs.is_empty() { return if options.network_mode == BwrapNetworkMode::FullAccess { Ok(BwrapArgs { args: command, @@ -157,7 +179,13 @@ fn create_bwrap_flags( let BwrapArgs { args: filesystem_args, preserved_files, - } = create_filesystem_args(file_system_sandbox_policy, sandbox_policy_cwd)?; + } = create_filesystem_args( + file_system_sandbox_policy, + sandbox_policy_cwd, + options + .glob_scan_max_depth + .or(file_system_sandbox_policy.glob_scan_max_depth), + )?; let normalized_command_cwd = normalize_command_cwd_for_bwrap(command_cwd); let mut args = Vec::new(); args.push("--new-session".to_string()); @@ -210,16 +238,41 @@ fn create_bwrap_flags( fn create_filesystem_args( file_system_sandbox_policy: &FileSystemSandboxPolicy, cwd: &Path, + glob_scan_max_depth: Option, ) -> Result { + let unreadable_globs = file_system_sandbox_policy.get_unreadable_globs_with_cwd(cwd); // Bubblewrap requires bind mount targets to exist. Skip missing writable // roots so mixed-platform configs can keep harmless paths for other // environments without breaking Linux command startup. - let writable_roots = file_system_sandbox_policy + let mut writable_roots = file_system_sandbox_policy .get_writable_roots_with_cwd(cwd) .into_iter() .filter(|writable_root| writable_root.root.as_path().exists()) .collect::>(); - let unreadable_roots = file_system_sandbox_policy.get_unreadable_roots_with_cwd(cwd); + if writable_roots.is_empty() + && file_system_sandbox_policy.has_full_disk_write_access() + && !unreadable_globs.is_empty() + { + writable_roots.push(WritableRoot { + root: AbsolutePathBuf::from_absolute_path("/")?, + read_only_subpaths: Vec::new(), + }); + } + let mut unreadable_roots = file_system_sandbox_policy + .get_unreadable_roots_with_cwd(cwd) + .into_iter() + .map(AbsolutePathBuf::into_path_buf) + .collect::>(); + // Bubblewrap can only mask concrete paths. Expand unreadable glob patterns + // to the existing matches we can see before constructing the mount overlay; + // core tool helpers still evaluate the original patterns directly at read time. + unreadable_roots.extend( + expand_unreadable_globs_with_ripgrep(&unreadable_globs, cwd, glob_scan_max_depth)? + .into_iter() + .map(AbsolutePathBuf::into_path_buf), + ); + unreadable_roots.sort(); + unreadable_roots.dedup(); let mut args = if file_system_sandbox_policy.has_full_disk_read_access() { // Read-only root, then mount a minimal device tree. @@ -303,10 +356,7 @@ fn create_filesystem_args( allowed_write_paths.push(target); } } - let unreadable_paths: HashSet = unreadable_roots - .iter() - .map(|path| path.as_path().to_path_buf()) - .collect(); + let unreadable_paths: HashSet = unreadable_roots.iter().cloned().collect(); let mut sorted_writable_roots = writable_roots; sorted_writable_roots.sort_by_key(|writable_root| path_depth(writable_root.root.as_path())); // Mask only the unreadable ancestors that sit outside every writable root. @@ -323,7 +373,7 @@ fn create_filesystem_args( .iter() .any(|root| root.starts_with(unreadable_root)) }) - .map(|path| path.as_path().to_path_buf()) + .cloned() .collect(); unreadable_ancestors_of_writable_roots.sort_by_key(|path| path_depth(path)); @@ -343,7 +393,7 @@ fn create_filesystem_args( // target parents before binding the narrower writable descendant. if let Some(masking_root) = unreadable_roots .iter() - .map(AbsolutePathBuf::as_path) + .map(PathBuf::as_path) .filter(|unreadable_root| root.starts_with(unreadable_root)) .max_by_key(|unreadable_root| path_depth(unreadable_root)) { @@ -366,12 +416,12 @@ fn create_filesystem_args( } read_only_subpaths.sort_by_key(|path| path_depth(path)); for subpath in read_only_subpaths { - append_read_only_subpath_args(&mut args, &subpath, &allowed_write_paths); + append_read_only_subpath_args(&mut args, &subpath, &allowed_write_paths)?; } let mut nested_unreadable_roots: Vec = unreadable_roots .iter() - .filter(|path| path.as_path().starts_with(root)) - .map(|path| path.as_path().to_path_buf()) + .filter(|path| path.starts_with(root)) + .cloned() .collect(); if let Some(target) = &symlink_target { nested_unreadable_roots = @@ -396,7 +446,7 @@ fn create_filesystem_args( .iter() .any(|root| unreadable_root.starts_with(root) || root.starts_with(unreadable_root)) }) - .map(|path| path.as_path().to_path_buf()) + .cloned() .collect(); rootless_unreadable_roots.sort_by_key(|path| path_depth(path)); for unreadable_root in rootless_unreadable_roots { @@ -414,6 +464,244 @@ fn create_filesystem_args( }) } +fn expand_unreadable_globs_with_ripgrep( + patterns: &[String], + cwd: &Path, + max_depth: Option, +) -> Result> { + if patterns.is_empty() || max_depth == Some(0) { + return Ok(Vec::new()); + } + + // Group each pattern by the static path prefix before its first glob + // metacharacter. That keeps scans narrow, avoids searching from `/`, and + // lets one `rg --files` call handle all patterns under the same root. + let mut patterns_by_search_root: BTreeMap> = BTreeMap::new(); + for pattern in patterns { + if let Some((search_root, glob)) = split_pattern_for_ripgrep(pattern, cwd) + && search_root.as_path().is_dir() + { + patterns_by_search_root + .entry(search_root) + .or_default() + .push(glob); + } + } + + // Record both the logical match and any canonical symlink target. The bwrap + // overlay needs the resolved target to prevent a readable symlink path from + // bypassing an unreadable glob match. + let mut expanded_paths = BTreeSet::new(); + for (search_root, globs) in patterns_by_search_root { + for path in ripgrep_files(search_root.as_path(), &globs, max_depth)? { + if let Some(target) = canonical_target_if_symlinked_path(path.as_path()) { + expanded_paths.insert(AbsolutePathBuf::from_absolute_path_checked(target)?); + } + expanded_paths.insert(path); + if expanded_paths.len() > MAX_UNREADABLE_GLOB_MATCHES { + return Err(CodexErr::Fatal(format!( + "unreadable glob expansion for {} matched more than {MAX_UNREADABLE_GLOB_MATCHES} paths", + search_root.display() + ))); + } + } + } + + Ok(expanded_paths.into_iter().collect()) +} + +fn split_pattern_for_ripgrep(pattern: &str, cwd: &Path) -> Option<(AbsolutePathBuf, String)> { + // Resolve relative patterns once, then split at the first glob + // metacharacter. The prefix becomes the search root and the suffix stays as + // the ripgrep glob. Root-level glob scans are intentionally skipped because + // they are too broad for startup-time sandbox construction. + let absolute_pattern = AbsolutePathBuf::resolve_path_against_base(pattern, cwd); + let pattern = absolute_pattern.to_string_lossy(); + let first_glob_index = pattern + .char_indices() + .find_map(|(index, ch)| matches!(ch, '*' | '?' | '[' | ']').then_some(index))?; + let static_prefix = &pattern[..first_glob_index]; + if static_prefix.is_empty() || static_prefix == "/" { + return None; + } + let search_root_end = if static_prefix.ends_with('/') { + static_prefix.len() - 1 + } else { + static_prefix.rfind('/').unwrap_or(0) + }; + let search_root = if search_root_end == 0 { + PathBuf::from("/") + } else { + PathBuf::from(&pattern[..search_root_end]) + }; + let search_root = AbsolutePathBuf::from_absolute_path_checked(search_root).ok()?; + let glob = escape_unclosed_glob_classes(&pattern[search_root_end + 1..]); + (!glob.is_empty()).then_some((search_root, glob)) +} + +fn escape_unclosed_glob_classes(glob: &str) -> String { + // The filesystem policy accepts an unclosed `[` as a literal. Ripgrep treats + // that as invalid glob syntax, so escape only the unclosed class opener. + let mut escaped = String::with_capacity(glob.len()); + let mut chars = glob.chars(); + + while let Some(ch) = chars.next() { + if ch != '[' { + escaped.push(ch); + continue; + } + + let mut class = String::new(); + let mut closed = false; + for class_ch in chars.by_ref() { + if class_ch == ']' { + closed = true; + break; + } + class.push(class_ch); + } + + if closed { + escaped.push('['); + escaped.push_str(&class); + escaped.push(']'); + } else { + escaped.push_str(r"\["); + escaped.push_str(&class); + } + } + + escaped +} + +fn ripgrep_files( + search_root: &Path, + globs: &[String], + max_depth: Option, +) -> Result> { + // Use `rg --files` rather than shell expansion so dotfiles and ignored files + // are still considered. A status 1 with no stderr is ripgrep's "no matches" + // case, not a sandbox construction error. + let mut command = Command::new("rg"); + command + .arg("--files") + .arg("--hidden") + .arg("--no-ignore") + .arg("--null"); + if let Some(max_depth) = max_depth { + command.arg("--max-depth").arg(max_depth.to_string()); + } + for glob in globs { + command.arg("--glob").arg(glob); + } + command.arg("--").arg(search_root); + + /* + * Prefer ripgrep for unreadable glob expansion because it is fast and + * already implements the file-walking semantics we want here: include + * dotfiles, ignore ignore files, and do not recurse through symlinked + * directories. If `rg` is not installed in the runtime environment, fall + * back to the internal globset walker so sandbox construction still masks + * matching paths. Other ripgrep failures stay fatal so deny-read does not + * silently weaken. + */ + let output = match command.output() { + Ok(output) => output, + Err(err) if err.kind() == io::ErrorKind::NotFound => { + return glob_files(search_root, globs, max_depth); + } + Err(err) => return Err(err.into()), + }; + if !output.status.success() { + if output.status.code() == Some(1) && output.stderr.is_empty() { + return Ok(Vec::new()); + } + + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(CodexErr::Fatal(format!( + "ripgrep unreadable glob scan failed for {}: {stderr}", + search_root.display() + ))); + } + + let paths = output + .stdout + .split(|byte| *byte == b'\0') + .filter(|path| !path.is_empty()) + .map(|path| { + let path = PathBuf::from(OsString::from_vec(path.to_vec())); + if path.is_absolute() { + path + } else { + search_root.join(path) + } + }) + .map(AbsolutePathBuf::from_absolute_path_checked) + .collect::>>()?; + Ok(paths) +} + +fn glob_files( + search_root: &Path, + globs: &[String], + max_depth: Option, +) -> Result> { + let mut builder = GlobSetBuilder::new(); + for glob in globs { + let glob = GlobBuilder::new(glob) + .literal_separator(true) + .allow_unclosed_class(true) + .build() + .map_err(|err| { + CodexErr::Fatal(format!( + "unreadable glob pattern is invalid for {}: {err}", + search_root.display() + )) + })?; + builder.add(glob); + } + let glob_set = builder.build().map_err(|err| { + CodexErr::Fatal(format!( + "unreadable glob matcher failed for {}: {err}", + search_root.display() + )) + })?; + + let mut paths = Vec::new(); + collect_glob_files(search_root, search_root, &glob_set, max_depth, &mut paths)?; + Ok(paths) +} + +fn collect_glob_files( + search_root: &Path, + dir: &Path, + glob_set: &GlobSet, + remaining_depth: Option, + paths: &mut Vec, +) -> Result<()> { + for entry in fs::read_dir(dir)? { + let entry = entry?; + let path = entry.path(); + let file_type = entry.file_type()?; + let relative = path.strip_prefix(search_root).unwrap_or(path.as_path()); + + if (file_type.is_file() || file_type.is_symlink()) && glob_set.is_match(relative) { + paths.push(AbsolutePathBuf::from_absolute_path_checked(&path)?); + } + + if !file_type.is_dir() { + continue; + } + let remaining_depth = match remaining_depth { + Some(0 | 1) => continue, + Some(depth) => Some(depth - 1), + None => None, + }; + collect_glob_files(search_root, &path, glob_set, remaining_depth, paths)?; + } + Ok(()) +} + fn path_to_string(path: &Path) -> String { path.to_string_lossy().to_string() } @@ -423,6 +711,9 @@ fn path_depth(path: &Path) -> usize { } fn canonical_target_if_symlinked_path(path: &Path) -> Option { + // Return the fully resolved target only when some path component is a + // symlink. Callers use this to bind/mask the real filesystem location while + // leaving ordinary paths in their logical form. let mut current = PathBuf::new(); for component in path.components() { use std::path::Component; @@ -498,17 +789,19 @@ fn append_read_only_subpath_args( args: &mut Vec, subpath: &Path, allowed_write_paths: &[PathBuf], -) { - if let Some(target) = canonical_target_for_symlink_in_path(subpath, allowed_write_paths) { - // bwrap takes `--ro-bind `. Use the resolved target - // for both operands so a protected symlinked directory is remounted - // read-only in place instead of binding onto the symlink path itself. - let mount_source = path_to_string(&target); - let mount_destination = path_to_string(&target); - args.push("--ro-bind".to_string()); - args.push(mount_source); - args.push(mount_destination); - return; +) -> Result<()> { + if let Some(symlink) = first_writable_symlink_component_in_path(subpath, allowed_write_paths) { + /* + * A read-only carveout under a writable symlink cannot be made reliable + * with bwrap path arguments. Binding the symlink's current target would + * only protect a startup-time snapshot; the sandboxed process could + * replace the writable symlink before it reads through the logical path. + */ + return Err(CodexErr::Fatal(format!( + "cannot enforce sandbox read-only path {} because it crosses writable symlink {}", + subpath.display(), + symlink.display() + ))); } if !subpath.exists() { @@ -519,7 +812,7 @@ fn append_read_only_subpath_args( args.push("/dev/null".to_string()); args.push(path_to_string(&first_missing_component)); } - return; + return Ok(()); } if is_within_allowed_write_paths(subpath, allowed_write_paths) { @@ -527,6 +820,7 @@ fn append_read_only_subpath_args( args.push(path_to_string(subpath)); args.push(path_to_string(subpath)); } + Ok(()) } fn append_unreadable_root_args( @@ -535,16 +829,21 @@ fn append_unreadable_root_args( unreadable_root: &Path, allowed_write_paths: &[PathBuf], ) -> Result<()> { - if let Some(target) = canonical_target_for_symlink_in_path(unreadable_root, allowed_write_paths) + if let Some(symlink) = + first_writable_symlink_component_in_path(unreadable_root, allowed_write_paths) { - // Apply unreadable handling to the resolved symlink target, not the - // logical symlink path, to avoid file-vs-directory bind mismatches. - return append_existing_unreadable_path_args( - args, - preserved_files, - &target, - allowed_write_paths, - ); + /* + * Deny-read masks must fail closed when the protected path crosses a + * symlink that remains writable to the sandboxed process. Resolving and + * masking the symlink's current target is a TOCTTOU snapshot: bwrap would + * protect the old target while the logical path could later point + * somewhere else. + */ + return Err(CodexErr::Fatal(format!( + "cannot enforce sandbox deny-read path {} because it crosses writable symlink {}", + unreadable_root.display(), + symlink.display() + ))); } if !unreadable_root.exists() { @@ -621,10 +920,16 @@ fn is_within_allowed_write_paths(path: &Path, allowed_write_paths: &[PathBuf]) - .any(|root| path.starts_with(root)) } -fn canonical_target_for_symlink_in_path( +fn first_writable_symlink_component_in_path( target_path: &Path, allowed_write_paths: &[PathBuf], ) -> Option { + /* + * Walk the logical path and report the first symlink component that lives + * under a writable root. These symlinks are mutable from inside the sandbox, + * so any mount or mask based on their resolved target would be racing a path + * the sandboxed process can change. + */ let mut current = PathBuf::new(); for component in target_path.components() { @@ -651,7 +956,7 @@ fn canonical_target_for_symlink_in_path( if metadata.file_type().is_symlink() && is_within_allowed_write_paths(¤t, allowed_write_paths) { - return fs::canonicalize(¤t).ok(); + return Some(current); } } @@ -703,6 +1008,26 @@ mod tests { use pretty_assertions::assert_eq; use tempfile::TempDir; + const NO_UNREADABLE_GLOB_SCAN_MAX_DEPTH: Option = None; + + #[test] + fn default_unreadable_glob_scan_has_no_depth_cap() { + assert_eq!(BwrapOptions::default().glob_scan_max_depth, None); + } + + fn unreadable_glob_entry(pattern: String) -> FileSystemSandboxEntry { + FileSystemSandboxEntry { + path: FileSystemPath::GlobPattern { pattern }, + access: FileSystemAccessMode::None, + } + } + + fn default_policy_with_unreadable_glob(pattern: String) -> FileSystemSandboxPolicy { + let mut policy = FileSystemSandboxPolicy::default(); + policy.entries.push(unreadable_glob_entry(pattern)); + policy + } + #[test] fn full_disk_write_full_network_returns_unwrapped_command() { let command = vec!["/bin/true".to_string()]; @@ -714,6 +1039,7 @@ mod tests { BwrapOptions { mount_proc: true, network_mode: BwrapNetworkMode::FullAccess, + ..Default::default() }, ) .expect("create bwrap args"); @@ -732,6 +1058,7 @@ mod tests { BwrapOptions { mount_proc: true, network_mode: BwrapNetworkMode::ProxyOnly, + ..Default::default() }, ) .expect("create bwrap args"); @@ -755,6 +1082,42 @@ mod tests { ); } + #[test] + fn full_disk_write_with_unreadable_glob_still_wraps_and_masks_match() { + if !ripgrep_available() { + return; + } + + let temp_dir = TempDir::new().expect("temp dir"); + let root_env = temp_dir.path().join(".env"); + std::fs::write(&root_env, "secret").expect("write env"); + let policy = FileSystemSandboxPolicy::restricted(vec![ + FileSystemSandboxEntry { + path: FileSystemPath::Special { + value: FileSystemSpecialPath::Root, + }, + access: FileSystemAccessMode::Write, + }, + unreadable_glob_entry(format!("{}/**/*.env", temp_dir.path().display())), + ]); + let command = vec!["/bin/true".to_string()]; + + let args = create_bwrap_command_args( + command.clone(), + &policy, + temp_dir.path(), + temp_dir.path(), + BwrapOptions::default(), + ) + .expect("create bwrap args"); + + assert_ne!( + args.args, command, + "full-write policy with unreadable globs must still use bwrap" + ); + assert_file_masked(&args.args, &root_env); + } + #[cfg(unix)] #[test] fn restricted_policy_chdirs_to_canonical_command_cwd() { @@ -859,7 +1222,9 @@ mod tests { }, ]); - let args = create_filesystem_args(&policy, temp_dir.path()).expect("filesystem args"); + let args = + create_filesystem_args(&policy, temp_dir.path(), NO_UNREADABLE_GLOB_SCAN_MAX_DEPTH) + .expect("filesystem args"); assert!(args.args.windows(3).any(|window| { window == ["--bind", real_root_str.as_str(), real_root_str.as_str()] @@ -902,7 +1267,9 @@ mod tests { access: FileSystemAccessMode::Write, }]); - let args = create_filesystem_args(&policy, temp_dir.path()).expect("filesystem args"); + let args = + create_filesystem_args(&policy, temp_dir.path(), NO_UNREADABLE_GLOB_SCAN_MAX_DEPTH) + .expect("filesystem args"); assert!(args.args.windows(3).any(|window| { window @@ -924,7 +1291,7 @@ mod tests { #[cfg(unix)] #[test] - fn protected_symlinked_directory_subpaths_bind_target_read_only() { + fn protected_symlinked_directory_subpaths_fail_closed() { let temp_dir = TempDir::new().expect("temp dir"); let root = temp_dir.path().join("root"); let agents_target = root.join("agents-target"); @@ -934,28 +1301,26 @@ mod tests { let root = AbsolutePathBuf::from_absolute_path(&root).expect("absolute root"); let agents_link_str = path_to_string(&agents_link); - let agents_target_str = path_to_string(&agents_target); let policy = FileSystemSandboxPolicy::restricted(vec![FileSystemSandboxEntry { path: FileSystemPath::Path { path: root }, access: FileSystemAccessMode::Write, }]); - let args = create_filesystem_args(&policy, temp_dir.path()).expect("filesystem args"); + let err = + create_filesystem_args(&policy, temp_dir.path(), NO_UNREADABLE_GLOB_SCAN_MAX_DEPTH) + .expect_err("protected symlinked subpath should fail closed"); + let message = err.to_string(); - assert!(args.args.windows(3).any(|window| { - window - == [ - "--ro-bind", - agents_target_str.as_str(), - agents_target_str.as_str(), - ] - })); - assert!(!args.args.iter().any(|arg| arg == agents_link_str.as_str())); + assert!( + message.contains("cannot enforce sandbox read-only path"), + "{message}" + ); + assert!(message.contains(&agents_link_str), "{message}"); } #[cfg(unix)] #[test] - fn symlinked_writable_roots_mask_nested_symlink_escape_paths_without_binding_targets() { + fn symlinked_writable_roots_nested_symlink_escape_paths_fail_closed() { let temp_dir = TempDir::new().expect("temp dir"); let real_root = temp_dir.path().join("real"); let link_root = temp_dir.path().join("link"); @@ -971,7 +1336,6 @@ mod tests { AbsolutePathBuf::from_absolute_path(&link_root).expect("absolute symlinked root"); let link_private = link_root.join("linked-private"); let real_linked_private_str = path_to_string(&linked_private); - let outside_str = path_to_string(&outside); let policy = FileSystemSandboxPolicy::restricted(vec![ FileSystemSandboxEntry { path: FileSystemPath::Path { path: link_root }, @@ -983,25 +1347,16 @@ mod tests { }, ]); - let args = create_filesystem_args(&policy, temp_dir.path()).expect("filesystem args"); + let err = + create_filesystem_args(&policy, temp_dir.path(), NO_UNREADABLE_GLOB_SCAN_MAX_DEPTH) + .expect_err("deny-read path crossing writable symlink should fail closed"); + let message = err.to_string(); - assert!(args.args.windows(6).any(|window| { - window - == [ - "--perms", - "000", - "--tmpfs", - outside_str.as_str(), - "--remount-ro", - outside_str.as_str(), - ] - })); assert!( - !args - .args - .iter() - .any(|arg| arg == real_linked_private_str.as_str()) + message.contains("cannot enforce sandbox deny-read path"), + "{message}" ); + assert!(message.contains(&real_linked_private_str), "{message}"); } #[test] @@ -1022,8 +1377,12 @@ mod tests { exclude_slash_tmp: true, }; - let args = create_filesystem_args(&FileSystemSandboxPolicy::from(&policy), temp_dir.path()) - .expect("filesystem args"); + let args = create_filesystem_args( + &FileSystemSandboxPolicy::from(&policy), + temp_dir.path(), + NO_UNREADABLE_GLOB_SCAN_MAX_DEPTH, + ) + .expect("filesystem args"); let existing_root = path_to_string(&existing_root); let missing_root = path_to_string(&missing_root); @@ -1052,6 +1411,7 @@ mod tests { let args = create_filesystem_args( &FileSystemSandboxPolicy::from(&sandbox_policy), Path::new("/"), + NO_UNREADABLE_GLOB_SCAN_MAX_DEPTH, ) .expect("bwrap fs args"); assert_eq!( @@ -1100,8 +1460,12 @@ mod tests { network_access: false, }; - let args = create_filesystem_args(&FileSystemSandboxPolicy::from(&policy), temp_dir.path()) - .expect("filesystem args"); + let args = create_filesystem_args( + &FileSystemSandboxPolicy::from(&policy), + temp_dir.path(), + NO_UNREADABLE_GLOB_SCAN_MAX_DEPTH, + ) + .expect("filesystem args"); assert_eq!(args.args[0..4], ["--tmpfs", "/", "--dev", "/dev"]); @@ -1130,8 +1494,12 @@ mod tests { // `ReadOnlyAccess::Restricted` always includes `cwd` as a readable // root. Using `"/"` here would intentionally collapse to broad read // access, so use a non-root cwd to exercise the restricted path. - let args = create_filesystem_args(&FileSystemSandboxPolicy::from(&policy), temp_dir.path()) - .expect("filesystem args"); + let args = create_filesystem_args( + &FileSystemSandboxPolicy::from(&policy), + temp_dir.path(), + NO_UNREADABLE_GLOB_SCAN_MAX_DEPTH, + ) + .expect("filesystem args"); assert!( args.args @@ -1171,7 +1539,9 @@ mod tests { }, ]); - let args = create_filesystem_args(&policy, temp_dir.path()).expect("filesystem args"); + let args = + create_filesystem_args(&policy, temp_dir.path(), NO_UNREADABLE_GLOB_SCAN_MAX_DEPTH) + .expect("filesystem args"); assert!(args.args.windows(3).any(|window| { window @@ -1248,7 +1618,9 @@ mod tests { }, ]); - let args = create_filesystem_args(&policy, temp_dir.path()).expect("filesystem args"); + let args = + create_filesystem_args(&policy, temp_dir.path(), NO_UNREADABLE_GLOB_SCAN_MAX_DEPTH) + .expect("filesystem args"); let docs_str = path_to_string(docs.as_path()); let docs_public_str = path_to_string(docs_public.as_path()); let docs_ro_index = args @@ -1300,7 +1672,9 @@ mod tests { }, ]); - let args = create_filesystem_args(&policy, temp_dir.path()).expect("filesystem args"); + let args = + create_filesystem_args(&policy, temp_dir.path(), NO_UNREADABLE_GLOB_SCAN_MAX_DEPTH) + .expect("filesystem args"); let blocked_str = path_to_string(blocked.as_path()); let allowed_str = path_to_string(allowed.as_path()); let blocked_none_index = args @@ -1367,7 +1741,9 @@ mod tests { }, ]); - let args = create_filesystem_args(&policy, temp_dir.path()).expect("filesystem args"); + let args = + create_filesystem_args(&policy, temp_dir.path(), NO_UNREADABLE_GLOB_SCAN_MAX_DEPTH) + .expect("filesystem args"); let blocked_str = path_to_string(blocked.as_path()); let allowed_dir_str = path_to_string(allowed_dir.as_path()); let allowed_file_str = path_to_string(allowed_file.as_path()); @@ -1442,7 +1818,9 @@ mod tests { }, ]); - let args = create_filesystem_args(&policy, temp_dir.path()).expect("filesystem args"); + let args = + create_filesystem_args(&policy, temp_dir.path(), NO_UNREADABLE_GLOB_SCAN_MAX_DEPTH) + .expect("filesystem args"); let blocked_none_index = args .args .windows(4) @@ -1487,7 +1865,9 @@ mod tests { }, ]); - let args = create_filesystem_args(&policy, temp_dir.path()).expect("filesystem args"); + let args = + create_filesystem_args(&policy, temp_dir.path(), NO_UNREADABLE_GLOB_SCAN_MAX_DEPTH) + .expect("filesystem args"); let blocked_str = path_to_string(blocked.as_path()); assert!( @@ -1529,7 +1909,9 @@ mod tests { }, ]); - let args = create_filesystem_args(&policy, temp_dir.path()).expect("filesystem args"); + let args = + create_filesystem_args(&policy, temp_dir.path(), NO_UNREADABLE_GLOB_SCAN_MAX_DEPTH) + .expect("filesystem args"); let blocked_file_str = path_to_string(blocked_file.as_path()); assert_eq!(args.preserved_files.len(), 1); @@ -1540,4 +1922,100 @@ mod tests { && window[4] == blocked_file_str })); } + + #[test] + fn unreadable_globs_expand_existing_matches_with_configured_depth() { + if !ripgrep_available() { + return; + } + + let temp_dir = TempDir::new().expect("temp dir"); + let root_env = temp_dir.path().join(".env"); + let nested_env = temp_dir.path().join("app").join(".env"); + let too_deep_env = temp_dir.path().join("app").join("deep").join(".env"); + std::fs::create_dir_all(too_deep_env.parent().expect("parent")).expect("create parent"); + std::fs::write(temp_dir.path().join(".gitignore"), ".env\n").expect("write gitignore"); + std::fs::write(&root_env, "secret").expect("write root env"); + std::fs::write(&nested_env, "secret").expect("write nested env"); + std::fs::write(&too_deep_env, "secret").expect("write deep env"); + let policy = + default_policy_with_unreadable_glob(format!("{}/**/*.env", temp_dir.path().display())); + + let args = + create_filesystem_args(&policy, temp_dir.path(), Some(2)).expect("filesystem args"); + + assert_file_masked(&args.args, &root_env); + assert_file_masked(&args.args, &nested_env); + assert!( + !args + .args + .iter() + .any(|arg| arg == &path_to_string(&too_deep_env)), + "max depth should keep deeper matches out of bwrap args: {:#?}", + args.args + ); + } + + #[cfg(unix)] + #[test] + fn unreadable_globs_add_canonical_targets_for_symlink_matches() { + if !ripgrep_available() { + return; + } + + let temp_dir = TempDir::new().expect("temp dir"); + let real_root = temp_dir.path().join("real"); + let link_root = temp_dir.path().join("link"); + let real_secret = real_root.join("secret.env"); + std::fs::create_dir_all(&real_root).expect("create real root"); + std::fs::write(&real_secret, "secret").expect("write real secret"); + std::os::unix::fs::symlink(&real_root, &link_root).expect("create symlink"); + let policy = + default_policy_with_unreadable_glob(format!("{}/**/*.env", link_root.display())); + + let args = + create_filesystem_args(&policy, temp_dir.path(), Some(2)).expect("filesystem args"); + + assert_file_masked(&args.args, &real_secret); + } + + #[test] + fn root_prefix_unreadable_globs_are_too_broad_for_linux_expansion() { + assert_eq!( + split_pattern_for_ripgrep("/**/*.env", Path::new("/tmp")), + None + ); + } + + #[test] + fn unclosed_character_classes_are_escaped_for_ripgrep() { + let (search_root, glob) = + split_pattern_for_ripgrep("/tmp/[*.env", Path::new("/")).expect("split pattern"); + + assert_eq!(search_root.as_path(), Path::new("/tmp")); + assert_eq!(glob, r"\[*.env"); + } + + fn ripgrep_available() -> bool { + Command::new("rg") + .arg("--version") + .output() + .is_ok_and(|output| output.status.success()) + } + + /// Assert that `path` is masked due to a bwrap arg sequence like: + /// + /// `bwrap ... --perms 000 --ro-bind-data FD PATH` + fn assert_file_masked(args: &[String], path: &Path) { + let path = path_to_string(path); + assert!( + args.windows(5).any(|window| { + window[0] == "--perms" + && window[1] == "000" + && window[2] == "--ro-bind-data" + && window[4] == path + }), + "expected file mask for {path}: {args:#?}" + ); + } } diff --git a/codex-rs/linux-sandbox/src/linux_run_main.rs b/codex-rs/linux-sandbox/src/linux_run_main.rs index 8f39837de1..958d8645bd 100644 --- a/codex-rs/linux-sandbox/src/linux_run_main.rs +++ b/codex-rs/linux-sandbox/src/linux_run_main.rs @@ -427,6 +427,7 @@ fn run_bwrap_with_proc_fallback( let options = BwrapOptions { mount_proc, network_mode, + ..Default::default() }; let mut bwrap_args = build_bwrap_argv( inner, @@ -547,6 +548,7 @@ fn build_preflight_bwrap_argv( BwrapOptions { mount_proc: true, network_mode, + ..Default::default() }, ) } diff --git a/codex-rs/linux-sandbox/src/linux_run_main_tests.rs b/codex-rs/linux-sandbox/src/linux_run_main_tests.rs index 96c3c8a737..0ed10717f2 100644 --- a/codex-rs/linux-sandbox/src/linux_run_main_tests.rs +++ b/codex-rs/linux-sandbox/src/linux_run_main_tests.rs @@ -48,6 +48,7 @@ fn inserts_bwrap_argv0_before_command_separator() { BwrapOptions { mount_proc: true, network_mode: BwrapNetworkMode::FullAccess, + ..Default::default() }, ) .args; @@ -90,6 +91,7 @@ fn rewrites_inner_command_path_when_bwrap_lacks_argv0() { BwrapOptions { mount_proc: true, network_mode: BwrapNetworkMode::FullAccess, + ..Default::default() }, ) .args; @@ -157,6 +159,7 @@ fn inserts_unshare_net_when_network_isolation_requested() { BwrapOptions { mount_proc: true, network_mode: BwrapNetworkMode::Isolated, + ..Default::default() }, ) .args; @@ -174,6 +177,7 @@ fn inserts_unshare_net_when_proxy_only_network_mode_requested() { BwrapOptions { mount_proc: true, network_mode: BwrapNetworkMode::ProxyOnly, + ..Default::default() }, ) .args; diff --git a/codex-rs/protocol/src/permissions.rs b/codex-rs/protocol/src/permissions.rs index d774f6dc0d..80cfa823e7 100644 --- a/codex-rs/protocol/src/permissions.rs +++ b/codex-rs/protocol/src/permissions.rs @@ -138,6 +138,9 @@ pub enum FileSystemSandboxKind { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema, TS)] pub struct FileSystemSandboxPolicy { pub kind: FileSystemSandboxKind, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub glob_scan_max_depth: Option, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub entries: Vec, } @@ -257,6 +260,7 @@ impl Default for FileSystemSandboxPolicy { fn default() -> Self { Self { kind: FileSystemSandboxKind::Restricted, + glob_scan_max_depth: None, entries: vec![FileSystemSandboxEntry { path: FileSystemPath::Special { value: FileSystemSpecialPath::Root, @@ -271,6 +275,7 @@ impl FileSystemSandboxPolicy { pub fn unrestricted() -> Self { Self { kind: FileSystemSandboxKind::Unrestricted, + glob_scan_max_depth: None, entries: Vec::new(), } } @@ -278,6 +283,7 @@ impl FileSystemSandboxPolicy { pub fn external_sandbox() -> Self { Self { kind: FileSystemSandboxKind::ExternalSandbox, + glob_scan_max_depth: None, entries: Vec::new(), } } @@ -285,6 +291,7 @@ impl FileSystemSandboxPolicy { pub fn restricted(entries: Vec) -> Self { Self { kind: FileSystemSandboxKind::Restricted, + glob_scan_max_depth: None, entries, } } @@ -317,6 +324,7 @@ impl FileSystemSandboxPolicy { if !matches!(rebuilt.kind, FileSystemSandboxKind::Restricted) { return rebuilt; } + rebuilt.glob_scan_max_depth = existing.glob_scan_max_depth; for deny_entry in existing .entries diff --git a/codex-rs/sandboxing/Cargo.toml b/codex-rs/sandboxing/Cargo.toml index 18c5ffb45d..49fd33e01d 100644 --- a/codex-rs/sandboxing/Cargo.toml +++ b/codex-rs/sandboxing/Cargo.toml @@ -18,6 +18,7 @@ codex-utils-absolute-path = { workspace = true } dunce = { workspace = true } libc = { workspace = true } serde_json = { workspace = true } +regex-lite = { workspace = true } tracing = { workspace = true, features = ["log"] } url = { workspace = true } which = { workspace = true } diff --git a/codex-rs/sandboxing/src/lib.rs b/codex-rs/sandboxing/src/lib.rs index dbb8323649..38aed6ce24 100644 --- a/codex-rs/sandboxing/src/lib.rs +++ b/codex-rs/sandboxing/src/lib.rs @@ -34,9 +34,6 @@ impl From for CodexErr { SandboxTransformError::MissingLinuxSandboxExecutable => { CodexErr::LandlockSandboxExecutableNotProvided } - SandboxTransformError::UnreadableGlobPatternsUnsupported => { - CodexErr::UnsupportedOperation(err.to_string()) - } #[cfg(target_os = "linux")] SandboxTransformError::Wsl1UnsupportedForBubblewrap => { CodexErr::UnsupportedOperation(crate::bwrap::WSL1_BWRAP_WARNING.to_string()) diff --git a/codex-rs/sandboxing/src/manager.rs b/codex-rs/sandboxing/src/manager.rs index 1e3ea9c196..0836eb2565 100644 --- a/codex-rs/sandboxing/src/manager.rs +++ b/codex-rs/sandboxing/src/manager.rs @@ -109,7 +109,6 @@ pub struct SandboxTransformRequest<'a> { #[derive(Debug)] pub enum SandboxTransformError { MissingLinuxSandboxExecutable, - UnreadableGlobPatternsUnsupported, #[cfg(target_os = "linux")] Wsl1UnsupportedForBubblewrap, #[cfg(not(target_os = "macos"))] @@ -122,10 +121,6 @@ impl std::fmt::Display for SandboxTransformError { Self::MissingLinuxSandboxExecutable => { write!(f, "missing codex-linux-sandbox executable path") } - Self::UnreadableGlobPatternsUnsupported => write!( - f, - "platform sandbox backend cannot enforce unreadable glob patterns" - ), #[cfg(target_os = "linux")] Self::Wsl1UnsupportedForBubblewrap => write!(f, "{WSL1_BWRAP_WARNING}"), #[cfg(not(target_os = "macos"))] @@ -201,15 +196,6 @@ impl SandboxManager { ); let effective_network_policy = effective_network_sandbox_policy(network_policy, additional_permissions.as_ref()); - if matches!( - sandbox, - SandboxType::MacosSeatbelt | SandboxType::LinuxSeccomp - ) && !effective_file_system_policy - .get_unreadable_globs_with_cwd(sandbox_policy_cwd) - .is_empty() - { - return Err(SandboxTransformError::UnreadableGlobPatternsUnsupported); - } let mut argv = Vec::with_capacity(1 + command.args.len()); argv.push(command.program); argv.extend(command.args.into_iter().map(OsString::from)); diff --git a/codex-rs/sandboxing/src/seatbelt.rs b/codex-rs/sandboxing/src/seatbelt.rs index b12492210c..57a152e023 100644 --- a/codex-rs/sandboxing/src/seatbelt.rs +++ b/codex-rs/sandboxing/src/seatbelt.rs @@ -9,6 +9,7 @@ use codex_utils_absolute_path::AbsolutePathBuf; use std::collections::BTreeMap; use std::collections::BTreeSet; use std::collections::HashMap; +use std::collections::VecDeque; use std::ffi::CStr; use std::path::Path; use std::path::PathBuf; @@ -379,6 +380,150 @@ fn build_seatbelt_access_policy( } } +fn build_seatbelt_unreadable_glob_policy( + file_system_sandbox_policy: &FileSystemSandboxPolicy, + cwd: &Path, +) -> String { + // Seatbelt does not understand the filesystem policy's glob syntax directly. + // Convert each unreadable pattern into an anchored regex deny rule and apply + // it to both reads and unlink-style writes so a denied path cannot be probed + // through destructive filesystem operations. + let unreadable_globs = file_system_sandbox_policy.get_unreadable_globs_with_cwd(cwd); + if unreadable_globs.is_empty() { + return String::new(); + } + + let mut policy_components = Vec::new(); + for pattern in unreadable_globs { + let mut regexes = BTreeSet::new(); + if let Some(regex) = seatbelt_regex_for_unreadable_glob(&pattern) { + regexes.insert(regex); + } + if let Some(pattern) = canonicalize_glob_static_prefix_for_sandbox(&pattern) + && let Some(regex) = seatbelt_regex_for_unreadable_glob(&pattern) + { + regexes.insert(regex); + } + for regex in regexes { + let regex = regex.replace('"', "\\\""); + policy_components.push(format!(r#"(deny file-read* (regex #"{regex}"))"#)); + policy_components.push(format!(r#"(deny file-write-unlink (regex #"{regex}"))"#)); + } + } + + policy_components.join("\n") +} + +fn canonicalize_glob_static_prefix_for_sandbox(pattern: &str) -> Option { + let first_glob_index = pattern + .char_indices() + .find_map(|(index, ch)| matches!(ch, '*' | '?' | '[' | ']').then_some(index)); + let Some(first_glob_index) = first_glob_index else { + return normalize_path_for_sandbox(Path::new(pattern)) + .map(|path| path.to_string_lossy().to_string()); + }; + + let static_prefix = &pattern[..first_glob_index]; + let prefix_end = if static_prefix.ends_with('/') { + static_prefix.len() - 1 + } else { + static_prefix.rfind('/').unwrap_or(0) + }; + if prefix_end == 0 { + return None; + } + + let root = normalize_path_for_sandbox(Path::new(&pattern[..prefix_end]))?; + let root = root.to_string_lossy(); + let suffix = &pattern[prefix_end..]; + let normalized_pattern = format!("{root}{suffix}"); + (normalized_pattern != pattern).then_some(normalized_pattern) +} + +fn seatbelt_regex_for_unreadable_glob(pattern: &str) -> Option { + if pattern.is_empty() { + return None; + } + + // Translate the supported git-style glob subset into a Seatbelt regex: + // `*` and `?` stay within one path component, `**/` can consume zero or + // more components, and closed character classes remain character classes. + // A pattern with no glob metacharacters is treated as exact path plus subtree. + let mut regex = String::from("^"); + let mut chars = pattern.chars().collect::>(); + let mut saw_glob = false; + + while let Some(ch) = chars.pop_front() { + match ch { + '*' => { + saw_glob = true; + if chars.front() == Some(&'*') { + chars.pop_front(); + if chars.front() == Some(&'/') { + chars.pop_front(); + regex.push_str("(.*/)?"); + } else { + regex.push_str(".*"); + } + } else { + regex.push_str("[^/]*"); + } + } + '?' => { + saw_glob = true; + regex.push_str("[^/]"); + } + '[' => { + saw_glob = true; + let mut class = Vec::new(); + let mut closed = false; + while let Some(class_ch) = chars.pop_front() { + if class_ch == ']' { + closed = true; + break; + } + class.push(class_ch); + } + if !closed { + regex.push_str("\\["); + for class_ch in class.into_iter().rev() { + chars.push_front(class_ch); + } + continue; + } + + regex.push('['); + let mut class_chars = class.into_iter(); + if let Some(first) = class_chars.next() { + match first { + '!' => regex.push('^'), + '^' => regex.push_str("\\^"), + _ => regex.push(first), + } + } + for class_ch in class_chars { + match class_ch { + '\\' => regex.push_str("\\\\"), + _ => regex.push(class_ch), + } + } + regex.push(']'); + } + ']' => { + saw_glob = true; + regex.push_str("\\]"); + } + _ => regex.push_str(®ex_lite::escape(&ch.to_string())), + } + } + + if !saw_glob { + regex.push_str("(/.*)?"); + } + regex.push('$'); + Some(regex) +} + #[cfg_attr(not(test), allow(dead_code))] fn create_seatbelt_command_args_for_legacy_policy( command: Vec, @@ -510,10 +655,13 @@ pub fn create_seatbelt_command_args(args: CreateSeatbeltCommandArgsParams<'_>) - dynamic_network_policy_for_network(network_sandbox_policy, enforce_managed_network, &proxy); let include_platform_defaults = file_system_sandbox_policy.include_platform_defaults(); + let deny_read_policy = + build_seatbelt_unreadable_glob_policy(file_system_sandbox_policy, sandbox_policy_cwd); let mut policy_sections = vec![ MACOS_SEATBELT_BASE_POLICY.to_string(), file_read_policy, file_write_policy, + deny_read_policy, network_policy, ]; if include_platform_defaults { diff --git a/codex-rs/sandboxing/src/seatbelt_tests.rs b/codex-rs/sandboxing/src/seatbelt_tests.rs index f222dd15d1..9d958c9564 100644 --- a/codex-rs/sandboxing/src/seatbelt_tests.rs +++ b/codex-rs/sandboxing/src/seatbelt_tests.rs @@ -3,11 +3,13 @@ use super::MACOS_PATH_TO_SEATBELT_EXECUTABLE; use super::MACOS_SEATBELT_BASE_POLICY; use super::ProxyPolicyInputs; use super::UnixDomainSocketPolicy; +use super::build_seatbelt_unreadable_glob_policy; use super::create_seatbelt_command_args; use super::create_seatbelt_command_args_for_legacy_policy; use super::dynamic_network_policy; use super::macos_dir_params; use super::normalize_path_for_sandbox; +use super::seatbelt_regex_for_unreadable_glob; use super::unix_socket_dir_params; use super::unix_socket_policy; use codex_network_proxy::ConfigReloader; @@ -257,6 +259,82 @@ fn explicit_unreadable_paths_are_excluded_from_readable_roots() { ); } +#[test] +fn unreadable_globstar_slash_matches_zero_or_more_directories() { + let regex = seatbelt_regex_for_unreadable_glob("/tmp/repo/**/*.env"); + assert_eq!(regex.as_deref(), Some(r"^/tmp/repo/(.*/)?[^/]*\.env$")); + let regex = regex_lite::Regex::new(regex.as_deref().expect("glob should compile")) + .expect("regex should compile"); + + assert!(regex.is_match("/tmp/repo/.env")); + assert!(regex.is_match("/tmp/repo/app/.env")); + assert!(regex.is_match("/tmp/repo/app/config.env")); + assert!(!regex.is_match("/tmp/repo/app/config.toml")); +} + +#[test] +fn unreadable_globs_use_git_style_component_matching() { + let regex = seatbelt_regex_for_unreadable_glob("/tmp/repo/*/file[0-9]?.txt"); + assert_eq!( + regex.as_deref(), + Some(r"^/tmp/repo/[^/]*/file[0-9][^/]\.txt$") + ); + let regex = regex_lite::Regex::new(regex.as_deref().expect("glob should compile")) + .expect("regex should compile"); + + assert!(regex.is_match("/tmp/repo/app/file42.txt")); + assert!(!regex.is_match("/tmp/repo/app/nested/file42.txt")); + assert!(!regex.is_match("/tmp/repo/app/file4.txt")); + assert!(!regex.is_match("/tmp/repo/app/fileab.txt")); +} + +#[test] +fn unreadable_globs_treat_unclosed_character_classes_as_literals() { + let regex = seatbelt_regex_for_unreadable_glob("/tmp/repo/[*.env"); + assert_eq!(regex.as_deref(), Some(r"^/tmp/repo/\[[^/]*\.env$")); + let regex = regex_lite::Regex::new(regex.as_deref().expect("glob should compile")) + .expect("regex should compile"); + + assert!(regex.is_match("/tmp/repo/[local.env")); + assert!(regex.is_match("/tmp/repo/[.env")); + assert!(!regex.is_match("/tmp/repo/local.env")); +} + +#[cfg(unix)] +#[test] +fn unreadable_glob_policy_includes_canonicalized_static_prefix() { + use std::os::unix::fs::symlink; + + let temp_dir = TempDir::new().expect("temp dir"); + let real_root = temp_dir.path().join("real-root"); + let link_root = temp_dir.path().join("link-root"); + fs::create_dir(&real_root).expect("create real root"); + symlink(&real_root, &link_root).expect("create symlinked root"); + + let pattern = format!("{}/**/*.env", link_root.display()); + let canonical_pattern = format!( + "{}/**/*.env", + real_root + .canonicalize() + .expect("canonicalize real root") + .display() + ); + let expected_regex = seatbelt_regex_for_unreadable_glob(&canonical_pattern) + .expect("canonical glob should compile"); + let mut policy = FileSystemSandboxPolicy::default(); + policy.entries.push(FileSystemSandboxEntry { + path: FileSystemPath::GlobPattern { pattern }, + access: FileSystemAccessMode::None, + }); + + let seatbelt_policy = build_seatbelt_unreadable_glob_policy(&policy, temp_dir.path()); + + assert!( + seatbelt_policy.contains(&format!(r#"(deny file-read* (regex #"{expected_regex}"))"#)), + "expected canonicalized glob regex in policy:\n{seatbelt_policy}" + ); +} + #[test] fn seatbelt_args_without_extension_profile_keep_legacy_preferences_read_access() { let cwd = std::env::temp_dir(); @@ -1179,6 +1257,7 @@ fn create_seatbelt_args_for_cwd_as_git_repo() { (allow file-write* (require-all (subpath (param "WRITABLE_ROOT_0")) (require-not (literal (param "WRITABLE_ROOT_0_EXCLUDED_0"))) (require-not (subpath (param "WRITABLE_ROOT_0_EXCLUDED_0"))) (require-not (literal (param "WRITABLE_ROOT_0_EXCLUDED_1"))) (require-not (subpath (param "WRITABLE_ROOT_0_EXCLUDED_1"))) ) (subpath (param "WRITABLE_ROOT_1")){tempdir_policy_entry} ) + "#, ); From 2967900d810e3f304999e6c51f5a76088759eee1 Mon Sep 17 00:00:00 2001 From: viyatb-oai Date: Thu, 16 Apr 2026 17:37:15 -0700 Subject: [PATCH 50/59] fix: deprecate use_legacy_landlock feature flag (#17971) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - mark `features.use_legacy_landlock` as a deprecated feature flag - emit a startup deprecation notice when the flag is configured - add feature- and core-level regression coverage for the notice Screenshot 2026-04-15 at 11 14 00 PM --------- Co-authored-by: Codex --- .../core/tests/suite/deprecation_notice.rs | 42 +++++++++++++++++++ codex-rs/features/src/lib.rs | 21 +++++++++- codex-rs/features/src/tests.rs | 26 +++++++++++- 3 files changed, 86 insertions(+), 3 deletions(-) diff --git a/codex-rs/core/tests/suite/deprecation_notice.rs b/codex-rs/core/tests/suite/deprecation_notice.rs index 26a56c86e2..dc7280ea32 100644 --- a/codex-rs/core/tests/suite/deprecation_notice.rs +++ b/codex-rs/core/tests/suite/deprecation_notice.rs @@ -161,3 +161,45 @@ async fn emits_deprecation_notice_for_web_search_feature_flag_values() -> anyhow Ok(()) } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn emits_deprecation_notice_for_use_legacy_landlock() -> anyhow::Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + + let mut builder = test_codex().with_config(|config| { + let mut entries = BTreeMap::new(); + entries.insert("use_legacy_landlock".to_string(), true); + let mut features = config.features.get().clone(); + features.apply_map(&entries); + config + .features + .set(features) + .expect("test config should allow managed feature map updates"); + }); + + let TestCodex { codex, .. } = builder.build(&server).await?; + + let notice = wait_for_event_match(&codex, |event| match event { + EventMsg::DeprecationNotice(ev) + if ev.summary.contains("[features].use_legacy_landlock") => + { + Some(ev.clone()) + } + _ => None, + }) + .await; + + let DeprecationNoticeEvent { summary, details } = notice; + assert_eq!( + summary, + "`[features].use_legacy_landlock` is deprecated and will be removed soon.".to_string(), + ); + assert_eq!( + details.as_deref(), + Some("Remove this setting to stop opting into the legacy Linux sandbox behavior."), + ); + + Ok(()) +} diff --git a/codex-rs/features/src/lib.rs b/codex-rs/features/src/lib.rs index a2e656be44..586c366850 100644 --- a/codex-rs/features/src/lib.rs +++ b/codex-rs/features/src/lib.rs @@ -371,6 +371,12 @@ impl Features { "image_detail_original" => { continue; } + "use_legacy_landlock" => { + self.record_legacy_usage_force( + "features.use_legacy_landlock", + Feature::UseLegacyLandlock, + ); + } _ => {} } match feature_for_key(k) { @@ -456,6 +462,19 @@ fn legacy_usage_notice(alias: &str, feature: Feature) -> (String, Option format!("`{label}` is deprecated because web search is enabled by default."); (summary, Some(web_search_details().to_string())) } + Feature::UseLegacyLandlock => { + let label = match alias { + "features.use_legacy_landlock" | "use_legacy_landlock" => { + "[features].use_legacy_landlock" + } + _ => alias, + }; + let summary = format!("`{label}` is deprecated and will be removed soon."); + let details = + "Remove this setting to stop opting into the legacy Linux sandbox behavior." + .to_string(); + (summary, Some(details)) + } _ => { let label = if alias.contains('.') || alias.starts_with('[') { alias.to_string() @@ -728,7 +747,7 @@ pub const FEATURES: &[FeatureSpec] = &[ FeatureSpec { id: Feature::UseLegacyLandlock, key: "use_legacy_landlock", - stage: Stage::Stable, + stage: Stage::Deprecated, default_enabled: false, }, FeatureSpec { diff --git a/codex-rs/features/src/tests.rs b/codex-rs/features/src/tests.rs index d2f73cedad..fc81c73085 100644 --- a/codex-rs/features/src/tests.rs +++ b/codex-rs/features/src/tests.rs @@ -42,8 +42,8 @@ fn default_enabled_features_are_stable() { } #[test] -fn use_legacy_landlock_is_stable_and_disabled_by_default() { - assert_eq!(Feature::UseLegacyLandlock.stage(), Stage::Stable); +fn use_legacy_landlock_is_deprecated_and_disabled_by_default() { + assert_eq!(Feature::UseLegacyLandlock.stage(), Stage::Deprecated); assert_eq!(Feature::UseLegacyLandlock.default_enabled(), false); } @@ -166,6 +166,28 @@ fn image_generation_is_stable_and_enabled_by_default() { assert_eq!(Feature::ImageGeneration.default_enabled(), true); } +#[test] +fn use_legacy_landlock_config_records_deprecation_notice() { + let mut entries = BTreeMap::new(); + entries.insert("use_legacy_landlock".to_string(), true); + + let mut features = Features::with_defaults(); + features.apply_map(&entries); + + let usages = features.legacy_feature_usages().collect::>(); + assert_eq!(usages.len(), 1); + assert_eq!(usages[0].alias, "features.use_legacy_landlock"); + assert_eq!(usages[0].feature, Feature::UseLegacyLandlock); + assert_eq!( + usages[0].summary, + "`[features].use_legacy_landlock` is deprecated and will be removed soon." + ); + assert_eq!( + usages[0].details.as_deref(), + Some("Remove this setting to stop opting into the legacy Linux sandbox behavior.") + ); +} + #[test] fn image_detail_original_is_a_removed_feature_key() { assert_eq!( From 9effa0509f90a3f661dfdba9bd92cec571e9e5d9 Mon Sep 17 00:00:00 2001 From: pakrym-oai Date: Thu, 16 Apr 2026 17:51:21 -0700 Subject: [PATCH 51/59] Refactor config loading to use filesystem abstraction (#18209) Initial pass propagating FileSystem through config loading. --- codex-rs/Cargo.lock | 7 +- codex-rs/app-server-protocol/Cargo.toml | 1 - codex-rs/app-server-protocol/src/lib.rs | 2 +- .../app-server-protocol/src/protocol/v1.rs | 2 +- .../app-server/src/codex_message_processor.rs | 5 +- .../app-server/tests/suite/v2/thread_start.rs | 14 +- codex-rs/config/Cargo.toml | 3 +- codex-rs/config/src/config_toml.rs | 23 +-- codex-rs/core/src/agent/role.rs | 2 + codex-rs/core/src/codex/handlers.rs | 2 + codex-rs/core/src/config/agent_roles.rs | 166 ++++++++++-------- codex-rs/core/src/config/config_tests.rs | 35 ++-- codex-rs/core/src/config/mod.rs | 60 +++++-- codex-rs/core/src/config/service.rs | 2 + codex-rs/core/src/config_loader/README.md | 10 +- codex-rs/core/src/config_loader/layer_io.rs | 20 ++- codex-rs/core/src/config_loader/mod.rs | 88 ++++++---- codex-rs/core/src/config_loader/tests.rs | 35 +++- codex-rs/core/src/git_info_tests.rs | 67 ++++--- codex-rs/core/src/guardian/tests.rs | 3 + codex-rs/core/src/network_proxy_loader.rs | 2 + codex-rs/core/src/realtime_context.rs | 60 ++++--- codex-rs/core/src/realtime_context_tests.rs | 17 +- codex-rs/git-utils/Cargo.toml | 2 + codex-rs/git-utils/src/info.rs | 46 +++-- codex-rs/git-utils/src/lib.rs | 74 +------- codex-rs/protocol/Cargo.toml | 1 - codex-rs/protocol/src/models.rs | 57 +++++- codex-rs/protocol/src/protocol.rs | 12 +- .../tui/src/onboarding/onboarding_screen.rs | 4 +- 30 files changed, 507 insertions(+), 315 deletions(-) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 3f27fe10e5..76188da5b8 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1540,7 +1540,6 @@ dependencies = [ "anyhow", "clap", "codex-experimental-api-macros", - "codex-git-utils", "codex-protocol", "codex-shell-command", "codex-utils-absolute-path", @@ -1866,12 +1865,11 @@ dependencies = [ "codex-app-server-protocol", "codex-execpolicy", "codex-features", - "codex-git-utils", "codex-model-provider-info", "codex-network-proxy", "codex-protocol", "codex-utils-absolute-path", - "dunce", + "codex-utils-path", "futures", "multimap", "pretty_assertions", @@ -2264,6 +2262,8 @@ name = "codex-git-utils" version = "0.0.0" dependencies = [ "assert_matches", + "codex-exec-server", + "codex-protocol", "codex-utils-absolute-path", "futures", "once_cell", @@ -2621,7 +2621,6 @@ dependencies = [ "chrono", "codex-async-utils", "codex-execpolicy", - "codex-git-utils", "codex-network-proxy", "codex-utils-absolute-path", "codex-utils-image", diff --git a/codex-rs/app-server-protocol/Cargo.toml b/codex-rs/app-server-protocol/Cargo.toml index d9ed5e8730..0cb50d8549 100644 --- a/codex-rs/app-server-protocol/Cargo.toml +++ b/codex-rs/app-server-protocol/Cargo.toml @@ -15,7 +15,6 @@ workspace = true anyhow = { workspace = true } clap = { workspace = true, features = ["derive"] } codex-experimental-api-macros = { workspace = true } -codex-git-utils = { workspace = true } codex-protocol = { workspace = true } codex-shell-command = { workspace = true } codex-utils-absolute-path = { workspace = true } diff --git a/codex-rs/app-server-protocol/src/lib.rs b/codex-rs/app-server-protocol/src/lib.rs index d5c2f4b243..46f0c9ae41 100644 --- a/codex-rs/app-server-protocol/src/lib.rs +++ b/codex-rs/app-server-protocol/src/lib.rs @@ -4,7 +4,6 @@ mod jsonrpc_lite; mod protocol; mod schema_fixtures; -pub use codex_git_utils::GitSha; pub use experimental_api::*; pub use export::GenerateTsOptions; pub use export::generate_internal_json_schema; @@ -30,6 +29,7 @@ pub use protocol::v1::GetConversationSummaryParams; pub use protocol::v1::GetConversationSummaryResponse; pub use protocol::v1::GitDiffToRemoteParams; pub use protocol::v1::GitDiffToRemoteResponse; +pub use protocol::v1::GitSha; pub use protocol::v1::InitializeCapabilities; pub use protocol::v1::InitializeParams; pub use protocol::v1::InitializeResponse; diff --git a/codex-rs/app-server-protocol/src/protocol/v1.rs b/codex-rs/app-server-protocol/src/protocol/v1.rs index 6aa2e9fa30..d642e7fab9 100644 --- a/codex-rs/app-server-protocol/src/protocol/v1.rs +++ b/codex-rs/app-server-protocol/src/protocol/v1.rs @@ -1,7 +1,6 @@ use std::collections::HashMap; use std::path::PathBuf; -use codex_git_utils::GitSha; use codex_protocol::ThreadId; use codex_protocol::config_types::ForcedLoginMethod; use codex_protocol::config_types::ReasoningSummary; @@ -11,6 +10,7 @@ use codex_protocol::openai_models::ReasoningEffort; use codex_protocol::parse_command::ParsedCommand; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::FileChange; +pub use codex_protocol::protocol::GitSha; use codex_protocol::protocol::ReviewDecision; use codex_protocol::protocol::SandboxPolicy; use codex_protocol::protocol::SessionSource; diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index 0f3b8bf522..6a9dfb4042 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -2441,9 +2441,9 @@ impl CodexMessageProcessor { | codex_protocol::protocol::SandboxPolicy::ExternalSandbox { .. } )) { - let trust_target = resolve_root_git_project_for_trust(config.cwd.as_path()) + let trust_target = resolve_root_git_project_for_trust(LOCAL_FS.as_ref(), &config.cwd) .await - .unwrap_or_else(|| config.cwd.to_path_buf()); + .unwrap_or_else(|| config.cwd.clone()); let cli_overrides_with_trust; let cli_overrides_for_reload = if let Err(err) = codex_core::config::set_project_trust_level( @@ -6155,6 +6155,7 @@ impl CodexMessageProcessor { } }; let config_layer_stack = match load_config_layers_state( + LOCAL_FS.as_ref(), &self.config.codex_home, Some(cwd_abs.clone()), &cli_overrides, diff --git a/codex-rs/app-server/tests/suite/v2/thread_start.rs b/codex-rs/app-server/tests/suite/v2/thread_start.rs index ab514252e4..508c77f7ff 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_start.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_start.rs @@ -1,6 +1,7 @@ use anyhow::Result; use app_test_support::ChatGptAuthFixture; use app_test_support::McpProcess; +use app_test_support::PathBufExt; use app_test_support::create_mock_responses_server_repeating_assistant; use app_test_support::to_response; use app_test_support::write_chatgpt_auth; @@ -20,6 +21,7 @@ use codex_app_server_protocol::ThreadStatus; use codex_app_server_protocol::ThreadStatusChangedNotification; use codex_config::types::AuthCredentialsStoreMode; use codex_core::config::set_project_trust_level; +use codex_exec_server::LOCAL_FS; use codex_git_utils::resolve_root_git_project_for_trust; use codex_login::REFRESH_TOKEN_URL_OVERRIDE_ENV_VAR; use codex_protocol::config_types::ServiceTier; @@ -716,10 +718,11 @@ model_reasoning_effort = "high" assert_eq!(reasoning_effort, Some(ReasoningEffort::High)); let config_toml = std::fs::read_to_string(codex_home.path().join("config.toml"))?; - let trusted_root = resolve_root_git_project_for_trust(workspace.path()) + let workspace_abs = workspace.path().to_path_buf().abs(); + let trusted_root = resolve_root_git_project_for_trust(LOCAL_FS.as_ref(), &workspace_abs) .await - .unwrap_or_else(|| workspace.path().to_path_buf()); - assert!(config_toml.contains(&persisted_trust_path(&trusted_root))); + .unwrap_or(workspace_abs); + assert!(config_toml.contains(&persisted_trust_path(trusted_root.as_path()))); assert!(config_toml.contains("trust_level = \"trusted\"")); Ok(()) @@ -754,10 +757,11 @@ async fn thread_start_with_nested_git_cwd_trusts_repo_root() -> Result<()> { .await??; let config_toml = std::fs::read_to_string(codex_home.path().join("config.toml"))?; - let trusted_root = resolve_root_git_project_for_trust(&nested) + let nested_abs = nested.abs(); + let trusted_root = resolve_root_git_project_for_trust(LOCAL_FS.as_ref(), &nested_abs) .await .expect("git root should resolve"); - assert!(config_toml.contains(&persisted_trust_path(&trusted_root))); + assert!(config_toml.contains(&persisted_trust_path(trusted_root.as_path()))); assert!(!config_toml.contains(&persisted_trust_path(&nested))); Ok(()) diff --git a/codex-rs/config/Cargo.toml b/codex-rs/config/Cargo.toml index 9532d74d00..93ca283b86 100644 --- a/codex-rs/config/Cargo.toml +++ b/codex-rs/config/Cargo.toml @@ -12,12 +12,11 @@ anyhow = { workspace = true } codex-app-server-protocol = { workspace = true } codex-execpolicy = { workspace = true } codex-features = { workspace = true } -codex-git-utils = { workspace = true } codex-model-provider-info = { workspace = true } codex-network-proxy = { workspace = true } codex-protocol = { workspace = true } codex-utils-absolute-path = { workspace = true } -dunce = { workspace = true } +codex-utils-path = { workspace = true } futures = { workspace = true, features = ["alloc", "std"] } multimap = { workspace = true } schemars = { workspace = true } diff --git a/codex-rs/config/src/config_toml.rs b/codex-rs/config/src/config_toml.rs index 7de4e253e5..83fe6c88e7 100644 --- a/codex-rs/config/src/config_toml.rs +++ b/codex-rs/config/src/config_toml.rs @@ -29,7 +29,6 @@ use crate::types::WindowsToml; use codex_app_server_protocol::Tools; use codex_app_server_protocol::UserSavedConfig; use codex_features::FeaturesToml; -use codex_git_utils::resolve_root_git_project_for_trust; use codex_model_provider_info::LEGACY_OLLAMA_CHAT_PROVIDER_ID; use codex_model_provider_info::LMSTUDIO_OSS_PROVIDER_ID; use codex_model_provider_info::ModelProviderInfo; @@ -51,6 +50,7 @@ use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::ReadOnlyAccess; use codex_protocol::protocol::SandboxPolicy; use codex_utils_absolute_path::AbsolutePathBuf; +use codex_utils_path::normalize_for_path_comparison; use schemars::JsonSchema; use serde::Deserialize; use serde::Deserializer; @@ -601,7 +601,7 @@ impl ConfigToml { sandbox_mode_override: Option, profile_sandbox_mode: Option, windows_sandbox_level: WindowsSandboxLevel, - resolved_cwd: &Path, + active_project: Option<&ProjectConfig>, sandbox_policy_constraint: Option<&crate::Constrained>, ) -> SandboxPolicy { let sandbox_mode_was_explicit = sandbox_mode_override.is_some() @@ -616,7 +616,7 @@ impl ConfigToml { // If no sandbox_mode is set but this directory has a trust decision, // default to workspace-write except on unsandboxed Windows where we // default to read-only. - self.get_active_project(resolved_cwd).await.and_then(|p| { + active_project.and_then(|p| { if p.is_trusted() || p.is_untrusted() { if cfg!(target_os = "windows") && windows_sandbox_level == WindowsSandboxLevel::Disabled @@ -677,9 +677,13 @@ impl ConfigToml { } /// Resolves the cwd to an existing project, or returns None if ConfigToml - /// does not contain a project corresponding to cwd or a git repo for cwd - pub async fn get_active_project(&self, resolved_cwd: &Path) -> Option { - let repo_root = resolve_root_git_project_for_trust(resolved_cwd).await; + /// does not contain a project corresponding to cwd or the resolved git repo + /// root for cwd. + pub fn get_active_project( + &self, + resolved_cwd: &Path, + repo_root: Option<&Path>, + ) -> Option { let projects = self.projects.clone().unwrap_or_default(); let resolved_cwd_key = project_trust_key(resolved_cwd); @@ -691,10 +695,7 @@ impl ConfigToml { return Some(project_config.clone()); } - // If cwd lives inside a git repo/worktree, check whether the root git project - // (the primary repository working directory) is trusted. This lets - // worktrees inherit trust from the main project. - if let Some(repo_root) = repo_root.as_deref() { + if let Some(repo_root) = repo_root { let repo_root_key = project_trust_key(repo_root); let repo_root_raw_key = repo_root.to_string_lossy().to_string(); if let Some(project_config_for_root) = projects @@ -734,7 +735,7 @@ impl ConfigToml { /// projects trust map. On Windows, strips UNC, when possible, to try to ensure /// that different paths that point to the same location have the same key. fn project_trust_key(project_path: &Path) -> String { - dunce::canonicalize(project_path) + normalize_for_path_comparison(project_path) .unwrap_or_else(|_| project_path.to_path_buf()) .to_string_lossy() .to_string() diff --git a/codex-rs/core/src/agent/role.rs b/codex-rs/core/src/agent/role.rs index 61847f49ba..9569c02d71 100644 --- a/codex-rs/core/src/agent/role.rs +++ b/codex-rs/core/src/agent/role.rs @@ -18,6 +18,7 @@ use crate::config_loader::resolve_relative_paths_in_config_toml; use anyhow::anyhow; use codex_app_server_protocol::ConfigLayerSource; use codex_config::config_toml::ConfigToml; +use codex_exec_server::LOCAL_FS; use std::collections::BTreeMap; use std::collections::BTreeSet; use std::path::Path; @@ -168,6 +169,7 @@ mod reload { } let mut next_config = Config::load_config_with_layer_stack( + LOCAL_FS.as_ref(), merged_config, reload_overrides(config, preserve_current_provider), config.codex_home.clone(), diff --git a/codex-rs/core/src/codex/handlers.rs b/codex-rs/core/src/codex/handlers.rs index 88f3f38b65..7e0ba45dad 100644 --- a/codex-rs/core/src/codex/handlers.rs +++ b/codex-rs/core/src/codex/handlers.rs @@ -22,6 +22,7 @@ use crate::realtime_context::REALTIME_TURN_TOKEN_BUDGET; use crate::realtime_context::truncate_realtime_text_to_token_budget; use crate::realtime_conversation::REALTIME_USER_TEXT_PREFIX; use crate::realtime_conversation::prefix_realtime_v2_text; +use codex_exec_server::LOCAL_FS; use codex_features::Feature; use codex_utils_absolute_path::AbsolutePathBuf; @@ -526,6 +527,7 @@ pub async fn list_skills(sess: &Session, sub_id: String, cwds: Vec, for } }; let config_layer_stack = match load_config_layers_state( + LOCAL_FS.as_ref(), &codex_home, Some(cwd_abs.clone()), empty_cli_overrides, diff --git a/codex-rs/core/src/config/agent_roles.rs b/codex-rs/core/src/config/agent_roles.rs index 24d26ebf47..b1d28cf838 100644 --- a/codex-rs/core/src/config/agent_roles.rs +++ b/codex-rs/core/src/config/agent_roles.rs @@ -4,6 +4,7 @@ use crate::config_loader::ConfigLayerStackOrdering; use codex_config::config_toml::AgentRoleToml; use codex_config::config_toml::AgentsToml; use codex_config::config_toml::ConfigToml; +use codex_exec_server::ExecutorFileSystem; use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_absolute_path::AbsolutePathBufGuard; use serde::Deserialize; @@ -14,7 +15,8 @@ use std::path::Path; use std::path::PathBuf; use toml::Value as TomlValue; -pub(crate) fn load_agent_roles( +pub(crate) async fn load_agent_roles( + fs: &dyn ExecutorFileSystem, cfg: &ConfigToml, config_layer_stack: &ConfigLayerStack, startup_warnings: &mut Vec, @@ -24,7 +26,7 @@ pub(crate) fn load_agent_roles( /*include_disabled*/ false, ); if layers.is_empty() { - return load_agent_roles_without_layers(cfg); + return load_agent_roles_without_layers(fs, cfg).await; } let mut roles: BTreeMap = BTreeMap::new(); @@ -40,13 +42,14 @@ pub(crate) fn load_agent_roles( }; if let Some(agents_toml) = agents_toml { for (declared_role_name, role_toml) in &agents_toml.roles { - let (role_name, role) = match read_declared_role(declared_role_name, role_toml) { - Ok(role) => role, - Err(err) => { - push_agent_role_warning(startup_warnings, err); - continue; - } - }; + let (role_name, role) = + match read_declared_role(fs, declared_role_name, role_toml).await { + Ok(role) => role, + Err(err) => { + push_agent_role_warning(startup_warnings, err); + continue; + } + }; if let Some(config_file) = role.config_file.clone() { declared_role_files.insert(config_file); } @@ -68,10 +71,13 @@ pub(crate) fn load_agent_roles( if let Some(config_folder) = layer.config_folder() { for (role_name, role) in discover_agent_roles_in_dir( - config_folder.as_path().join("agents").as_path(), + fs, + &config_folder.join("agents"), &declared_role_files, startup_warnings, - )? { + ) + .await? + { if layer_roles.contains_key(&role_name) { push_agent_role_warning( startup_warnings, @@ -113,13 +119,14 @@ fn push_agent_role_warning(startup_warnings: &mut Vec, err: std::io::Err startup_warnings.push(message); } -fn load_agent_roles_without_layers( +async fn load_agent_roles_without_layers( + fs: &dyn ExecutorFileSystem, cfg: &ConfigToml, ) -> std::io::Result> { let mut roles = BTreeMap::new(); if let Some(agents_toml) = cfg.agents.as_ref() { for (declared_role_name, role_toml) in &agents_toml.roles { - let (role_name, role) = read_declared_role(declared_role_name, role_toml)?; + let (role_name, role) = read_declared_role(fs, declared_role_name, role_toml).await?; validate_required_agent_role_description(&role_name, role.description.as_deref())?; if roles.insert(role_name.clone(), role).is_some() { @@ -134,14 +141,17 @@ fn load_agent_roles_without_layers( Ok(roles) } -fn read_declared_role( +async fn read_declared_role( + fs: &dyn ExecutorFileSystem, declared_role_name: &str, role_toml: &AgentRoleToml, ) -> std::io::Result<(String, AgentRoleConfig)> { - let mut role = agent_role_config_from_toml(declared_role_name, role_toml)?; + let mut role = agent_role_config_from_toml(fs, declared_role_name, role_toml).await?; let mut role_name = declared_role_name.to_string(); if let Some(config_file) = role.config_file.as_deref() { - let parsed_file = read_resolved_agent_role_file(config_file, Some(declared_role_name))?; + let config_file = AbsolutePathBuf::from_absolute_path(config_file)?; + let parsed_file = + read_resolved_agent_role_file(fs, &config_file, Some(declared_role_name)).await?; role_name = parsed_file.role_name; role.description = parsed_file.description.or(role.description); role.nickname_candidates = parsed_file.nickname_candidates.or(role.nickname_candidates); @@ -171,12 +181,17 @@ fn agents_toml_from_layer(layer_toml: &TomlValue) -> std::io::Result std::io::Result { - let config_file = role.config_file.as_ref().map(AbsolutePathBuf::to_path_buf); - validate_agent_role_config_file(role_name, config_file.as_deref())?; + let config_file = role + .config_file + .as_ref() + .map(AbsolutePathBuf::from_absolute_path) + .transpose()?; + validate_agent_role_config_file(fs, role_name, config_file.as_ref()).await?; let description = normalize_agent_role_description( &format!("agents.{role_name}.description"), role.description.as_deref(), @@ -188,7 +203,7 @@ fn agent_role_config_from_toml( Ok(AgentRoleConfig { description, - config_file, + config_file: config_file.map(AbsolutePathBuf::into_path_buf), nickname_candidates, }) } @@ -293,15 +308,17 @@ pub(crate) fn parse_agent_role_file_contents( }) } -fn read_resolved_agent_role_file( - path: &Path, +async fn read_resolved_agent_role_file( + fs: &dyn ExecutorFileSystem, + path: &AbsolutePathBuf, role_name_hint: Option<&str>, ) -> std::io::Result { - let contents = std::fs::read_to_string(path)?; + let contents = fs.read_file_text(path, /*sandbox*/ None).await?; + let config_base_dir = path.parent().unwrap_or_else(|| path.clone()); parse_agent_role_file_contents( &contents, - path, - path.parent().unwrap_or(path), + path.as_path(), + config_base_dir.as_path(), role_name_hint, ) } @@ -359,31 +376,35 @@ fn validate_agent_role_file_developer_instructions( } } -fn validate_agent_role_config_file( +async fn validate_agent_role_config_file( + fs: &dyn ExecutorFileSystem, role_name: &str, - config_file: Option<&Path>, + config_file: Option<&AbsolutePathBuf>, ) -> std::io::Result<()> { let Some(config_file) = config_file else { return Ok(()); }; - let metadata = std::fs::metadata(config_file).map_err(|e| { - std::io::Error::new( - std::io::ErrorKind::InvalidInput, - format!( - "agents.{role_name}.config_file must point to an existing file at {}: {e}", - config_file.display() - ), - ) - })?; - if metadata.is_file() { + let metadata = fs + .get_metadata(config_file, /*sandbox*/ None) + .await + .map_err(|e| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!( + "agents.{role_name}.config_file must point to an existing file at {}: {e}", + config_file.as_path().display() + ), + ) + })?; + if metadata.is_file { Ok(()) } else { Err(std::io::Error::new( std::io::ErrorKind::InvalidInput, format!( "agents.{role_name}.config_file must point to a file: {}", - config_file.display() + config_file.as_path().display() ), )) } @@ -441,19 +462,20 @@ fn normalize_agent_role_nickname_candidates( Ok(Some(normalized_candidates)) } -fn discover_agent_roles_in_dir( - agents_dir: &Path, +async fn discover_agent_roles_in_dir( + fs: &dyn ExecutorFileSystem, + agents_dir: &AbsolutePathBuf, declared_role_files: &BTreeSet, startup_warnings: &mut Vec, ) -> std::io::Result> { let mut roles = BTreeMap::new(); - for agent_file in collect_agent_role_files(agents_dir)? { - if declared_role_files.contains(&agent_file) { + for agent_file in collect_agent_role_files(fs, agents_dir).await? { + if declared_role_files.contains(agent_file.as_path()) { continue; } let parsed_file = - match read_resolved_agent_role_file(&agent_file, /*role_name_hint*/ None) { + match read_resolved_agent_role_file(fs, &agent_file, /*role_name_hint*/ None).await { Ok(parsed_file) => parsed_file, Err(err) => { push_agent_role_warning(startup_warnings, err); @@ -468,7 +490,7 @@ fn discover_agent_roles_in_dir( std::io::ErrorKind::InvalidInput, format!( "duplicate agent role name `{role_name}` discovered in {}", - agents_dir.display() + agents_dir.as_path().display() ), ), ); @@ -478,7 +500,7 @@ fn discover_agent_roles_in_dir( role_name, AgentRoleConfig { description: parsed_file.description, - config_file: Some(agent_file), + config_file: Some(agent_file.to_path_buf()), nickname_candidates: parsed_file.nickname_candidates, }, ); @@ -487,36 +509,36 @@ fn discover_agent_roles_in_dir( Ok(roles) } -fn collect_agent_role_files(dir: &Path) -> std::io::Result> { +async fn collect_agent_role_files( + fs: &dyn ExecutorFileSystem, + dir: &AbsolutePathBuf, +) -> std::io::Result> { let mut files = Vec::new(); - collect_agent_role_files_recursive(dir, &mut files)?; - files.sort(); - Ok(files) -} + let mut dirs = vec![dir.clone()]; + while let Some(dir) = dirs.pop() { + let entries = match fs.read_directory(&dir, /*sandbox*/ None).await { + Ok(entries) => entries, + Err(err) if err.kind() == ErrorKind::NotFound => continue, + Err(err) => return Err(err), + }; -fn collect_agent_role_files_recursive(dir: &Path, files: &mut Vec) -> std::io::Result<()> { - let read_dir = match std::fs::read_dir(dir) { - Ok(read_dir) => read_dir, - Err(err) if err.kind() == ErrorKind::NotFound => return Ok(()), - Err(err) => return Err(err), - }; - - for entry in read_dir { - let entry = entry?; - let path = entry.path(); - let file_type = entry.file_type()?; - if file_type.is_dir() { - collect_agent_role_files_recursive(&path, files)?; - continue; - } - if file_type.is_file() - && path - .extension() - .is_some_and(|extension| extension == "toml") - { - files.push(path); + for entry in entries { + let path = dir.join(entry.file_name); + if entry.is_directory { + dirs.push(path); + continue; + } + if entry.is_file + && path + .as_path() + .extension() + .is_some_and(|extension| extension == "toml") + { + files.push(path); + } } } - Ok(()) + files.sort(); + Ok(files) } diff --git a/codex-rs/core/src/config/config_tests.rs b/codex-rs/core/src/config/config_tests.rs index 028ec1686a..5bdf768814 100644 --- a/codex-rs/core/src/config/config_tests.rs +++ b/codex-rs/core/src/config/config_tests.rs @@ -44,6 +44,7 @@ use codex_config::types::SkillsConfig; use codex_config::types::ToolSuggestDiscoverableType; use codex_config::types::Tui; use codex_config::types::TuiNotificationSettings; +use codex_exec_server::LOCAL_FS; use codex_features::Feature; use codex_features::FeaturesToml; use codex_model_provider_info::LMSTUDIO_OSS_PROVIDER_ID; @@ -1194,7 +1195,7 @@ network_access = false # This should be ignored. sandbox_mode_override, /*profile_sandbox_mode*/ None, WindowsSandboxLevel::Disabled, - &PathBuf::from("/tmp/test"), + /*active_project*/ None, /*sandbox_policy_constraint*/ None, ) .await; @@ -1215,7 +1216,7 @@ network_access = true # This should be ignored. sandbox_mode_override, /*profile_sandbox_mode*/ None, WindowsSandboxLevel::Disabled, - &PathBuf::from("/tmp/test"), + /*active_project*/ None, /*sandbox_policy_constraint*/ None, ) .await; @@ -1232,6 +1233,9 @@ writable_roots = [ ] exclude_tmpdir_env_var = true exclude_slash_tmp = true + +[projects."/tmp/test"] +trust_level = "trusted" "#, serde_json::json!(writable_root) ); @@ -1244,7 +1248,7 @@ exclude_slash_tmp = true sandbox_mode_override, /*profile_sandbox_mode*/ None, WindowsSandboxLevel::Disabled, - &PathBuf::from("/tmp/test"), + /*active_project*/ None, /*sandbox_policy_constraint*/ None, ) .await; @@ -1273,9 +1277,6 @@ writable_roots = [ ] exclude_tmpdir_env_var = true exclude_slash_tmp = true - -[projects."/tmp/test"] -trust_level = "trusted" "#, serde_json::json!(writable_root) ); @@ -1288,7 +1289,7 @@ trust_level = "trusted" sandbox_mode_override, /*profile_sandbox_mode*/ None, WindowsSandboxLevel::Disabled, - &PathBuf::from("/tmp/test"), + /*active_project*/ None, /*sandbox_policy_constraint*/ None, ) .await; @@ -2085,6 +2086,7 @@ async fn managed_config_overrides_oauth_store_mode() -> anyhow::Result<()> { let cwd = codex_home.path().abs(); let config_layer_stack = load_config_layers_state( + LOCAL_FS.as_ref(), codex_home.path(), Some(cwd), &Vec::new(), @@ -2218,6 +2220,7 @@ async fn managed_config_wins_over_cli_overrides() -> anyhow::Result<()> { let cwd = codex_home.path().abs(); let config_layer_stack = load_config_layers_state( + LOCAL_FS.as_ref(), codex_home.path(), Some(cwd), &[("model".to_string(), TomlValue::String("cli".to_string()))], @@ -3486,6 +3489,7 @@ async fn load_config_uses_requirements_guardian_policy_config() -> std::io::Resu .map_err(std::io::Error::other)?; let config = Config::load_config_with_layer_stack( + LOCAL_FS.as_ref(), ConfigToml::default(), ConfigOverrides { cwd: Some(codex_home.path().to_path_buf()), @@ -3518,6 +3522,7 @@ async fn load_config_ignores_empty_requirements_guardian_policy_config() -> std: .map_err(std::io::Error::other)?; let config = Config::load_config_with_layer_stack( + LOCAL_FS.as_ref(), ConfigToml::default(), ConfigOverrides { cwd: Some(codex_home.path().to_path_buf()), @@ -5330,6 +5335,7 @@ async fn test_requirements_web_search_mode_allowlist_does_not_warn_when_unset() .expect("config layer stack"); let config = Config::load_config_with_layer_stack( + LOCAL_FS.as_ref(), fixture.cfg.clone(), ConfigOverrides { cwd: Some(fixture.cwd_path()), @@ -5537,13 +5543,16 @@ trust_level = "untrusted" let cfg = toml::from_str::(config_with_untrusted) .expect("TOML deserialization should succeed"); + let active_project = ProjectConfig { + trust_level: Some(TrustLevel::Untrusted), + }; let resolution = cfg .derive_sandbox_policy( /*sandbox_mode_override*/ None, /*profile_sandbox_mode*/ None, WindowsSandboxLevel::Disabled, - &PathBuf::from("/tmp/test"), + Some(&active_project), /*sandbox_policy_constraint*/ None, ) .await; @@ -5579,6 +5588,9 @@ async fn derive_sandbox_policy_falls_back_to_constraint_value_for_implicit_defau )])), ..Default::default() }; + let active_project = ProjectConfig { + trust_level: Some(TrustLevel::Trusted), + }; let constrained = Constrained::new(SandboxPolicy::DangerFullAccess, |candidate| { if matches!(candidate, SandboxPolicy::DangerFullAccess) { Ok(()) @@ -5597,7 +5609,7 @@ async fn derive_sandbox_policy_falls_back_to_constraint_value_for_implicit_defau /*sandbox_mode_override*/ None, /*profile_sandbox_mode*/ None, WindowsSandboxLevel::Disabled, - &project_path, + Some(&active_project), Some(&constrained), ) .await; @@ -5621,6 +5633,9 @@ async fn derive_sandbox_policy_preserves_windows_downgrade_for_unsupported_fallb )])), ..Default::default() }; + let active_project = ProjectConfig { + trust_level: Some(TrustLevel::Trusted), + }; let constrained = Constrained::new(SandboxPolicy::new_workspace_write_policy(), |candidate| { if matches!(candidate, SandboxPolicy::WorkspaceWrite { .. }) { Ok(()) @@ -5639,7 +5654,7 @@ async fn derive_sandbox_policy_preserves_windows_downgrade_for_unsupported_fallb /*sandbox_mode_override*/ None, /*profile_sandbox_mode*/ None, WindowsSandboxLevel::Disabled, - &project_path, + Some(&active_project), Some(&constrained), ) .await; diff --git a/codex-rs/core/src/config/mod.rs b/codex-rs/core/src/config/mod.rs index c4217bdf2a..47a8bdf304 100644 --- a/codex-rs/core/src/config/mod.rs +++ b/codex-rs/core/src/config/mod.rs @@ -47,6 +47,8 @@ use codex_config::types::ToolSuggestDiscoverable; use codex_config::types::TuiNotificationSettings; use codex_config::types::UriBasedFileOpener; use codex_config::types::WindowsSandboxModeToml; +use codex_exec_server::ExecutorFileSystem; +use codex_exec_server::LOCAL_FS; use codex_features::Feature; use codex_features::FeatureConfigSource; use codex_features::FeatureOverrides; @@ -54,6 +56,7 @@ use codex_features::FeatureToml; use codex_features::Features; use codex_features::FeaturesToml; use codex_features::MultiAgentV2ConfigToml; +use codex_git_utils::resolve_root_git_project_for_trust; use codex_login::AuthManagerConfig; use codex_mcp::McpConfig; use codex_model_provider_info::LEGACY_OLLAMA_CHAT_PROVIDER_ID; @@ -689,6 +692,7 @@ impl ConfigBuilder { }; harness_overrides.cwd = Some(cwd.to_path_buf()); let config_layer_stack = load_config_layers_state( + LOCAL_FS.as_ref(), &codex_home, Some(cwd), &cli_overrides, @@ -718,6 +722,7 @@ impl ConfigBuilder { } }; Config::load_config_with_layer_stack( + LOCAL_FS.as_ref(), config_toml, harness_overrides, codex_home, @@ -812,6 +817,7 @@ impl Config { let codex_home = AbsolutePathBuf::from_absolute_path_checked(codex_home)?; let config_toml = deserialize_config_toml_with_base(merged, &codex_home)?; Self::load_config_with_layer_stack( + LOCAL_FS.as_ref(), config_toml, ConfigOverrides::default(), codex_home, @@ -849,6 +855,7 @@ pub async fn load_config_as_toml_with_cli_overrides( cli_overrides: Vec<(String, TomlValue)>, ) -> std::io::Result { let config_layer_stack = load_config_layers_state( + LOCAL_FS.as_ref(), codex_home, cwd.cloned(), &cli_overrides, @@ -1019,6 +1026,7 @@ pub async fn load_global_mcp_servers( // MCP servers defined in in-repo .codex/ folders. let cwd: Option = None; let config_layer_stack = load_config_layers_state( + LOCAL_FS.as_ref(), codex_home, cwd, &cli_overrides, @@ -1420,10 +1428,18 @@ impl Config { ) -> std::io::Result { // Note this ignores requirements.toml enforcement for tests. let config_layer_stack = ConfigLayerStack::default(); - Self::load_config_with_layer_stack(cfg, overrides, codex_home, config_layer_stack).await + Self::load_config_with_layer_stack( + LOCAL_FS.as_ref(), + cfg, + overrides, + codex_home, + config_layer_stack, + ) + .await } pub(crate) async fn load_config_with_layer_stack( + fs: &dyn ExecutorFileSystem, cfg: ConfigToml, overrides: ConfigOverrides, codex_home: AbsolutePathBuf, @@ -1545,9 +1561,12 @@ impl Config { .into_iter() .map(|path| AbsolutePathBuf::resolve_path_against_base(path, resolved_cwd.as_path())) .collect(); + let repo_root = resolve_root_git_project_for_trust(fs, &resolved_cwd).await; let active_project = cfg - .get_active_project(resolved_cwd.as_path()) - .await + .get_active_project( + resolved_cwd.as_path(), + repo_root.as_ref().map(AbsolutePathBuf::as_path), + ) .unwrap_or(ProjectConfig { trust_level: None }); let permission_config_syntax = resolve_permission_config_syntax( &config_layer_stack, @@ -1643,7 +1662,7 @@ impl Config { sandbox_mode, config_profile.sandbox_mode, windows_sandbox_level, - resolved_cwd.as_path(), + Some(&active_project), Some(&constrained_sandbox_policy), ) .await; @@ -1712,7 +1731,8 @@ impl Config { let multi_agent_v2 = resolve_multi_agent_v2_config(&cfg, &config_profile); let agent_roles = - agent_roles::load_agent_roles(&cfg, &config_layer_stack, &mut startup_warnings)?; + agent_roles::load_agent_roles(fs, &cfg, &config_layer_stack, &mut startup_warnings) + .await?; let openai_base_url = cfg .openai_base_url @@ -1861,8 +1881,12 @@ impl Config { .model_instructions_file .as_ref() .or(cfg.model_instructions_file.as_ref()); - let file_base_instructions = - Self::try_read_non_empty_file(model_instructions_path, "model instructions file")?; + let file_base_instructions = Self::try_read_non_empty_file( + fs, + model_instructions_path, + "model instructions file", + ) + .await?; let base_instructions = base_instructions.or(file_base_instructions); let developer_instructions = developer_instructions.or(cfg.developer_instructions); let include_permissions_instructions = config_profile @@ -1893,9 +1917,11 @@ impl Config { .as_ref() .or(cfg.experimental_compact_prompt_file.as_ref()); let file_compact_prompt = Self::try_read_non_empty_file( + fs, experimental_compact_prompt_path, "experimental compact prompt file", - )?; + ) + .await?; let compact_prompt = compact_prompt.or(file_compact_prompt); let js_repl_node_path = js_repl_node_path_override .or(config_profile.js_repl_node_path.map(Into::into)) @@ -2218,7 +2244,8 @@ impl Config { /// If `path` is `Some`, attempts to read the file at the given path and /// returns its contents as a trimmed `String`. If the file is empty, or /// is `Some` but cannot be read, returns an `Err`. - fn try_read_non_empty_file( + async fn try_read_non_empty_file( + fs: &dyn ExecutorFileSystem, path: Option<&AbsolutePathBuf>, context: &str, ) -> std::io::Result> { @@ -2226,12 +2253,15 @@ impl Config { return Ok(None); }; - let contents = std::fs::read_to_string(path).map_err(|e| { - std::io::Error::new( - e.kind(), - format!("failed to read {context} {}: {e}", path.display()), - ) - })?; + let contents = fs + .read_file_text(path, /*sandbox*/ None) + .await + .map_err(|e| { + std::io::Error::new( + e.kind(), + format!("failed to read {context} {}: {e}", path.display()), + ) + })?; let s = contents.trim().to_string(); if s.is_empty() { diff --git a/codex-rs/core/src/config/service.rs b/codex-rs/core/src/config/service.rs index 6c7f071f2b..51288e60b9 100644 --- a/codex-rs/core/src/config/service.rs +++ b/codex-rs/core/src/config/service.rs @@ -29,6 +29,7 @@ use codex_app_server_protocol::OverriddenMetadata; use codex_app_server_protocol::WriteStatus; use codex_config::CONFIG_TOML_FILE; use codex_config::config_toml::ConfigToml; +use codex_exec_server::LOCAL_FS; use codex_utils_absolute_path::AbsolutePathBuf; use serde_json::Value as JsonValue; use std::borrow::Cow; @@ -424,6 +425,7 @@ impl ConfigService { async fn load_thread_agnostic_config(&self) -> std::io::Result { let cwd: Option = None; load_config_layers_state( + LOCAL_FS.as_ref(), &self.codex_home, cwd, &self.cli_overrides, diff --git a/codex-rs/core/src/config_loader/README.md b/codex-rs/core/src/config_loader/README.md index 04b72e4ca1..44a514a10a 100644 --- a/codex-rs/core/src/config_loader/README.md +++ b/codex-rs/core/src/config_loader/README.md @@ -10,7 +10,7 @@ This module is the canonical place to **load and describe Codex configuration la Exported from `codex_core::config_loader`: -- `load_config_layers_state(codex_home, cwd_opt, cli_overrides, overrides, cloud_requirements) -> ConfigLayerStack` +- `load_config_layers_state(fs, codex_home, cwd_opt, cli_overrides, overrides, cloud_requirements) -> ConfigLayerStack` - `ConfigLayerStack` - `effective_config() -> toml::Value` - `origins() -> HashMap` @@ -38,18 +38,22 @@ computing the effective config and origins metadata. This is what Most callers want the effective config plus metadata: ```rust -use codex_core::config_loader::{load_config_layers_state, LoaderOverrides}; +use codex_core::config_loader::{ + CloudRequirementsLoader, LoaderOverrides, load_config_layers_state, +}; +use codex_exec_server::LOCAL_FS; use codex_utils_absolute_path::AbsolutePathBuf; use toml::Value as TomlValue; let cli_overrides: Vec<(String, TomlValue)> = Vec::new(); let cwd = AbsolutePathBuf::current_dir()?; let layers = load_config_layers_state( + LOCAL_FS.as_ref(), &codex_home, Some(cwd), &cli_overrides, LoaderOverrides::default(), - None, + CloudRequirementsLoader::default(), ).await?; let effective = layers.effective_config(); diff --git a/codex-rs/core/src/config_loader/layer_io.rs b/codex-rs/core/src/config_loader/layer_io.rs index af77bdafa5..6bd9a9130f 100644 --- a/codex-rs/core/src/config_loader/layer_io.rs +++ b/codex-rs/core/src/config_loader/layer_io.rs @@ -5,11 +5,11 @@ use super::macos::ManagedAdminConfigLayer; use super::macos::load_managed_admin_config_layer; use codex_config::config_error_from_toml; use codex_config::io_error_from_config_error; +use codex_exec_server::ExecutorFileSystem; use codex_utils_absolute_path::AbsolutePathBuf; use std::io; use std::path::Path; use std::path::PathBuf; -use tokio::fs; use toml::Value as TomlValue; #[cfg(unix)] @@ -36,6 +36,7 @@ pub(super) struct LoadedConfigLayers { } pub(super) async fn load_config_layers_internal( + fs: &dyn ExecutorFileSystem, codex_home: &Path, overrides: LoaderOverrides, ) -> io::Result { @@ -57,7 +58,7 @@ pub(super) async fn load_config_layers_internal( )?; let managed_config = - read_config_from_path(&managed_config_path, /*log_missing_as_info*/ false) + read_config_from_path(fs, &managed_config_path, /*log_missing_as_info*/ false) .await? .map(|managed_config| MangedConfigFromFile { managed_config, @@ -89,15 +90,16 @@ fn map_managed_admin_layer(layer: ManagedAdminConfigLayer) -> ManagedConfigFromM } pub(super) async fn read_config_from_path( - path: impl AsRef, + fs: &dyn ExecutorFileSystem, + path: &AbsolutePathBuf, log_missing_as_info: bool, ) -> io::Result> { - match fs::read_to_string(path.as_ref()).await { + match fs.read_file_text(path, /*sandbox*/ None).await { Ok(contents) => match toml::from_str::(&contents) { Ok(value) => Ok(Some(value)), Err(err) => { - tracing::error!("Failed to parse {}: {err}", path.as_ref().display()); - let config_error = config_error_from_toml(path.as_ref(), &contents, err.clone()); + tracing::error!("Failed to parse {}: {err}", path.as_path().display()); + let config_error = config_error_from_toml(path.as_path(), &contents, err.clone()); Err(io_error_from_config_error( io::ErrorKind::InvalidData, config_error, @@ -107,14 +109,14 @@ pub(super) async fn read_config_from_path( }, Err(err) if err.kind() == io::ErrorKind::NotFound => { if log_missing_as_info { - tracing::info!("{} not found, using defaults", path.as_ref().display()); + tracing::info!("{} not found, using defaults", path.as_path().display()); } else { - tracing::debug!("{} not found", path.as_ref().display()); + tracing::debug!("{} not found", path.as_path().display()); } Ok(None) } Err(err) => { - tracing::error!("Failed to read {}: {err}", path.as_ref().display()); + tracing::error!("Failed to read {}: {err}", path.as_path().display()); Err(err) } } diff --git a/codex-rs/core/src/config_loader/mod.rs b/codex-rs/core/src/config_loader/mod.rs index f7830b7089..fb02e670ef 100644 --- a/codex-rs/core/src/config_loader/mod.rs +++ b/codex-rs/core/src/config_loader/mod.rs @@ -11,6 +11,7 @@ use codex_config::CONFIG_TOML_FILE; use codex_config::ConfigRequirementsWithSources; use codex_config::config_toml::ConfigToml; use codex_config::config_toml::ProjectConfig; +use codex_exec_server::ExecutorFileSystem; use codex_git_utils::resolve_root_git_project_for_trust; use codex_protocol::config_types::ApprovalsReviewer; use codex_protocol::config_types::SandboxMode; @@ -118,6 +119,7 @@ pub(crate) async fn first_layer_config_error_from_entries( /// thread-agnostic config loading (e.g., for the app server's `/config` /// endpoint) should `cwd` be `None`. pub async fn load_config_layers_state( + fs: &dyn ExecutorFileSystem, codex_home: &Path, cwd: Option, cli_overrides: &[(String, TomlValue)], @@ -142,11 +144,12 @@ pub async fn load_config_layers_state( // Honor the system requirements.toml location. let requirements_toml_file = system_requirements_toml_file()?; - load_requirements_toml(&mut config_requirements_toml, requirements_toml_file).await?; + load_requirements_toml(fs, &mut config_requirements_toml, &requirements_toml_file).await?; // Make a best-effort to support the legacy `managed_config.toml` as a // requirements specification. - let loaded_config_layers = layer_io::load_config_layers_internal(codex_home, overrides).await?; + let loaded_config_layers = + layer_io::load_config_layers_internal(fs, codex_home, overrides).await?; load_requirements_from_legacy_scheme( &mut config_requirements_toml, loaded_config_layers.clone(), @@ -173,7 +176,7 @@ pub async fn load_config_layers_state( // if it exists. let system_config_toml_file = system_config_toml_file()?; let system_layer = - load_config_toml_for_required_layer(&system_config_toml_file, |config_toml| { + load_config_toml_for_required_layer(fs, &system_config_toml_file, |config_toml| { ConfigLayerEntry::new( ConfigLayerSource::System { file: system_config_toml_file.clone(), @@ -188,7 +191,7 @@ pub async fn load_config_layers_state( // exists, but is malformed, then this error should be propagated to the // user. let user_file = AbsolutePathBuf::resolve_path_against_base(CONFIG_TOML_FILE, codex_home); - let user_layer = load_config_toml_for_required_layer(&user_file, |config_toml| { + let user_layer = load_config_toml_for_required_layer(fs, &user_file, |config_toml| { ConfigLayerEntry::new( ConfigLayerSource::User { file: user_file.clone(), @@ -222,6 +225,7 @@ pub async fn load_config_layers_state( } }; let project_trust_context = match project_trust_context( + fs, &merged_so_far, &cwd, &project_root_markers, @@ -247,6 +251,7 @@ pub async fn load_config_layers_state( } }; let project_layers = load_project_layers( + fs, &cwd, &project_trust_context.project_root, &project_trust_context, @@ -320,22 +325,23 @@ pub async fn load_config_layers_state( /// - If there is an error reading the file or parsing the TOML, returns an /// error. async fn load_config_toml_for_required_layer( - config_toml: impl AsRef, + fs: &dyn ExecutorFileSystem, + toml_file: &AbsolutePathBuf, create_entry: impl FnOnce(TomlValue) -> ConfigLayerEntry, ) -> io::Result { - let toml_file = config_toml.as_ref(); - let toml_value = match tokio::fs::read_to_string(toml_file).await { + let toml_value = match fs.read_file_text(toml_file, /*sandbox*/ None).await { Ok(contents) => { let config: TomlValue = toml::from_str(&contents).map_err(|err| { - let config_error = config_error_from_toml(toml_file, &contents, err.clone()); + let config_error = + config_error_from_toml(toml_file.as_path(), &contents, err.clone()); io_error_from_config_error(io::ErrorKind::InvalidData, config_error, Some(err)) })?; - let config_parent = toml_file.parent().ok_or_else(|| { + let config_parent = toml_file.as_path().parent().ok_or_else(|| { io::Error::new( io::ErrorKind::InvalidData, format!( "Config file {} has no parent directory", - toml_file.display() + toml_file.as_path().display() ), ) })?; @@ -347,7 +353,10 @@ async fn load_config_toml_for_required_layer( } else { Err(io::Error::new( e.kind(), - format!("Failed to read config file {}: {e}", toml_file.display()), + format!( + "Failed to read config file {}: {e}", + toml_file.as_path().display() + ), )) } } @@ -360,12 +369,14 @@ async fn load_config_toml_for_required_layer( /// `requirements.toml` location to `config_requirements_toml` by filling in /// any unset fields. async fn load_requirements_toml( + fs: &dyn ExecutorFileSystem, config_requirements_toml: &mut ConfigRequirementsWithSources, - requirements_toml_file: impl AsRef, + requirements_toml_file: &AbsolutePathBuf, ) -> io::Result<()> { - let requirements_toml_file = - AbsolutePathBuf::from_absolute_path(requirements_toml_file.as_ref())?; - match tokio::fs::read_to_string(&requirements_toml_file).await { + match fs + .read_file_text(requirements_toml_file, /*sandbox*/ None) + .await + { Ok(contents) => { let requirements_config: ConfigRequirementsToml = toml::from_str(&contents).map_err(|e| { @@ -373,7 +384,7 @@ async fn load_requirements_toml( io::ErrorKind::InvalidData, format!( "Error parsing requirements file {}: {e}", - requirements_toml_file.as_ref().display(), + requirements_toml_file.as_path().display(), ), ) })?; @@ -390,7 +401,7 @@ async fn load_requirements_toml( e.kind(), format!( "Failed to read requirements file {}: {e}", - requirements_toml_file.as_ref().display(), + requirements_toml_file.as_path().display(), ), )); } @@ -632,6 +643,7 @@ fn project_layer_entry( } async fn project_trust_context( + fs: &dyn ExecutorFileSystem, merged_config: &TomlValue, cwd: &AbsolutePathBuf, project_root_markers: &[String], @@ -646,12 +658,14 @@ async fn project_trust_context( .map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidData, err))? }; - let project_root = find_project_root(cwd, project_root_markers).await?; + let project_root = find_project_root(fs, cwd, project_root_markers).await?; let projects = project_trust_config.projects.unwrap_or_default(); let project_root_key = project_trust_key(project_root.as_path()); - let repo_root = resolve_root_git_project_for_trust(cwd.as_path()).await; - let repo_root_key = repo_root.as_ref().map(|root| project_trust_key(root)); + let repo_root = resolve_root_git_project_for_trust(fs, cwd).await; + let repo_root_key = repo_root + .as_ref() + .map(|root| project_trust_key(root.as_path())); let projects_trust = projects .into_iter() @@ -742,6 +756,7 @@ fn copy_shape_from_original(original: &TomlValue, resolved: &TomlValue) -> TomlV } async fn find_project_root( + fs: &dyn ExecutorFileSystem, cwd: &AbsolutePathBuf, project_root_markers: &[String], ) -> io::Result { @@ -749,11 +764,15 @@ async fn find_project_root( return Ok(cwd.clone()); } - for ancestor in cwd.as_path().ancestors() { + for ancestor in cwd.ancestors() { for marker in project_root_markers { let marker_path = ancestor.join(marker); - if tokio::fs::metadata(&marker_path).await.is_ok() { - return AbsolutePathBuf::from_absolute_path(ancestor); + if fs + .get_metadata(&marker_path, /*sandbox*/ None) + .await + .is_ok() + { + return Ok(ancestor); } } } @@ -766,6 +785,7 @@ async fn find_project_root( /// starting from folders closest to `project_root` (which is the lowest /// precedence) to those closest to `cwd` (which is the highest precedence). async fn load_project_layers( + fs: &dyn ExecutorFileSystem, cwd: &AbsolutePathBuf, project_root: &AbsolutePathBuf, trust_context: &ProjectTrustContext, @@ -775,13 +795,12 @@ async fn load_project_layers( let codex_home_normalized = normalize_path(codex_home_abs.as_path()).unwrap_or_else(|_| codex_home_abs.to_path_buf()); let mut dirs = cwd - .as_path() .ancestors() .scan(false, |done, a| { if *done { None } else { - if a == project_root.as_path() { + if &a == project_root { *done = true; } Some(a) @@ -792,25 +811,24 @@ async fn load_project_layers( let mut layers = Vec::new(); for dir in dirs { - let dot_codex = dir.join(".codex"); - if !tokio::fs::metadata(&dot_codex) + let dot_codex_abs = dir.join(".codex"); + if !fs + .get_metadata(&dot_codex_abs, /*sandbox*/ None) .await - .map(|meta| meta.is_dir()) + .map(|metadata| metadata.is_directory) .unwrap_or(false) { continue; } - let layer_dir = AbsolutePathBuf::from_absolute_path(dir)?; - let decision = trust_context.decision_for_dir(&layer_dir); - let dot_codex_abs = AbsolutePathBuf::from_absolute_path(&dot_codex)?; + let decision = trust_context.decision_for_dir(&dir); let dot_codex_normalized = normalize_path(dot_codex_abs.as_path()).unwrap_or_else(|_| dot_codex_abs.to_path_buf()); if dot_codex_abs == codex_home_abs || dot_codex_normalized == codex_home_normalized { continue; } let config_file = dot_codex_abs.join(CONFIG_TOML_FILE); - match tokio::fs::read_to_string(&config_file).await { + match fs.read_file_text(&config_file, /*sandbox*/ None).await { Ok(contents) => { let config: TomlValue = match toml::from_str(&contents) { Ok(config) => config, @@ -827,7 +845,7 @@ async fn load_project_layers( layers.push(project_layer_entry( trust_context, &dot_codex_abs, - &layer_dir, + &dir, TomlValue::Table(toml::map::Map::new()), /*config_toml_exists*/ true, )); @@ -839,7 +857,7 @@ async fn load_project_layers( let entry = project_layer_entry( trust_context, &dot_codex_abs, - &layer_dir, + &dir, config, /*config_toml_exists*/ true, ); @@ -853,7 +871,7 @@ async fn load_project_layers( layers.push(project_layer_entry( trust_context, &dot_codex_abs, - &layer_dir, + &dir, TomlValue::Table(toml::map::Map::new()), /*config_toml_exists*/ false, )); diff --git a/codex-rs/core/src/config_loader/tests.rs b/codex-rs/core/src/config_loader/tests.rs index ed69682e86..c706e91062 100644 --- a/codex-rs/core/src/config_loader/tests.rs +++ b/codex-rs/core/src/config_loader/tests.rs @@ -16,6 +16,7 @@ use crate::config_loader::version_for_toml; use codex_config::CONFIG_TOML_FILE; use codex_config::config_toml::ConfigToml; use codex_config::config_toml::ProjectConfig; +use codex_exec_server::LOCAL_FS; use codex_protocol::config_types::TrustLevel; use codex_protocol::config_types::WebSearchMode; use codex_protocol::protocol::AskForApproval; @@ -92,6 +93,7 @@ async fn returns_config_error_for_invalid_user_config_toml() { let cwd = AbsolutePathBuf::try_from(tmp.path()).expect("cwd"); let err = load_config_layers_state( + LOCAL_FS.as_ref(), tmp.path(), Some(cwd), &[] as &[(String, TomlValue)], @@ -119,6 +121,7 @@ async fn returns_config_error_for_invalid_managed_config_toml() { let cwd = AbsolutePathBuf::try_from(tmp.path()).expect("cwd"); let err = load_config_layers_state( + LOCAL_FS.as_ref(), tmp.path(), Some(cwd), &[] as &[(String, TomlValue)], @@ -203,6 +206,7 @@ extra = true let cwd = AbsolutePathBuf::try_from(tmp.path()).expect("cwd"); let state = load_config_layers_state( + LOCAL_FS.as_ref(), tmp.path(), Some(cwd), &[] as &[(String, TomlValue)], @@ -235,6 +239,7 @@ async fn returns_empty_when_all_layers_missing() { let cwd = AbsolutePathBuf::try_from(tmp.path()).expect("cwd"); let layers = load_config_layers_state( + LOCAL_FS.as_ref(), tmp.path(), Some(cwd), &[] as &[(String, TomlValue)], @@ -327,6 +332,7 @@ flag = false let cwd = AbsolutePathBuf::try_from(tmp.path()).expect("cwd"); let state = load_config_layers_state( + LOCAL_FS.as_ref(), tmp.path(), Some(cwd), &[] as &[(String, TomlValue)], @@ -428,6 +434,7 @@ allowed_sandbox_modes = ["read-only"] ); let state = load_config_layers_state( + LOCAL_FS.as_ref(), tmp.path(), Some(AbsolutePathBuf::try_from(tmp.path())?), &[] as &[(String, TomlValue)], @@ -489,6 +496,7 @@ allowed_approval_policies = ["never"] ); let state = load_config_layers_state( + LOCAL_FS.as_ref(), tmp.path(), Some(AbsolutePathBuf::try_from(tmp.path())?), &[] as &[(String, TomlValue)], @@ -529,8 +537,14 @@ personality = true ) .await?; + let requirements_file = AbsolutePathBuf::try_from(requirements_file)?; let mut config_requirements_toml = ConfigRequirementsWithSources::default(); - load_requirements_toml(&mut config_requirements_toml, &requirements_file).await?; + load_requirements_toml( + LOCAL_FS.as_ref(), + &mut config_requirements_toml, + &requirements_file, + ) + .await?; assert_eq!( config_requirements_toml @@ -620,6 +634,7 @@ allowed_approval_policies = ["on-request"] ), ); let state = load_config_layers_state( + LOCAL_FS.as_ref(), tmp.path(), Some(AbsolutePathBuf::try_from(tmp.path())?), &[] as &[(String, TomlValue)], @@ -691,7 +706,12 @@ allowed_approval_policies = ["on-request"] guardian_policy_config: None, }, ); - load_requirements_toml(&mut config_requirements_toml, &requirements_file).await?; + load_requirements_toml( + LOCAL_FS.as_ref(), + &mut config_requirements_toml, + &AbsolutePathBuf::try_from(requirements_file)?, + ) + .await?; assert_eq!( config_requirements_toml @@ -735,6 +755,7 @@ async fn load_config_layers_includes_cloud_requirements() -> anyhow::Result<()> let cloud_requirements = CloudRequirementsLoader::new(async move { Ok(Some(requirements)) }); let layers = load_config_layers_state( + LOCAL_FS.as_ref(), &codex_home, Some(cwd), &[] as &[(String, TomlValue)], @@ -771,6 +792,7 @@ async fn load_config_layers_fails_when_cloud_requirements_loader_fails() -> anyh let cwd = AbsolutePathBuf::from_absolute_path(tmp.path())?; let err = load_config_layers_state( + LOCAL_FS.as_ref(), &codex_home, Some(cwd), &[] as &[(String, TomlValue)], @@ -823,6 +845,7 @@ async fn project_layers_prefer_closest_cwd() -> std::io::Result<()> { .await?; let cwd = AbsolutePathBuf::from_absolute_path(&nested)?; let layers = load_config_layers_state( + LOCAL_FS.as_ref(), &codex_home, Some(cwd), &[] as &[(String, TomlValue)], @@ -967,6 +990,7 @@ async fn project_layer_is_added_when_dot_codex_exists_without_config_toml() -> s .await?; let cwd = AbsolutePathBuf::from_absolute_path(&nested)?; let layers = load_config_layers_state( + LOCAL_FS.as_ref(), &codex_home, Some(cwd), &[] as &[(String, TomlValue)], @@ -1006,6 +1030,7 @@ async fn codex_home_is_not_loaded_as_project_layer_from_home_dir() -> std::io::R let cwd = AbsolutePathBuf::from_absolute_path(&home_dir)?; let layers = load_config_layers_state( + LOCAL_FS.as_ref(), &codex_home, Some(cwd), &[] as &[(String, TomlValue)], @@ -1062,6 +1087,7 @@ async fn codex_home_within_project_tree_is_not_double_loaded() -> std::io::Resul let cwd = AbsolutePathBuf::from_absolute_path(&nested)?; let layers = load_config_layers_state( + LOCAL_FS.as_ref(), &project_dot_codex, Some(cwd), &[] as &[(String, TomlValue)], @@ -1132,6 +1158,7 @@ async fn project_layers_disabled_when_untrusted_or_unknown() -> std::io::Result< .await?; let layers_untrusted = load_config_layers_state( + LOCAL_FS.as_ref(), &codex_home_untrusted, Some(cwd.clone()), &[] as &[(String, TomlValue)], @@ -1170,6 +1197,7 @@ async fn project_layers_disabled_when_untrusted_or_unknown() -> std::io::Result< .await?; let layers_unknown = load_config_layers_state( + LOCAL_FS.as_ref(), &codex_home_unknown, Some(cwd), &[] as &[(String, TomlValue)], @@ -1328,6 +1356,7 @@ async fn invalid_project_config_ignored_when_untrusted_or_unknown() -> std::io:: } let layers = load_config_layers_state( + LOCAL_FS.as_ref(), &codex_home, Some(cwd.clone()), &[] as &[(String, TomlValue)], @@ -1390,6 +1419,7 @@ async fn cli_overrides_with_relative_paths_do_not_break_trust_check() -> std::io )]; load_config_layers_state( + LOCAL_FS.as_ref(), &codex_home, Some(cwd), &cli_overrides, @@ -1432,6 +1462,7 @@ async fn project_root_markers_supports_alternate_markers() -> std::io::Result<() let cwd = AbsolutePathBuf::from_absolute_path(&nested)?; let layers = load_config_layers_state( + LOCAL_FS.as_ref(), &codex_home, Some(cwd), &[] as &[(String, TomlValue)], diff --git a/codex-rs/core/src/git_info_tests.rs b/codex-rs/core/src/git_info_tests.rs index e3ef1fa62f..0ffdd1c4df 100644 --- a/codex-rs/core/src/git_info_tests.rs +++ b/codex-rs/core/src/git_info_tests.rs @@ -1,3 +1,4 @@ +use codex_exec_server::LOCAL_FS; use codex_git_utils::GitInfo; use codex_git_utils::GitSha; use codex_git_utils::collect_git_info; @@ -5,6 +6,9 @@ use codex_git_utils::get_has_changes; use codex_git_utils::git_diff_to_remote; use codex_git_utils::recent_commits; use codex_git_utils::resolve_root_git_project_for_trust; +use codex_utils_path::normalize_for_path_comparison; +use core_test_support::PathBufExt; +use core_test_support::PathExt; use core_test_support::skip_if_sandbox; use std::fs; use std::path::PathBuf; @@ -430,7 +434,7 @@ async fn test_get_git_working_tree_state_branch_fallback() { async fn resolve_root_git_project_for_trust_returns_none_outside_repo() { let tmp = TempDir::new().expect("tempdir"); assert!( - resolve_root_git_project_for_trust(tmp.path()) + resolve_root_git_project_for_trust(LOCAL_FS.as_ref(), &tmp.path().abs()) .await .is_none() ); @@ -439,18 +443,17 @@ async fn resolve_root_git_project_for_trust_returns_none_outside_repo() { #[tokio::test] async fn resolve_root_git_project_for_trust_regular_repo_returns_repo_root() { let temp_dir = TempDir::new().expect("Failed to create temp dir"); - let repo_path = create_test_git_repo(&temp_dir).await; - let expected = std::fs::canonicalize(&repo_path).unwrap(); + let repo_path = create_test_git_repo(&temp_dir).await.abs(); assert_eq!( - resolve_root_git_project_for_trust(&repo_path).await, - Some(expected.clone()) + resolve_root_git_project_for_trust(LOCAL_FS.as_ref(), &repo_path).await, + Some(repo_path.clone()) ); let nested = repo_path.join("sub/dir"); - std::fs::create_dir_all(&nested).unwrap(); + std::fs::create_dir_all(nested.as_path()).unwrap(); assert_eq!( - resolve_root_git_project_for_trust(&nested).await, - Some(expected) + resolve_root_git_project_for_trust(LOCAL_FS.as_ref(), &nested).await, + Some(repo_path) ); } @@ -473,17 +476,27 @@ async fn resolve_root_git_project_for_trust_detects_worktree_and_returns_main_ro .output() .expect("git worktree add"); - let expected = std::fs::canonicalize(&repo_path).ok(); - let got = resolve_root_git_project_for_trust(&wt_root) - .await - .and_then(|p| std::fs::canonicalize(p).ok()); - assert_eq!(got, expected); + let expected = normalize_for_path_comparison(&repo_path).unwrap(); + let wt_root = wt_root.abs(); + let got = resolve_root_git_project_for_trust(LOCAL_FS.as_ref(), &wt_root).await; + assert_eq!( + got.as_ref() + .map(normalize_for_path_comparison) + .transpose() + .unwrap(), + Some(expected.clone()) + ); let nested = wt_root.join("nested/sub"); - std::fs::create_dir_all(&nested).unwrap(); - let got_nested = resolve_root_git_project_for_trust(&nested) - .await - .and_then(|p| std::fs::canonicalize(p).ok()); - assert_eq!(got_nested, expected); + std::fs::create_dir_all(nested.as_path()).unwrap(); + let got_nested = resolve_root_git_project_for_trust(LOCAL_FS.as_ref(), &nested).await; + assert_eq!( + got_nested + .as_ref() + .map(normalize_for_path_comparison) + .transpose() + .unwrap(), + Some(expected) + ); } #[tokio::test] @@ -502,13 +515,15 @@ async fn resolve_root_git_project_for_trust_detects_worktree_pointer_without_git ) .unwrap(); - let expected = std::fs::canonicalize(&repo_root).unwrap(); + let expected = repo_root.abs(); + let worktree_root = worktree_root.abs(); assert_eq!( - resolve_root_git_project_for_trust(&worktree_root).await, + resolve_root_git_project_for_trust(LOCAL_FS.as_ref(), &worktree_root).await, Some(expected.clone()) ); + let nested = worktree_root.join("nested"); assert_eq!( - resolve_root_git_project_for_trust(&worktree_root.join("nested")).await, + resolve_root_git_project_for_trust(LOCAL_FS.as_ref(), &nested).await, Some(expected) ); } @@ -529,9 +544,15 @@ async fn resolve_root_git_project_for_trust_non_worktrees_gitdir_returns_none() ) .unwrap(); - assert!(resolve_root_git_project_for_trust(&proj).await.is_none()); + let proj = proj.abs(); assert!( - resolve_root_git_project_for_trust(&proj.join("nested")) + resolve_root_git_project_for_trust(LOCAL_FS.as_ref(), &proj) + .await + .is_none() + ); + let nested = proj.join("nested"); + assert!( + resolve_root_git_project_for_trust(LOCAL_FS.as_ref(), &nested) .await .is_none() ); diff --git a/codex-rs/core/src/guardian/tests.rs b/codex-rs/core/src/guardian/tests.rs index 485f222551..2712a94039 100644 --- a/codex-rs/core/src/guardian/tests.rs +++ b/codex-rs/core/src/guardian/tests.rs @@ -16,6 +16,7 @@ use crate::config_loader::RequirementSource; use crate::config_loader::Sourced; use crate::test_support; use codex_config::config_toml::ConfigToml; +use codex_exec_server::LOCAL_FS; use codex_network_proxy::NetworkProxyConfig; use codex_protocol::ThreadId; use codex_protocol::approvals::NetworkApprovalProtocol; @@ -1740,6 +1741,7 @@ async fn guardian_review_session_config_uses_requirements_guardian_policy_config ) .expect("config layer stack"); let parent_config = Config::load_config_with_layer_stack( + LOCAL_FS.as_ref(), ConfigToml::default(), ConfigOverrides { cwd: Some(workspace.path().to_path_buf()), @@ -1776,6 +1778,7 @@ async fn guardian_review_session_config_uses_default_guardian_policy_without_req ConfigLayerStack::new(Vec::new(), Default::default(), Default::default()) .expect("config layer stack"); let parent_config = Config::load_config_with_layer_stack( + LOCAL_FS.as_ref(), ConfigToml::default(), ConfigOverrides { cwd: Some(workspace.path().to_path_buf()), diff --git a/codex-rs/core/src/network_proxy_loader.rs b/codex-rs/core/src/network_proxy_loader.rs index b5eaf26b7b..af4280bfb4 100644 --- a/codex-rs/core/src/network_proxy_loader.rs +++ b/codex-rs/core/src/network_proxy_loader.rs @@ -16,6 +16,7 @@ use codex_config::CONFIG_TOML_FILE; use codex_config::permissions_toml::NetworkToml; use codex_config::permissions_toml::PermissionsToml; use codex_config::permissions_toml::overlay_network_domain_permissions; +use codex_exec_server::LOCAL_FS; use codex_network_proxy::ConfigReloader; use codex_network_proxy::ConfigState; use codex_network_proxy::NetworkProxyConfig; @@ -46,6 +47,7 @@ async fn build_config_state_with_mtimes() -> Result<(ConfigState, Vec Vec { } } -async fn build_recent_work_section(cwd: &Path, recent_threads: &[StoredThread]) -> Option { +async fn build_recent_work_section( + cwd: &AbsolutePathBuf, + recent_threads: &[StoredThread], +) -> Option { let mut groups: HashMap> = HashMap::new(); for entry in recent_threads { - let group = resolve_root_git_project_for_trust(&entry.cwd) - .await - .unwrap_or_else(|| entry.cwd.clone()); + let group = match AbsolutePathBuf::from_absolute_path(entry.cwd.as_path()) { + Ok(entry_cwd) => resolve_root_git_project_for_trust(LOCAL_FS.as_ref(), &entry_cwd) + .await + .map(AbsolutePathBuf::into_path_buf) + .unwrap_or_else(|| entry.cwd.clone()), + Err(_) => entry.cwd.clone(), + }; groups.entry(group).or_default().push(entry); } - let current_group = resolve_root_git_project_for_trust(cwd) + let current_group = resolve_root_git_project_for_trust(LOCAL_FS.as_ref(), cwd) .await - .unwrap_or_else(|| cwd.to_path_buf()); + .map(AbsolutePathBuf::into_path_buf) + .unwrap_or_else(|| cwd.clone().into_path_buf()); let mut groups = groups.into_iter().collect::>(); groups.sort_by(|(left_group, left_entries), (right_group, right_entries)| { let left_latest = left_entries @@ -309,18 +319,19 @@ pub(crate) fn truncate_realtime_text_to_token_budget(text: &str, budget_tokens: } async fn build_workspace_section_with_user_root( - cwd: &Path, + cwd: &AbsolutePathBuf, user_root: Option, ) -> Option { - let git_root = resolve_root_git_project_for_trust(cwd).await; - let cwd_tree = render_tree(cwd); + let cwd_path = cwd.as_path(); + let git_root = resolve_root_git_project_for_trust(LOCAL_FS.as_ref(), cwd).await; + let cwd_tree = render_tree(cwd_path); let git_root_tree = git_root .as_ref() - .filter(|git_root| git_root.as_path() != cwd) - .and_then(|git_root| render_tree(git_root)); + .filter(|git_root| git_root.as_path() != cwd_path) + .and_then(|git_root| render_tree(git_root.as_path())); let user_root_tree = user_root .as_ref() - .filter(|user_root| user_root.as_path() != cwd) + .filter(|user_root| user_root.as_path() != cwd_path) .filter(|user_root| { git_root .as_ref() @@ -333,8 +344,8 @@ async fn build_workspace_section_with_user_root( } let mut lines = vec![ - format!("Current working directory: {}", cwd.display()), - format!("Working directory name: {}", file_name_string(cwd)), + format!("Current working directory: {}", cwd_path.display()), + format!("Working directory name: {}", file_name_string(cwd_path)), ]; if let Some(git_root) = &git_root { @@ -465,14 +476,19 @@ async fn format_thread_group( entries: Vec<&StoredThread>, ) -> Option { let latest = entries.first()?; - let group_label = if resolve_root_git_project_for_trust(latest.cwd.as_path()) - .await - .is_some() - { - format!("### Git repo: {}", group.display()) - } else { - format!("### Directory: {}", group.display()) - }; + let group_label = + if let Ok(latest_cwd) = AbsolutePathBuf::from_absolute_path(latest.cwd.as_path()) { + if resolve_root_git_project_for_trust(LOCAL_FS.as_ref(), &latest_cwd) + .await + .is_some() + { + format!("### Git repo: {}", group.display()) + } else { + format!("### Directory: {}", group.display()) + } + } else { + format!("### Directory: {}", group.display()) + }; let mut lines = vec![ group_label, format!("Recent sessions: {}", entries.len()), diff --git a/codex-rs/core/src/realtime_context_tests.rs b/codex-rs/core/src/realtime_context_tests.rs index 99cb00f0de..4096a1692a 100644 --- a/codex-rs/core/src/realtime_context_tests.rs +++ b/codex-rs/core/src/realtime_context_tests.rs @@ -19,6 +19,8 @@ use codex_protocol::protocol::GitInfo; use codex_protocol::protocol::SandboxPolicy; use codex_protocol::protocol::SessionSource; use codex_thread_store::StoredThread; +use core_test_support::PathBufExt; +use core_test_support::PathExt; use pretty_assertions::assert_eq; use std::fs; use std::path::PathBuf; @@ -234,7 +236,7 @@ fn fixed_section_budgets_apply_per_section_without_total_blob_truncation() { async fn workspace_section_requires_meaningful_structure() { let cwd = TempDir::new().expect("tempdir"); assert_eq!( - build_workspace_section_with_user_root(cwd.path(), /*user_root*/ None).await, + build_workspace_section_with_user_root(&cwd.path().abs(), /*user_root*/ None).await, None ); } @@ -245,9 +247,10 @@ async fn workspace_section_includes_tree_when_entries_exist() { fs::create_dir(cwd.path().join("docs")).expect("create docs dir"); fs::write(cwd.path().join("README.md"), "hello").expect("write readme"); - let section = build_workspace_section_with_user_root(cwd.path(), /*user_root*/ None) - .await - .expect("workspace section"); + let section = + build_workspace_section_with_user_root(&cwd.path().abs(), /*user_root*/ None) + .await + .expect("workspace section"); assert!(section.contains("Working directory tree:")); assert!(section.contains("- docs/")); assert!(section.contains("- README.md")); @@ -267,7 +270,7 @@ async fn workspace_section_includes_user_root_tree_when_distinct() { fs::create_dir_all(user_root.join("code")).expect("create user root child"); fs::write(user_root.join(".zshrc"), "export TEST=1").expect("write home file"); - let section = build_workspace_section_with_user_root(cwd.as_path(), Some(user_root)) + let section = build_workspace_section_with_user_root(&cwd.abs(), Some(user_root)) .await .expect("workspace section"); assert!(section.contains("User root tree:")); @@ -309,9 +312,9 @@ async fn recent_work_section_groups_threads_by_cwd() { stored_thread(outside.to_string_lossy().as_ref(), "", "Inspect flaky test"), ]; let current_cwd = workspace_a; - let repo = fs::canonicalize(repo).expect("canonicalize repo"); + let repo = repo.abs(); - let section = build_recent_work_section(current_cwd.as_path(), &recent_threads) + let section = build_recent_work_section(¤t_cwd.abs(), &recent_threads) .await .expect("recent work section"); assert!(section.contains(&format!("### Git repo: {}", repo.display()))); diff --git a/codex-rs/git-utils/Cargo.toml b/codex-rs/git-utils/Cargo.toml index 7ecc72dfa9..5500546db7 100644 --- a/codex-rs/git-utils/Cargo.toml +++ b/codex-rs/git-utils/Cargo.toml @@ -9,6 +9,8 @@ readme = "README.md" workspace = true [dependencies] +codex-exec-server = { workspace = true } +codex-protocol = { workspace = true } codex-utils-absolute-path = { workspace = true } futures = { workspace = true, features = ["alloc"] } once_cell = { workspace = true } diff --git a/codex-rs/git-utils/src/info.rs b/codex-rs/git-utils/src/info.rs index e28a0bae42..e7642a8658 100644 --- a/codex-rs/git-utils/src/info.rs +++ b/codex-rs/git-utils/src/info.rs @@ -4,6 +4,7 @@ use std::ffi::OsStr; use std::path::Path; use std::path::PathBuf; +use codex_exec_server::ExecutorFileSystem; use codex_utils_absolute_path::AbsolutePathBuf; use futures::future::join_all; use schemars::JsonSchema; @@ -618,30 +619,38 @@ async fn diff_against_sha(cwd: &Path, sha: &GitSha) -> Option { /// `[get_git_repo_root]`, but resolves to the root of the main /// repository. Handles worktrees via filesystem inspection without invoking /// the `git` executable. -pub async fn resolve_root_git_project_for_trust(cwd: &Path) -> Option { - let base = if cwd.is_dir() { cwd } else { cwd.parent()? }; - let (repo_root, dot_git) = find_ancestor_git_entry(base)?; - if dot_git.is_dir() { - return Some(canonicalize_or_raw(repo_root)); +pub async fn resolve_root_git_project_for_trust( + fs: &dyn ExecutorFileSystem, + cwd: &AbsolutePathBuf, +) -> Option { + let base = match fs.get_metadata(cwd, /*sandbox*/ None).await { + Ok(metadata) if metadata.is_directory => cwd.clone(), + _ => cwd.parent()?, + }; + let (repo_root, dot_git) = find_ancestor_git_entry_with_fs(fs, &base).await?; + if fs + .get_metadata(&dot_git, /*sandbox*/ None) + .await + .ok()? + .is_directory + { + return Some(repo_root); } - let git_dir_s = std::fs::read_to_string(&dot_git).ok()?; + let git_dir_s = fs.read_file_text(&dot_git, /*sandbox*/ None).await.ok()?; let git_dir_rel = git_dir_s.trim().strip_prefix("gitdir:")?.trim(); if git_dir_rel.is_empty() { return None; } - let git_dir_path = canonicalize_or_raw( - AbsolutePathBuf::resolve_path_against_base(git_dir_rel, &repo_root).into_path_buf(), - ); + let git_dir_path = AbsolutePathBuf::resolve_path_against_base(git_dir_rel, repo_root.as_path()); let worktrees_dir = git_dir_path.parent()?; - if worktrees_dir.file_name() != Some(OsStr::new("worktrees")) { + if worktrees_dir.as_path().file_name() != Some(OsStr::new("worktrees")) { return None; } let common_dir = worktrees_dir.parent()?; - let main_repo_root = common_dir.parent()?; - Some(canonicalize_or_raw(main_repo_root.to_path_buf())) + common_dir.parent() } fn find_ancestor_git_entry(base_dir: &Path) -> Option<(PathBuf, PathBuf)> { @@ -663,8 +672,17 @@ fn find_ancestor_git_entry(base_dir: &Path) -> Option<(PathBuf, PathBuf)> { None } -fn canonicalize_or_raw(path: PathBuf) -> PathBuf { - std::fs::canonicalize(&path).unwrap_or(path) +async fn find_ancestor_git_entry_with_fs( + fs: &dyn ExecutorFileSystem, + base_dir: &AbsolutePathBuf, +) -> Option<(AbsolutePathBuf, AbsolutePathBuf)> { + for dir in base_dir.ancestors() { + let dot_git = dir.join(".git"); + if fs.get_metadata(&dot_git, /*sandbox*/ None).await.is_ok() { + return Some((dir, dot_git)); + } + } + None } /// Returns a list of local git branches. diff --git a/codex-rs/git-utils/src/lib.rs b/codex-rs/git-utils/src/lib.rs index 8f188e4bf0..d9d61d8650 100644 --- a/codex-rs/git-utils/src/lib.rs +++ b/codex-rs/git-utils/src/lib.rs @@ -1,6 +1,3 @@ -use std::fmt; -use std::path::PathBuf; - mod apply; mod branch; mod errors; @@ -16,6 +13,8 @@ pub use apply::extract_paths_from_patch; pub use apply::parse_git_apply_output; pub use apply::stage_paths; pub use branch::merge_base_with_head; +pub use codex_protocol::models::GhostCommit; +pub use codex_protocol::protocol::GitSha; pub use errors::GitToolingError; pub use ghost_commits::CreateGhostCommitOptions; pub use ghost_commits::GhostSnapshotConfig; @@ -45,72 +44,3 @@ pub use info::local_git_branches; pub use info::recent_commits; pub use info::resolve_root_git_project_for_trust; pub use platform::create_symlink; -use schemars::JsonSchema; -use serde::Deserialize; -use serde::Serialize; -use ts_rs::TS; - -type CommitID = String; - -#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, JsonSchema, TS)] -#[serde(transparent)] -#[ts(type = "string")] -pub struct GitSha(pub String); - -impl GitSha { - pub fn new(sha: &str) -> Self { - Self(sha.to_string()) - } -} - -/// Details of a ghost commit created from a repository state. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema, TS)] -pub struct GhostCommit { - id: CommitID, - parent: Option, - preexisting_untracked_files: Vec, - preexisting_untracked_dirs: Vec, -} - -impl GhostCommit { - /// Create a new ghost commit wrapper from a raw commit ID and optional parent. - pub fn new( - id: CommitID, - parent: Option, - preexisting_untracked_files: Vec, - preexisting_untracked_dirs: Vec, - ) -> Self { - Self { - id, - parent, - preexisting_untracked_files, - preexisting_untracked_dirs, - } - } - - /// Commit ID for the snapshot. - pub fn id(&self) -> &str { - &self.id - } - - /// Parent commit ID, if the repository had a `HEAD` at creation time. - pub fn parent(&self) -> Option<&str> { - self.parent.as_deref() - } - - /// Untracked or ignored files that already existed when the snapshot was captured. - pub fn preexisting_untracked_files(&self) -> &[PathBuf] { - &self.preexisting_untracked_files - } - - /// Untracked or ignored directories that already existed when the snapshot was captured. - pub fn preexisting_untracked_dirs(&self) -> &[PathBuf] { - &self.preexisting_untracked_dirs - } -} - -impl fmt::Display for GhostCommit { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.id) - } -} diff --git a/codex-rs/protocol/Cargo.toml b/codex-rs/protocol/Cargo.toml index f70dc94585..2bd46d7d5a 100644 --- a/codex-rs/protocol/Cargo.toml +++ b/codex-rs/protocol/Cargo.toml @@ -16,7 +16,6 @@ chardetng = { workspace = true } chrono = { workspace = true, features = ["serde"] } codex-async-utils = { workspace = true } codex-execpolicy = { workspace = true } -codex-git-utils = { workspace = true } codex-network-proxy = { workspace = true } codex-utils-absolute-path = { workspace = true } codex-utils-image = { workspace = true } diff --git a/codex-rs/protocol/src/models.rs b/codex-rs/protocol/src/models.rs index 2200456527..8c55d056cd 100644 --- a/codex-rs/protocol/src/models.rs +++ b/codex-rs/protocol/src/models.rs @@ -1,5 +1,7 @@ use std::collections::HashMap; +use std::fmt; use std::path::Path; +use std::path::PathBuf; use std::sync::LazyLock; use codex_utils_image::PromptImageMode; @@ -25,7 +27,6 @@ use crate::protocol::SandboxPolicy; use crate::protocol::WritableRoot; use crate::user_input::UserInput; use codex_execpolicy::Policy; -use codex_git_utils::GhostCommit; use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_image::ImageProcessingError; use schemars::JsonSchema; @@ -45,6 +46,60 @@ static SANDBOX_MODE_READ_ONLY_TEMPLATE: LazyLock