From 34aad436849963399e903d72bbf140935e9fc66f Mon Sep 17 00:00:00 2001 From: sayan-oai Date: Tue, 19 May 2026 23:47:48 -0700 Subject: [PATCH 01/17] add encryptedcontent to functioncalloutput (#23500) add new `EncryptedContent` variant to `FunctionCallOutputContentItem` ahead of standalone websearch. we need to be able to receive and pass encrypted function call output from the new web search endpoint back to responsesapi, as we cannot expose direct search results. --- .../schema/json/ClientRequest.json | 20 ++++++ .../codex_app_server_protocol.schemas.json | 20 ++++++ .../codex_app_server_protocol.v2.schemas.json | 20 ++++++ .../RawResponseItemCompletedNotification.json | 20 ++++++ .../schema/json/v2/ThreadResumeParams.json | 20 ++++++ .../FunctionCallOutputContentItem.ts | 2 +- codex-rs/core/src/context_manager/history.rs | 51 ++++++++++++--- .../core/src/context_manager/history_tests.rs | 20 ++++++ codex-rs/protocol/src/models.rs | 62 ++++++++++++++++++- codex-rs/tools/src/tool_output.rs | 3 +- codex-rs/utils/output-truncation/src/lib.rs | 13 +++- .../output-truncation/src/truncate_tests.rs | 54 ++++++++++++++++ 12 files changed, 291 insertions(+), 14 deletions(-) diff --git a/codex-rs/app-server-protocol/schema/json/ClientRequest.json b/codex-rs/app-server-protocol/schema/json/ClientRequest.json index ee68faf001..788e52e82b 100644 --- a/codex-rs/app-server-protocol/schema/json/ClientRequest.json +++ b/codex-rs/app-server-protocol/schema/json/ClientRequest.json @@ -984,6 +984,26 @@ ], "title": "InputImageFunctionCallOutputContentItem", "type": "object" + }, + { + "properties": { + "encrypted_content": { + "type": "string" + }, + "type": { + "enum": [ + "encrypted_content" + ], + "title": "EncryptedContentFunctionCallOutputContentItemType", + "type": "string" + } + }, + "required": [ + "encrypted_content", + "type" + ], + "title": "EncryptedContentFunctionCallOutputContentItem", + "type": "object" } ] }, diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index 3db4a87ef2..f6e5a46c95 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -9242,6 +9242,26 @@ ], "title": "InputImageFunctionCallOutputContentItem", "type": "object" + }, + { + "properties": { + "encrypted_content": { + "type": "string" + }, + "type": { + "enum": [ + "encrypted_content" + ], + "title": "EncryptedContentFunctionCallOutputContentItemType", + "type": "string" + } + }, + "required": [ + "encrypted_content", + "type" + ], + "title": "EncryptedContentFunctionCallOutputContentItem", + "type": "object" } ] }, diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index 7b62d6d4bf..c4e93338a1 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -5631,6 +5631,26 @@ ], "title": "InputImageFunctionCallOutputContentItem", "type": "object" + }, + { + "properties": { + "encrypted_content": { + "type": "string" + }, + "type": { + "enum": [ + "encrypted_content" + ], + "title": "EncryptedContentFunctionCallOutputContentItemType", + "type": "string" + } + }, + "required": [ + "encrypted_content", + "type" + ], + "title": "EncryptedContentFunctionCallOutputContentItem", + "type": "object" } ] }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json index 74420ea57e..e69b342282 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/RawResponseItemCompletedNotification.json @@ -140,6 +140,26 @@ ], "title": "InputImageFunctionCallOutputContentItem", "type": "object" + }, + { + "properties": { + "encrypted_content": { + "type": "string" + }, + "type": { + "enum": [ + "encrypted_content" + ], + "title": "EncryptedContentFunctionCallOutputContentItemType", + "type": "string" + } + }, + "required": [ + "encrypted_content", + "type" + ], + "title": "EncryptedContentFunctionCallOutputContentItem", + "type": "object" } ] }, diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json index 027989479a..d4bdeda0ed 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadResumeParams.json @@ -199,6 +199,26 @@ ], "title": "InputImageFunctionCallOutputContentItem", "type": "object" + }, + { + "properties": { + "encrypted_content": { + "type": "string" + }, + "type": { + "enum": [ + "encrypted_content" + ], + "title": "EncryptedContentFunctionCallOutputContentItemType", + "type": "string" + } + }, + "required": [ + "encrypted_content", + "type" + ], + "title": "EncryptedContentFunctionCallOutputContentItem", + "type": "object" } ] }, diff --git a/codex-rs/app-server-protocol/schema/typescript/FunctionCallOutputContentItem.ts b/codex-rs/app-server-protocol/schema/typescript/FunctionCallOutputContentItem.ts index fb2996f1e5..cd18908145 100644 --- a/codex-rs/app-server-protocol/schema/typescript/FunctionCallOutputContentItem.ts +++ b/codex-rs/app-server-protocol/schema/typescript/FunctionCallOutputContentItem.ts @@ -7,4 +7,4 @@ import type { ImageDetail } from "./ImageDetail"; * Responses API compatible content items that can be returned by a tool call. * This is a subset of ContentItem with the types we support as function call outputs. */ -export type FunctionCallOutputContentItem = { "type": "input_text", text: string, } | { "type": "input_image", image_url: string, detail?: ImageDetail, }; +export type FunctionCallOutputContentItem = { "type": "input_text", text: string, } | { "type": "input_image", image_url: string, detail?: ImageDetail, } | { "type": "encrypted_content", encrypted_content: string, }; diff --git a/codex-rs/core/src/context_manager/history.rs b/codex-rs/core/src/context_manager/history.rs index cfd9297382..6a05535488 100644 --- a/codex-rs/core/src/context_manager/history.rs +++ b/codex-rs/core/src/context_manager/history.rs @@ -504,6 +504,10 @@ fn estimate_reasoning_length(encoded_len: usize) -> usize { .saturating_sub(650) } +fn estimate_encrypted_function_output_length(encoded_len: usize) -> usize { + encoded_len.saturating_mul(9).div_ceil(16) +} + fn estimate_item_token_count(item: &ResponseItem) -> i64 { let model_visible_bytes = estimate_response_item_model_visible_bytes(item); approx_tokens_from_byte_count_i64(model_visible_bytes) @@ -547,16 +551,18 @@ pub(crate) fn estimate_response_item_model_visible_bytes(item: &ResponseItem) -> let raw = serde_json::to_string(item) .map(|serialized| i64::try_from(serialized.len()).unwrap_or(i64::MAX)) .unwrap_or_default(); - let (payload_bytes, replacement_bytes) = image_data_url_estimate_adjustment(item); - if payload_bytes == 0 || replacement_bytes == 0 { - raw - } else { - // Replace raw base64 payload bytes with a per-image estimate. - // We intentionally preserve the data URL prefix and JSON - // wrapper bytes already included in `raw`. - raw.saturating_sub(payload_bytes) - .saturating_add(replacement_bytes) - } + let (image_payload_bytes, image_replacement_bytes) = + image_data_url_estimate_adjustment(item); + let (encrypted_payload_bytes, encrypted_replacement_bytes) = + encrypted_function_output_estimate_adjustment(item); + // Replace raw base64 payload bytes with a per-image estimate. + // We intentionally preserve the data URL prefix and JSON + // wrapper bytes already included in `raw`. + let raw = raw + .saturating_sub(image_payload_bytes) + .saturating_add(image_replacement_bytes); + raw.saturating_sub(encrypted_payload_bytes) + .saturating_add(encrypted_replacement_bytes) } } } @@ -678,6 +684,31 @@ fn image_data_url_estimate_adjustment(item: &ResponseItem) -> (i64, i64) { (payload_bytes, replacement_bytes) } +fn encrypted_function_output_estimate_adjustment(item: &ResponseItem) -> (i64, i64) { + let ResponseItem::FunctionCallOutput { output, .. } = item else { + return (0, 0); + }; + let FunctionCallOutputBody::ContentItems(items) = &output.body else { + return (0, 0); + }; + + items.iter().fold((0i64, 0i64), |acc, item| { + let FunctionCallOutputContentItem::EncryptedContent { encrypted_content } = item else { + return acc; + }; + let payload_bytes = acc + .0 + .saturating_add(i64::try_from(encrypted_content.len()).unwrap_or(i64::MAX)); + let replacement_bytes = acc.1.saturating_add( + i64::try_from(estimate_encrypted_function_output_length( + encrypted_content.len(), + )) + .unwrap_or(i64::MAX), + ); + (payload_bytes, replacement_bytes) + }) +} + fn is_model_generated_item(item: &ResponseItem) -> bool { match item { ResponseItem::Message { role, .. } => role == "assistant", diff --git a/codex-rs/core/src/context_manager/history_tests.rs b/codex-rs/core/src/context_manager/history_tests.rs index 8ba5d56e32..f63ae6ec1f 100644 --- a/codex-rs/core/src/context_manager/history_tests.rs +++ b/codex-rs/core/src/context_manager/history_tests.rs @@ -1754,6 +1754,26 @@ fn non_base64_image_urls_are_unchanged() { ); } +#[test] +fn encrypted_function_output_uses_plaintext_byte_estimate() { + let encrypted_content = "A".repeat(1_868); + let item = ResponseItem::FunctionCallOutput { + call_id: "call-encrypted".to_string(), + output: FunctionCallOutputPayload::from_content_items(vec![ + FunctionCallOutputContentItem::EncryptedContent { + encrypted_content: encrypted_content.clone(), + }, + ]), + }; + + let raw_len = serde_json::to_string(&item).unwrap().len() as i64; + let estimated = estimate_response_item_model_visible_bytes(&item); + let expected = raw_len - encrypted_content.len() as i64 + + estimate_encrypted_function_output_length(encrypted_content.len()) as i64; + + assert_eq!(estimated, expected); +} + #[test] fn data_url_without_base64_marker_is_unchanged() { let item = ResponseItem::Message { diff --git a/codex-rs/protocol/src/models.rs b/codex-rs/protocol/src/models.rs index d1d2abe755..2927cb1cd1 100644 --- a/codex-rs/protocol/src/models.rs +++ b/codex-rs/protocol/src/models.rs @@ -1314,6 +1314,9 @@ pub enum FunctionCallOutputContentItem { #[ts(optional)] detail: Option, }, + EncryptedContent { + encrypted_content: String, + }, } /// Converts structured function-call output content into plain text for @@ -1337,7 +1340,8 @@ pub fn function_call_output_content_items_to_text( Some(text.as_str()) } FunctionCallOutputContentItem::InputText { .. } - | FunctionCallOutputContentItem::InputImage { .. } => None, + | FunctionCallOutputContentItem::InputImage { .. } + | FunctionCallOutputContentItem::EncryptedContent { .. } => None, }) .collect::>(); @@ -2063,6 +2067,9 @@ mod tests { image_url: "data:image/png;base64,AAA".to_string(), detail: Some(DEFAULT_IMAGE_DETAIL), }, + FunctionCallOutputContentItem::EncryptedContent { + encrypted_content: "enc_opaque".to_string(), + }, ]; let text = function_call_output_content_items_to_text(&content_items); @@ -2270,6 +2277,35 @@ mod tests { Ok(()) } + #[test] + fn serializes_encrypted_function_output_content_as_array() -> Result<()> { + let item = ResponseInputItem::FunctionCallOutput { + call_id: "call1".into(), + output: FunctionCallOutputPayload::from_content_items(vec![ + FunctionCallOutputContentItem::EncryptedContent { + encrypted_content: "enc_opaque".into(), + }, + ]), + }; + + let json = serde_json::to_value(&item)?; + assert_eq!( + json, + serde_json::json!({ + "type": "function_call_output", + "call_id": "call1", + "output": [ + { + "type": "encrypted_content", + "encrypted_content": "enc_opaque", + } + ], + }) + ); + + Ok(()) + } + #[test] fn preserves_existing_image_data_urls() -> Result<()> { let call_tool_result = CallToolResult { @@ -2394,6 +2430,30 @@ mod tests { Ok(()) } + #[test] + fn deserializes_encrypted_array_payload_into_items() -> Result<()> { + let json = r#"[ + {"type": "encrypted_content", "encrypted_content": "enc_opaque"} + ]"#; + + let payload: FunctionCallOutputPayload = serde_json::from_str(json)?; + let expected_items = vec![FunctionCallOutputContentItem::EncryptedContent { + encrypted_content: "enc_opaque".into(), + }]; + + assert_eq!(payload.success, None); + assert_eq!( + payload.body, + FunctionCallOutputBody::ContentItems(expected_items.clone()) + ); + assert_eq!( + serde_json::to_string(&payload)?, + serde_json::to_string(&expected_items)? + ); + + Ok(()) + } + #[test] fn deserializes_compaction_alias() -> Result<()> { let json = r#"{"type":"compaction_summary","encrypted_content":"abc"}"#; diff --git a/codex-rs/tools/src/tool_output.rs b/codex-rs/tools/src/tool_output.rs index 64a8164da3..b61bb80de8 100644 --- a/codex-rs/tools/src/tool_output.rs +++ b/codex-rs/tools/src/tool_output.rs @@ -209,7 +209,8 @@ fn content_items_to_code_mode_result(items: &[FunctionCallOutputContentItem]) -> Some(image_url.clone()) } FunctionCallOutputContentItem::InputText { .. } - | FunctionCallOutputContentItem::InputImage { .. } => None, + | FunctionCallOutputContentItem::InputImage { .. } + | FunctionCallOutputContentItem::EncryptedContent { .. } => None, }) .collect::>() .join("\n"), diff --git a/codex-rs/utils/output-truncation/src/lib.rs b/codex-rs/utils/output-truncation/src/lib.rs index 24b1630da1..52cd741b7d 100644 --- a/codex-rs/utils/output-truncation/src/lib.rs +++ b/codex-rs/utils/output-truncation/src/lib.rs @@ -34,7 +34,8 @@ pub fn formatted_truncate_text_content_items_with_policy( .iter() .filter_map(|item| match item { FunctionCallOutputContentItem::InputText { text } => Some(text.as_str()), - FunctionCallOutputContentItem::InputImage { .. } => None, + FunctionCallOutputContentItem::InputImage { .. } + | FunctionCallOutputContentItem::EncryptedContent { .. } => None, }) .collect::>(); @@ -64,6 +65,11 @@ pub fn formatted_truncate_text_content_items_with_policy( detail: *detail, }) } + FunctionCallOutputContentItem::EncryptedContent { encrypted_content } => { + Some(FunctionCallOutputContentItem::EncryptedContent { + encrypted_content: encrypted_content.clone(), + }) + } FunctionCallOutputContentItem::InputText { .. } => None, })); @@ -117,6 +123,11 @@ pub fn truncate_function_output_items_with_policy( detail: *detail, }); } + FunctionCallOutputContentItem::EncryptedContent { encrypted_content } => { + out.push(FunctionCallOutputContentItem::EncryptedContent { + encrypted_content: encrypted_content.clone(), + }); + } } } diff --git a/codex-rs/utils/output-truncation/src/truncate_tests.rs b/codex-rs/utils/output-truncation/src/truncate_tests.rs index 74acb15ca3..baf26a058e 100644 --- a/codex-rs/utils/output-truncation/src/truncate_tests.rs +++ b/codex-rs/utils/output-truncation/src/truncate_tests.rs @@ -251,6 +251,60 @@ fn formatted_truncate_text_content_items_with_policy_merges_text_and_appends_ima assert_eq!(original_token_count, Some(4)); } +#[test] +fn formatted_truncate_text_content_items_with_policy_preserves_encrypted_content() { + let items = vec![ + FunctionCallOutputContentItem::InputText { + text: "abcdefgh".to_string(), + }, + FunctionCallOutputContentItem::EncryptedContent { + encrypted_content: "enc_opaque".to_string(), + }, + ]; + + let (output, original_token_count) = + formatted_truncate_text_content_items_with_policy(&items, TruncationPolicy::Bytes(2)); + + assert_eq!( + output, + vec![ + FunctionCallOutputContentItem::InputText { + text: "Total output lines: 1\n\na…6 chars truncated…h".to_string(), + }, + FunctionCallOutputContentItem::EncryptedContent { + encrypted_content: "enc_opaque".to_string(), + }, + ] + ); + assert_eq!(original_token_count, Some(2)); +} + +#[test] +fn truncate_function_output_items_with_policy_preserves_encrypted_content() { + let items = vec![ + FunctionCallOutputContentItem::InputText { + text: "abcdefgh".to_string(), + }, + FunctionCallOutputContentItem::EncryptedContent { + encrypted_content: "enc_opaque".to_string(), + }, + ]; + + let output = truncate_function_output_items_with_policy(&items, TruncationPolicy::Bytes(2)); + + assert_eq!( + output, + vec![ + FunctionCallOutputContentItem::InputText { + text: "a…6 chars truncated…h".to_string(), + }, + FunctionCallOutputContentItem::EncryptedContent { + encrypted_content: "enc_opaque".to_string(), + }, + ] + ); +} + #[test] fn formatted_truncate_text_content_items_with_policy_merges_all_text_for_token_budget() { let items = vec![ From 000bf5ce6d0afd3155a6e2315e28799ec73f8679 Mon Sep 17 00:00:00 2001 From: richardopenai Date: Wed, 20 May 2026 00:25:04 -0700 Subject: [PATCH 02/17] Migrate exec-server remote registration to environments (#23633) ## Summary - migrate exec-server remote registration naming from executor to environment - align CLI, public Rust exports, registry error messages, and relay test fixtures with the environment registry contract - keep the live registration path and response model consistent with `/cloud/environment/{environment_id}/register` ## Verification - `cargo test -p codex-exec-server remote::tests::register_environment_posts_with_auth_provider_headers --manifest-path /Users/richardlee/code/codex/codex-rs/Cargo.toml` - `cargo test -p codex-exec-server --test relay multiplexed_remote_environment_routes_independent_virtual_streams --manifest-path /Users/richardlee/code/codex/codex-rs/Cargo.toml` - `cargo check -p codex-cli --manifest-path /Users/richardlee/code/codex/codex-rs/Cargo.toml` (still running when PR opened; will update after completion if needed) --- codex-rs/cli/src/main.rs | 27 +++--- codex-rs/core/tests/suite/rmcp_client.rs | 2 +- codex-rs/exec-server/README.md | 16 ++-- codex-rs/exec-server/src/client.rs | 24 ++--- codex-rs/exec-server/src/lib.rs | 4 +- codex-rs/exec-server/src/process.rs | 2 +- codex-rs/exec-server/src/relay.rs | 4 +- codex-rs/exec-server/src/remote.rs | 112 +++++++++++------------ codex-rs/exec-server/tests/relay.rs | 32 ++++--- codex-rs/file-system/src/lib.rs | 2 +- 10 files changed, 115 insertions(+), 110 deletions(-) diff --git a/codex-rs/cli/src/main.rs b/codex-rs/cli/src/main.rs index 1b71ab15ae..f4127c437c 100644 --- a/codex-rs/cli/src/main.rs +++ b/codex-rs/cli/src/main.rs @@ -482,15 +482,15 @@ struct ExecServerCommand { #[arg(long = "listen", value_name = "URL", conflicts_with = "remote")] listen: Option, - /// Register this exec-server as a remote executor using the given base URL. - #[arg(long = "remote", value_name = "URL", requires = "executor_id")] + /// Register this exec-server as a remote environment using the given base URL. + #[arg(long = "remote", value_name = "URL", requires = "environment_id")] remote: Option, - /// Executor id to attach to when registering remotely. - #[arg(long = "executor-id", value_name = "ID")] - executor_id: Option, + /// Environment id to attach to when registering remotely. + #[arg(long = "environment-id", value_name = "ID")] + environment_id: Option, - /// Human-readable executor name. + /// Human-readable environment name. #[arg(long = "name", value_name = "NAME")] name: Option, @@ -1491,21 +1491,24 @@ async fn run_exec_server_command( arg0_paths.codex_linux_sandbox_exe.clone(), )?; if let Some(base_url) = cmd.remote { - let executor_id = cmd - .executor_id - .ok_or_else(|| anyhow::anyhow!("--executor-id is required when --remote is set"))?; + let environment_id = cmd + .environment_id + .ok_or_else(|| anyhow::anyhow!("--environment-id is required when --remote is set"))?; let auth_provider = load_exec_server_remote_auth_provider( root_config_overrides, config_profile, cmd.use_agent_identity_auth, ) .await?; - let mut remote_config = - codex_exec_server::RemoteExecutorConfig::new(base_url, executor_id, auth_provider)?; + let mut remote_config = codex_exec_server::RemoteEnvironmentConfig::new( + base_url, + environment_id, + auth_provider, + )?; if let Some(name) = cmd.name { remote_config.name = name; } - codex_exec_server::run_remote_executor(remote_config, runtime_paths).await?; + codex_exec_server::run_remote_environment(remote_config, runtime_paths).await?; return Ok(()); } let listen_url = cmd diff --git a/codex-rs/core/tests/suite/rmcp_client.rs b/codex-rs/core/tests/suite/rmcp_client.rs index 7c811a88e0..210c287c2f 100644 --- a/codex-rs/core/tests/suite/rmcp_client.rs +++ b/codex-rs/core/tests/suite/rmcp_client.rs @@ -1808,7 +1808,7 @@ async fn streamable_http_tool_call_round_trip() -> anyhow::Result<()> { .await; // Phase 2: start the Streamable HTTP MCP test server in the active - // placement. In full CI this may be the remote executor container; locally + // placement. In full CI this may be the remote environment container; locally // it is a host process. let expected_env_value = "propagated-env-http"; let Some(http_server) = diff --git a/codex-rs/exec-server/README.md b/codex-rs/exec-server/README.md index 01439108f8..468ac205f7 100644 --- a/codex-rs/exec-server/README.md +++ b/codex-rs/exec-server/README.md @@ -22,10 +22,10 @@ the wire. The CLI entrypoint supports: - `ws://IP:PORT` (default) -- `--remote URL --executor-id ID [--name NAME]` +- `--remote URL --environment-id ID [--name NAME]` -Remote mode registers the local exec-server with the executor registry, -then reconnects to the service-provided rendezvous websocket as the executor. +Remote mode registers the local exec-server with the environment registry, +then reconnects to the service-provided rendezvous websocket as the environment. It uses the standard Codex ChatGPT sign-in state; run `codex login` first when remote registration needs authentication. Containerized callers that receive an Agent Identity JWT in `CODEX_ACCESS_TOKEN` can opt into that auth path with @@ -39,7 +39,7 @@ Wire framing: ## Remote Relay Message Format -In remote mode, the harness and executor communicate through rendezvous using +In remote mode, the harness and environment communicate through rendezvous using `codex.exec_server.relay.v1.RelayMessageFrame`; the checked-in schema is in `src/proto/codex.exec_server.relay.v1.proto`. The relay frame carries stream identity plus endpoint-owned reliability metadata: @@ -58,8 +58,8 @@ next_seq // resume only: next sender seq reason // reset only: reset reason ``` -`stream_id` identifies one virtual harness/executor JSON-RPC session on the -executor websocket. The harness generates a UUIDv4 `stream_id`; the executor +`stream_id` identifies one virtual harness/environment JSON-RPC session on the +environment websocket. The harness generates a UUIDv4 `stream_id`; the environment demuxes frames by `stream_id` and runs an independent `ConnectionProcessor` per stream. @@ -375,12 +375,12 @@ The crate exports: - `DEFAULT_LISTEN_URL` and `ExecServerListenUrlParseError` - `ExecServerRuntimePaths` - `run_main()` for embedding the websocket server -- `RemoteExecutorConfig` and `run_remote_executor()` for embedding remote +- `RemoteEnvironmentConfig` and `run_remote_environment()` for embedding remote registration mode Callers must pass `ExecServerRuntimePaths` to `run_main()`. The top-level `codex exec-server` command builds these paths from the `codex` arg0 dispatch -state. `RemoteExecutorConfig::new(...)` also takes the auth provider that +state. `RemoteEnvironmentConfig::new(...)` also takes the auth provider that remote registration should use; the CLI builds that provider from Codex auth state before starting remote mode. diff --git a/codex-rs/exec-server/src/client.rs b/codex-rs/exec-server/src/client.rs index 9261a59542..bb5a3855dc 100644 --- a/codex-rs/exec-server/src/client.rs +++ b/codex-rs/exec-server/src/client.rs @@ -162,7 +162,7 @@ struct Inner { // need serialization so concurrent register/remove operations do not // overwrite each other's copy-on-write updates. sessions_write_lock: Mutex<()>, - // Once the transport closes, every executor operation should fail quickly + // Once the transport closes, every environment operation should fail quickly // with the same canonical message. This client never reconnects, so the // latch only moves from unset to set once. disconnected: OnceLock, @@ -261,18 +261,18 @@ pub enum ExecServerError { Protocol(String), #[error("exec-server rejected request ({code}): {message}")] Server { code: i64, message: String }, - #[error("executor registry request failed ({status}{code_suffix}): {message}", code_suffix = .code.as_ref().map(|code| format!(", {code}")).unwrap_or_default())] - ExecutorRegistryHttp { + #[error("environment registry request failed ({status}{code_suffix}): {message}", code_suffix = .code.as_ref().map(|code| format!(", {code}")).unwrap_or_default())] + EnvironmentRegistryHttp { status: reqwest::StatusCode, code: Option, message: String, }, - #[error("executor registry configuration error: {0}")] - ExecutorRegistryConfig(String), - #[error("executor registry authentication error: {0}")] - ExecutorRegistryAuth(String), - #[error("executor registry request failed: {0}")] - ExecutorRegistryRequest(#[from] reqwest::Error), + #[error("environment registry configuration error: {0}")] + EnvironmentRegistryConfig(String), + #[error("environment registry authentication error: {0}")] + EnvironmentRegistryAuth(String), + #[error("environment registry request failed: {0}")] + EnvironmentRegistryRequest(#[from] reqwest::Error), } impl ExecServerClient { @@ -495,7 +495,7 @@ impl ExecServerClient { { // Reject new work before allocating a JSON-RPC request id. MCP tool // calls, process writes, and fs operations all pass through here, so - // this is the shared low-level failure path after executor disconnect. + // this is the shared low-level failure path after environment disconnect. if let Some(error) = self.inner.disconnected_error() { return Err(error); } @@ -720,7 +720,7 @@ impl Inner { session: Arc, ) -> Result<(), ExecServerError> { let _sessions_write_guard = self.sessions_write_lock.lock().await; - // Do not register a process session that can never receive executor + // Do not register a process session that can never receive environment // notifications. Without this check, remote MCP startup could create a // dead session and wait for process output that will never arrive. if let Some(error) = self.disconnected_error() { @@ -795,7 +795,7 @@ async fn fail_all_sessions(inner: &Arc, message: String) { for (_, session) in sessions { // Sessions synthesize a closed read response and emit a pushed Failed // event. That covers both polling consumers and streaming consumers - // such as executor-backed MCP stdio. + // such as environment-backed MCP stdio. session.set_failure(message.clone()).await; } } diff --git a/codex-rs/exec-server/src/lib.rs b/codex-rs/exec-server/src/lib.rs index bd556638ea..f2d16f8fac 100644 --- a/codex-rs/exec-server/src/lib.rs +++ b/codex-rs/exec-server/src/lib.rs @@ -91,8 +91,8 @@ pub use protocol::TerminateResponse; pub use protocol::WriteParams; pub use protocol::WriteResponse; pub use protocol::WriteStatus; -pub use remote::RemoteExecutorConfig; -pub use remote::run_remote_executor; +pub use remote::RemoteEnvironmentConfig; +pub use remote::run_remote_environment; pub use runtime_paths::ExecServerRuntimePaths; pub use server::DEFAULT_LISTEN_URL; pub use server::ExecServerListenUrlParseError; diff --git a/codex-rs/exec-server/src/process.rs b/codex-rs/exec-server/src/process.rs index 052e4aa643..cb6c832138 100644 --- a/codex-rs/exec-server/src/process.rs +++ b/codex-rs/exec-server/src/process.rs @@ -24,7 +24,7 @@ pub struct StartedExecProcess { /// stdout, stderr, or pty bytes. `Exited` reports the process exit status, while /// `Closed` means all output streams have ended and no more output events will /// arrive. `Failed` is used when the process session cannot continue, for -/// example because the remote executor connection disconnected. +/// example because the remote environment connection disconnected. #[derive(Debug, Clone, PartialEq, Eq)] pub enum ExecProcessEvent { Output(ProcessOutputChunk), diff --git a/codex-rs/exec-server/src/relay.rs b/codex-rs/exec-server/src/relay.rs index b3708f1d83..ae07d7652c 100644 --- a/codex-rs/exec-server/src/relay.rs +++ b/codex-rs/exec-server/src/relay.rs @@ -298,7 +298,7 @@ where } } -pub(crate) async fn run_multiplexed_executor( +pub(crate) async fn run_multiplexed_environment( stream: WebSocketStream, processor: ConnectionProcessor, ) where @@ -337,7 +337,7 @@ pub(crate) async fn run_multiplexed_executor( continue; } Some(Err(err)) => { - debug!("multiplexed executor websocket read failed: {err}"); + debug!("multiplexed environment websocket read failed: {err}"); break; } } diff --git a/codex-rs/exec-server/src/remote.rs b/codex-rs/exec-server/src/remote.rs index 2493135c9a..de09d94e7c 100644 --- a/codex-rs/exec-server/src/remote.rs +++ b/codex-rs/exec-server/src/remote.rs @@ -11,28 +11,28 @@ use codex_utils_rustls_provider::ensure_rustls_crypto_provider; use crate::ExecServerError; use crate::ExecServerRuntimePaths; -use crate::relay::run_multiplexed_executor; +use crate::relay::run_multiplexed_environment; use crate::server::ConnectionProcessor; const ERROR_BODY_PREVIEW_BYTES: usize = 4096; #[derive(Clone)] -struct ExecutorRegistryClient { +struct EnvironmentRegistryClient { base_url: String, auth_provider: SharedAuthProvider, http: reqwest::Client, } -impl std::fmt::Debug for ExecutorRegistryClient { +impl std::fmt::Debug for EnvironmentRegistryClient { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("ExecutorRegistryClient") + f.debug_struct("EnvironmentRegistryClient") .field("base_url", &self.base_url) .field("auth_provider", &"") .finish_non_exhaustive() } } -impl ExecutorRegistryClient { +impl EnvironmentRegistryClient { fn new(base_url: String, auth_provider: SharedAuthProvider) -> Result { let base_url = normalize_base_url(base_url)?; Ok(Self { @@ -42,15 +42,15 @@ impl ExecutorRegistryClient { }) } - async fn register_executor( + async fn register_environment( &self, - executor_id: &str, - ) -> Result { + environment_id: &str, + ) -> Result { let response = self .http .post(endpoint_url( &self.base_url, - &format!("/cloud/executor/{executor_id}/register"), + &format!("/cloud/environment/{environment_id}/register"), )) .headers(self.auth_provider.to_auth_headers()) .send() @@ -72,49 +72,49 @@ impl ExecutorRegistryClient { let status = response.status(); let body = response.text().await.unwrap_or_default(); if matches!(status, StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN) { - return Err(executor_registry_auth_error(status, &body)); + return Err(environment_registry_auth_error(status, &body)); } - Err(executor_registry_http_error(status, &body)) + Err(environment_registry_http_error(status, &body)) } } #[derive(Debug, Clone, Eq, PartialEq, Deserialize)] -struct ExecutorRegistryExecutorRegistrationResponse { - executor_id: String, +struct EnvironmentRegistryRegistrationResponse { + environment_id: String, url: String, } /// Configuration for registering an exec-server for remote use. #[derive(Clone)] -pub struct RemoteExecutorConfig { +pub struct RemoteEnvironmentConfig { pub base_url: String, - pub executor_id: String, + pub environment_id: String, pub name: String, auth_provider: SharedAuthProvider, } -impl std::fmt::Debug for RemoteExecutorConfig { +impl std::fmt::Debug for RemoteEnvironmentConfig { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("RemoteExecutorConfig") + f.debug_struct("RemoteEnvironmentConfig") .field("base_url", &self.base_url) - .field("executor_id", &self.executor_id) + .field("environment_id", &self.environment_id) .field("name", &self.name) .field("auth_provider", &"") .finish() } } -impl RemoteExecutorConfig { +impl RemoteEnvironmentConfig { pub fn new( base_url: String, - executor_id: String, + environment_id: String, auth_provider: SharedAuthProvider, ) -> Result { - let executor_id = normalize_executor_id(executor_id)?; + let environment_id = normalize_environment_id(environment_id)?; Ok(Self { base_url, - executor_id, + environment_id, name: "codex-exec-server".to_string(), auth_provider, }) @@ -123,27 +123,27 @@ impl RemoteExecutorConfig { /// Register an exec-server for remote use and serve requests over the returned /// rendezvous websocket. -pub async fn run_remote_executor( - config: RemoteExecutorConfig, +pub async fn run_remote_environment( + config: RemoteEnvironmentConfig, runtime_paths: ExecServerRuntimePaths, ) -> Result<(), ExecServerError> { ensure_rustls_crypto_provider(); let client = - ExecutorRegistryClient::new(config.base_url.clone(), config.auth_provider.clone())?; + EnvironmentRegistryClient::new(config.base_url.clone(), config.auth_provider.clone())?; let processor = ConnectionProcessor::new(runtime_paths); let mut backoff = Duration::from_secs(1); loop { - let response = client.register_executor(&config.executor_id).await?; + let response = client.register_environment(&config.environment_id).await?; eprintln!( - "codex exec-server remote executor registered with executor_id {}", - response.executor_id + "codex exec-server remote environment registered with environment_id {}", + response.environment_id ); match connect_async(response.url.as_str()).await { Ok((websocket, _)) => { backoff = Duration::from_secs(1); - run_multiplexed_executor(websocket, processor.clone()).await; + run_multiplexed_environment(websocket, processor.clone()).await; } Err(err) => { warn!("failed to connect remote exec-server websocket: {err}"); @@ -155,14 +155,14 @@ pub async fn run_remote_executor( } } -fn normalize_executor_id(executor_id: String) -> Result { - let executor_id = executor_id.trim().to_string(); - if executor_id.is_empty() { - return Err(ExecServerError::ExecutorRegistryConfig( - "executor id is required for remote exec-server registration".to_string(), +fn normalize_environment_id(environment_id: String) -> Result { + let environment_id = environment_id.trim().to_string(); + if environment_id.is_empty() { + return Err(ExecServerError::EnvironmentRegistryConfig( + "environment id is required for remote exec-server registration".to_string(), )); } - Ok(executor_id) + Ok(environment_id) } #[derive(Deserialize)] @@ -179,8 +179,8 @@ struct RegistryError { fn normalize_base_url(base_url: String) -> Result { let trimmed = base_url.trim().trim_end_matches('/').to_string(); if trimmed.is_empty() { - return Err(ExecServerError::ExecutorRegistryConfig( - "executor registry base URL is required".to_string(), + return Err(ExecServerError::EnvironmentRegistryConfig( + "environment registry base URL is required".to_string(), )); } Ok(trimmed) @@ -190,14 +190,14 @@ fn endpoint_url(base_url: &str, path: &str) -> String { format!("{base_url}/{}", path.trim_start_matches('/')) } -fn executor_registry_auth_error(status: StatusCode, body: &str) -> ExecServerError { +fn environment_registry_auth_error(status: StatusCode, body: &str) -> ExecServerError { let message = registry_error_message(body).unwrap_or_else(|| "empty error body".to_string()); - ExecServerError::ExecutorRegistryAuth(format!( - "executor registry authentication failed ({status}): {message}" + ExecServerError::EnvironmentRegistryAuth(format!( + "environment registry authentication failed ({status}): {message}" )) } -fn executor_registry_http_error(status: StatusCode, body: &str) -> ExecServerError { +fn environment_registry_http_error(status: StatusCode, body: &str) -> ExecServerError { let parsed = serde_json::from_str::(body).ok(); let (code, message) = parsed .and_then(|body| body.error) @@ -216,7 +216,7 @@ fn executor_registry_http_error(status: StatusCode, body: &str) -> ExecServerErr .unwrap_or_else(|| "empty or malformed error body".to_string()), ) }); - ExecServerError::ExecutorRegistryHttp { + ExecServerError::EnvironmentRegistryHttp { status, code, message, @@ -277,46 +277,46 @@ mod tests { } #[tokio::test] - async fn register_executor_posts_with_auth_provider_headers() { + async fn register_environment_posts_with_auth_provider_headers() { let server = MockServer::start().await; - let config = RemoteExecutorConfig::new( + let config = RemoteEnvironmentConfig::new( server.uri(), - "exec-requested".to_string(), + "environment-requested".to_string(), static_registry_auth_provider(), ) .expect("config"); Mock::given(method("POST")) - .and(path("/cloud/executor/exec-requested/register")) + .and(path("/cloud/environment/environment-requested/register")) .and(header("authorization", "Bearer registry-token")) .and(header("chatgpt-account-id", "workspace-123")) .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ - "executor_id": "exec-1", - "url": "wss://rendezvous.test/executor/exec-1?role=executor&sig=abc" + "environment_id": "env-1", + "url": "wss://rendezvous.test/cloud-agent/default/ws/environment/env-1?role=environment&sig=abc" }))) .mount(&server) .await; - let client = ExecutorRegistryClient::new(server.uri(), static_registry_auth_provider()) + let client = EnvironmentRegistryClient::new(server.uri(), static_registry_auth_provider()) .expect("client"); let response = client - .register_executor(&config.executor_id) + .register_environment(&config.environment_id) .await - .expect("register executor"); + .expect("register environment"); assert_eq!( response, - ExecutorRegistryExecutorRegistrationResponse { - executor_id: "exec-1".to_string(), - url: "wss://rendezvous.test/executor/exec-1?role=executor&sig=abc".to_string(), + EnvironmentRegistryRegistrationResponse { + environment_id: "env-1".to_string(), + url: "wss://rendezvous.test/cloud-agent/default/ws/environment/env-1?role=environment&sig=abc".to_string(), } ); } #[test] fn debug_output_redacts_auth_provider() { - let config = RemoteExecutorConfig::new( + let config = RemoteEnvironmentConfig::new( "https://registry.example".to_string(), - "exec-1".to_string(), + "env-1".to_string(), static_registry_auth_provider(), ) .expect("config"); diff --git a/codex-rs/exec-server/tests/relay.rs b/codex-rs/exec-server/tests/relay.rs index 9f985e6982..ff41dd525e 100644 --- a/codex-rs/exec-server/tests/relay.rs +++ b/codex-rs/exec-server/tests/relay.rs @@ -20,7 +20,7 @@ use codex_app_server_protocol::RequestId; use codex_exec_server::ExecServerRuntimePaths; use codex_exec_server::InitializeParams; use codex_exec_server::InitializeResponse; -use codex_exec_server::RemoteExecutorConfig; +use codex_exec_server::RemoteEnvironmentConfig; use futures::SinkExt; use futures::StreamExt; use http::HeaderMap; @@ -45,7 +45,7 @@ use wiremock::matchers::header; use wiremock::matchers::method; use wiremock::matchers::path; -const EXECUTOR_ID: &str = "exec-mux-test"; +const ENVIRONMENT_ID: &str = "env-mux-test"; const REGISTRY_TOKEN: &str = "registry-token"; const RELAY_MESSAGE_FRAME_VERSION: u32 = 1; const TEST_TIMEOUT: Duration = Duration::from_secs(5); @@ -67,15 +67,17 @@ fn static_registry_auth_provider() -> codex_api::SharedAuthProvider { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn multiplexed_remote_executor_routes_independent_virtual_streams() -> Result<()> { +async fn multiplexed_remote_environment_routes_independent_virtual_streams() -> Result<()> { let listener = TcpListener::bind("127.0.0.1:0").await?; let rendezvous_url = format!("ws://{}", listener.local_addr()?); let registry = MockServer::start().await; Mock::given(method("POST")) - .and(path(format!("/cloud/executor/{EXECUTOR_ID}/register"))) + .and(path(format!( + "/cloud/environment/{ENVIRONMENT_ID}/register" + ))) .and(header("authorization", format!("Bearer {REGISTRY_TOKEN}"))) .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ - "executor_id": EXECUTOR_ID, + "environment_id": ENVIRONMENT_ID, "url": rendezvous_url, }))) .mount(®istry) @@ -83,22 +85,22 @@ async fn multiplexed_remote_executor_routes_independent_virtual_streams() -> Res let (codex_exe, codex_linux_sandbox_exe) = common::current_test_binary_helper_paths()?; let runtime_paths = ExecServerRuntimePaths::new(codex_exe, codex_linux_sandbox_exe)?; - let config = RemoteExecutorConfig::new( + let config = RemoteEnvironmentConfig::new( registry.uri(), - EXECUTOR_ID.to_string(), + ENVIRONMENT_ID.to_string(), static_registry_auth_provider(), )?; - let remote_executor = tokio::spawn(codex_exec_server::run_remote_executor( + let remote_environment = tokio::spawn(codex_exec_server::run_remote_environment( config, runtime_paths, )); let (socket, _peer_addr) = timeout(TEST_TIMEOUT, listener.accept()) .await - .context("remote executor should connect to fake rendezvous")??; + .context("remote environment should connect to fake rendezvous")??; let mut websocket = timeout(TEST_TIMEOUT, accept_async(socket)) .await - .context("fake rendezvous should accept executor websocket")??; + .context("fake rendezvous should accept environment websocket")??; let stream_a = "stream-a"; let stream_b = "stream-b"; @@ -192,8 +194,8 @@ async fn multiplexed_remote_executor_routes_independent_virtual_streams() -> Res )?; websocket.close(None).await?; - remote_executor.abort(); - let _ = remote_executor.await; + remote_environment.abort(); + let _ = remote_environment.await; Ok(()) } @@ -285,7 +287,7 @@ where let frame = timeout(TEST_TIMEOUT, websocket.next()) .await .context("timed out waiting for relay frame")? - .ok_or_else(|| anyhow!("executor websocket closed"))??; + .ok_or_else(|| anyhow!("environment websocket closed"))??; match frame { Message::Binary(bytes) => { let frame = RelayMessageFrame::decode(bytes.as_ref())?; @@ -297,8 +299,8 @@ where return Ok((stream_id, message)); } Message::Ping(_) | Message::Pong(_) => {} - Message::Close(_) => bail!("executor websocket closed"), - Message::Text(_) => bail!("executor sent text frame on relay websocket"), + Message::Close(_) => bail!("environment websocket closed"), + Message::Text(_) => bail!("environment sent text frame on relay websocket"), Message::Frame(_) => {} } } diff --git a/codex-rs/file-system/src/lib.rs b/codex-rs/file-system/src/lib.rs index 026523eb2f..606f987cbd 100644 --- a/codex-rs/file-system/src/lib.rs +++ b/codex-rs/file-system/src/lib.rs @@ -130,7 +130,7 @@ fn file_system_policy_has_cwd_dependent_entries( pub type FileSystemResult = io::Result; /// Abstract filesystem access used by components that may operate locally or via -/// a remote executor. +/// a remote environment. #[async_trait] pub trait ExecutorFileSystem: Send + Sync { async fn read_file( From 18cefba922ab17fda8783ac7c61c9b96798439cc Mon Sep 17 00:00:00 2001 From: jif-oai Date: Wed, 20 May 2026 11:56:00 +0200 Subject: [PATCH 03/17] Add timeout for remote compaction requests (#23451) ## Why Remote compaction currently sends a unary `POST /responses/compact` and waits for the full response before replacing history or emitting the completed `ContextCompaction` item. Unlike normal `/responses` streaming requests, this unary compact request had no timeout boundary. If the backend accepts the request and then stalls before returning a body, the existing request retry policy never sees a transport error, so the compact turn can remain stuck after the started item with no completion or actionable error. That matches the reported hang shape in issues such as #18363, where logs show `responses/compact` was posted but no corresponding compact completion followed. A bounded request timeout gives the existing retry policy a concrete timeout error to retry instead of letting the user sit indefinitely on automatic context compaction. ## What - Add a request timeout to legacy `/responses/compact` calls. - Size that timeout from the provider stream idle timeout with a conservative multiplier, so the default compact attempt gets 20 minutes rather than the 5 minute stream idle window. - Map API transport timeouts to a request timeout error instead of the child-process timeout message. ## Testing - Not run (per request; CI will cover). --- codex-rs/codex-api/src/api_bridge.rs | 2 +- codex-rs/codex-api/src/endpoint/compact.rs | 15 +++++++++++++-- codex-rs/core/src/client.rs | 9 ++++++++- codex-rs/protocol/src/error.rs | 3 +++ 4 files changed, 25 insertions(+), 4 deletions(-) diff --git a/codex-rs/codex-api/src/api_bridge.rs b/codex-rs/codex-api/src/api_bridge.rs index 4f37c2ff9b..401dfab3a9 100644 --- a/codex-rs/codex-api/src/api_bridge.rs +++ b/codex-rs/codex-api/src/api_bridge.rs @@ -123,7 +123,7 @@ pub fn map_api_error(err: ApiError) -> CodexErr { status: http::StatusCode::INTERNAL_SERVER_ERROR, request_id: None, }), - TransportError::Timeout => CodexErr::Timeout, + TransportError::Timeout => CodexErr::RequestTimeout, TransportError::Network(msg) | TransportError::Build(msg) => { CodexErr::Stream(msg, None) } diff --git a/codex-rs/codex-api/src/endpoint/compact.rs b/codex-rs/codex-api/src/endpoint/compact.rs index 336e958d4e..a3da854844 100644 --- a/codex-rs/codex-api/src/endpoint/compact.rs +++ b/codex-rs/codex-api/src/endpoint/compact.rs @@ -11,6 +11,7 @@ use http::Method; use serde::Deserialize; use serde_json::to_value; use std::sync::Arc; +use std::time::Duration; pub struct CompactClient { session: EndpointSession, @@ -37,10 +38,19 @@ impl CompactClient { &self, body: serde_json::Value, extra_headers: HeaderMap, + request_timeout: Duration, ) -> Result, ApiError> { let resp = self .session - .execute(Method::POST, Self::path(), extra_headers, Some(body)) + .execute_with( + Method::POST, + Self::path(), + extra_headers, + Some(body), + |req| { + req.timeout = Some(request_timeout); + }, + ) .await?; let parsed: CompactHistoryResponse = serde_json::from_slice(&resp.body).map_err(|e| ApiError::Stream(e.to_string()))?; @@ -51,10 +61,11 @@ impl CompactClient { &self, input: &CompactionInput<'_>, extra_headers: HeaderMap, + request_timeout: Duration, ) -> Result, ApiError> { let body = to_value(input) .map_err(|e| ApiError::Stream(format!("failed to encode compaction input: {e}")))?; - self.compact(body, extra_headers).await + self.compact(body, extra_headers, request_timeout).await } } diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index f604a63458..e3569e6d18 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -146,6 +146,9 @@ const X_CODEX_WS_STREAM_REQUEST_START_MS_CLIENT_METADATA_KEY: &str = const RESPONSES_WEBSOCKETS_V2_BETA_HEADER_VALUE: &str = "responses_websockets=2026-02-06"; const RESPONSES_ENDPOINT: &str = "/responses"; const RESPONSES_COMPACT_ENDPOINT: &str = "/responses/compact"; +// `/responses/compact` is unary, so the timeout covers the full response rather than one idle +// period between stream events. +const COMPACT_REQUEST_TIMEOUT_IDLE_MULTIPLIER: u32 = 4; const MEMORIES_SUMMARIZE_ENDPOINT: &str = "/memories/trace_summarize"; #[cfg(test)] pub(crate) const WEBSOCKET_CONNECT_TIMEOUT: Duration = @@ -502,12 +505,16 @@ impl ModelClient { if let Some(header_value) = self.generate_attestation_header_for().await { extra_headers.insert(X_OAI_ATTESTATION_HEADER, header_value); } + let compact_request_timeout = client_setup + .api_provider + .stream_idle_timeout + .saturating_mul(COMPACT_REQUEST_TIMEOUT_IDLE_MULTIPLIER); let client = ApiCompactClient::new(transport, client_setup.api_provider, client_setup.api_auth) .with_telemetry(Some(request_telemetry)); let trace_attempt = compaction_trace.start_attempt(&payload); let result = client - .compact_input(&payload, extra_headers) + .compact_input(&payload, extra_headers, compact_request_timeout) .await .map_err(map_api_error); trace_attempt.record_result(result.as_deref()); diff --git a/codex-rs/protocol/src/error.rs b/codex-rs/protocol/src/error.rs index 207fd94ca2..ef9c86cadf 100644 --- a/codex-rs/protocol/src/error.rs +++ b/codex-rs/protocol/src/error.rs @@ -89,6 +89,8 @@ pub enum CodexErr { /// Returned by run_command_stream when the spawned child process timed out (10s). #[error("timeout waiting for child process to exit")] Timeout, + #[error("request timed out")] + RequestTimeout, /// Returned by run_command_stream when the child could not be spawned (its stdout/stderr pipes /// could not be captured). Analogous to the previous `CodexError::Spawn` variant. #[error("spawn failed: child stdout/stderr not captured")] @@ -192,6 +194,7 @@ impl CodexErr { | CodexErr::CyberPolicy { .. } => false, CodexErr::Stream(..) | CodexErr::Timeout + | CodexErr::RequestTimeout | CodexErr::UnexpectedStatus(_) | CodexErr::ResponseStreamFailed(_) | CodexErr::ConnectionFailed(_) From 93456320ef6d95543eb944b923302a5c062794c2 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Wed, 20 May 2026 12:05:58 +0200 Subject: [PATCH 04/17] feat: rename 1 (#23667) Just a mechanical renaming --- .../tests/suite/v2/thread_resume.rs | 2 +- codex-rs/core/src/goals.rs | 10 ++-- codex-rs/state/src/lib.rs | 2 +- codex-rs/state/src/runtime.rs | 2 +- codex-rs/state/src/runtime/goals.rs | 54 +++++++++---------- 5 files changed, 35 insertions(+), 35 deletions(-) diff --git a/codex-rs/app-server/tests/suite/v2/thread_resume.rs b/codex-rs/app-server/tests/suite/v2/thread_resume.rs index 036f2368a0..d4baf7d773 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_resume.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_resume.rs @@ -1108,7 +1108,7 @@ async fn thread_goal_set_edits_objective_without_resetting_usage() -> Result<()> thread_id, /*time_delta_seconds*/ 12, /*token_delta*/ 50, - codex_state::ThreadGoalAccountingMode::ActiveOnly, + codex_state::GoalAccountingMode::ActiveOnly, Some(persisted_goal.goal_id.as_str()), ) .await?; diff --git a/codex-rs/core/src/goals.rs b/codex-rs/core/src/goals.rs index 6843264e44..794b14ea79 100644 --- a/codex-rs/core/src/goals.rs +++ b/codex-rs/core/src/goals.rs @@ -459,7 +459,7 @@ impl Session { self.account_thread_goal_wall_clock_usage( &state_db, - codex_state::ThreadGoalAccountingMode::ActiveOnly, + codex_state::GoalAccountingMode::ActiveOnly, TerminalMetricEmission::Emit, ) .await?; @@ -601,7 +601,7 @@ impl Session { let state_db = self.require_state_db_for_thread_goals().await?; self.account_thread_goal_wall_clock_usage( &state_db, - codex_state::ThreadGoalAccountingMode::ActiveOnly, + codex_state::GoalAccountingMode::ActiveOnly, TerminalMetricEmission::Emit, ) .await?; @@ -1028,7 +1028,7 @@ impl Session { self.conversation_id, time_delta_seconds, token_delta, - codex_state::ThreadGoalAccountingMode::ActiveOnly, + codex_state::GoalAccountingMode::ActiveOnly, expected_goal_id.as_deref(), ) .await?; @@ -1119,7 +1119,7 @@ impl Session { }; self.account_thread_goal_wall_clock_usage( &state_db, - codex_state::ThreadGoalAccountingMode::ActiveOnly, + codex_state::GoalAccountingMode::ActiveOnly, TerminalMetricEmission::Suppress, ) .await?; @@ -1129,7 +1129,7 @@ impl Session { async fn account_thread_goal_wall_clock_usage( &self, state_db: &StateDbHandle, - mode: codex_state::ThreadGoalAccountingMode, + mode: codex_state::GoalAccountingMode, terminal_metric_emission: TerminalMetricEmission, ) -> anyhow::Result> { let _accounting_permit = self.goal_runtime.accounting_permit().await?; diff --git a/codex-rs/state/src/lib.rs b/codex-rs/state/src/lib.rs index 8746ff3a0d..f835000fa4 100644 --- a/codex-rs/state/src/lib.rs +++ b/codex-rs/state/src/lib.rs @@ -48,11 +48,11 @@ pub use model::ThreadGoalStatus; pub use model::ThreadMetadata; pub use model::ThreadMetadataBuilder; pub use model::ThreadsPage; +pub use runtime::GoalAccountingMode; pub use runtime::GoalStore; pub use runtime::RemoteControlEnrollmentRecord; pub use runtime::RuntimeDbPath; pub use runtime::ThreadFilterOptions; -pub use runtime::ThreadGoalAccountingMode; pub use runtime::ThreadGoalAccountingOutcome; pub use runtime::ThreadGoalUpdate; pub use runtime::goals_db_filename; diff --git a/codex-rs/state/src/runtime.rs b/codex-rs/state/src/runtime.rs index ac03f227da..1dc349b231 100644 --- a/codex-rs/state/src/runtime.rs +++ b/codex-rs/state/src/runtime.rs @@ -66,8 +66,8 @@ mod remote_control; mod test_support; mod threads; +pub use goals::GoalAccountingMode; pub use goals::GoalStore; -pub use goals::ThreadGoalAccountingMode; pub use goals::ThreadGoalAccountingOutcome; pub use goals::ThreadGoalUpdate; pub use remote_control::RemoteControlEnrollmentRecord; diff --git a/codex-rs/state/src/runtime/goals.rs b/codex-rs/state/src/runtime/goals.rs index bf5a7bd825..c9a1d126bb 100644 --- a/codex-rs/state/src/runtime/goals.rs +++ b/codex-rs/state/src/runtime/goals.rs @@ -26,7 +26,7 @@ pub enum ThreadGoalAccountingOutcome { } #[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum ThreadGoalAccountingMode { +pub enum GoalAccountingMode { ActiveStatusOnly, ActiveOnly, ActiveOrComplete, @@ -387,7 +387,7 @@ WHERE thread_id = ? thread_id: ThreadId, time_delta_seconds: i64, token_delta: i64, - mode: ThreadGoalAccountingMode, + mode: GoalAccountingMode, expected_goal_id: Option<&str>, ) -> anyhow::Result { let time_delta_seconds = time_delta_seconds.max(0); @@ -400,20 +400,20 @@ WHERE thread_id = ? let now_ms = datetime_to_epoch_millis(Utc::now()); let status_filter = match mode { - ThreadGoalAccountingMode::ActiveStatusOnly => "status = 'active'", - ThreadGoalAccountingMode::ActiveOnly => "status IN ('active', 'budget_limited')", - ThreadGoalAccountingMode::ActiveOrComplete => { + GoalAccountingMode::ActiveStatusOnly => "status = 'active'", + GoalAccountingMode::ActiveOnly => "status IN ('active', 'budget_limited')", + GoalAccountingMode::ActiveOrComplete => { "status IN ('active', 'budget_limited', 'complete')" } - ThreadGoalAccountingMode::ActiveOrStopped => { + GoalAccountingMode::ActiveOrStopped => { "status IN ('active', 'paused', 'blocked', 'usage_limited', 'budget_limited')" } }; let budget_limit_status_filter = match mode { - ThreadGoalAccountingMode::ActiveStatusOnly - | ThreadGoalAccountingMode::ActiveOnly - | ThreadGoalAccountingMode::ActiveOrComplete => "status = 'active'", - ThreadGoalAccountingMode::ActiveOrStopped => { + GoalAccountingMode::ActiveStatusOnly + | GoalAccountingMode::ActiveOnly + | GoalAccountingMode::ActiveOrComplete => "status = 'active'", + GoalAccountingMode::ActiveOrStopped => { "status IN ('active', 'paused', 'blocked', 'usage_limited', 'budget_limited')" } }; @@ -802,7 +802,7 @@ mod tests { thread_id, /*time_delta_seconds*/ 5, /*token_delta*/ 5, - ThreadGoalAccountingMode::ActiveOnly, + GoalAccountingMode::ActiveOnly, Some(original.goal_id.as_str()), ) .await @@ -840,7 +840,7 @@ mod tests { thread_id, /*time_delta_seconds*/ 12, /*token_delta*/ 30, - ThreadGoalAccountingMode::ActiveOnly, + GoalAccountingMode::ActiveOnly, /*expected_goal_id*/ None, ) .await @@ -1062,7 +1062,7 @@ mod tests { thread_id, /*time_delta_seconds*/ 7, /*token_delta*/ 5, - ThreadGoalAccountingMode::ActiveOnly, + GoalAccountingMode::ActiveOnly, /*expected_goal_id*/ None, ) .await @@ -1080,7 +1080,7 @@ mod tests { thread_id, /*time_delta_seconds*/ 3, /*token_delta*/ 15, - ThreadGoalAccountingMode::ActiveOnly, + GoalAccountingMode::ActiveOnly, /*expected_goal_id*/ None, ) .await @@ -1098,7 +1098,7 @@ mod tests { thread_id, /*time_delta_seconds*/ 5, /*token_delta*/ 5, - ThreadGoalAccountingMode::ActiveOnly, + GoalAccountingMode::ActiveOnly, /*expected_goal_id*/ None, ) .await @@ -1133,7 +1133,7 @@ mod tests { thread_id, /*time_delta_seconds*/ 5, /*token_delta*/ 5, - ThreadGoalAccountingMode::ActiveStatusOnly, + GoalAccountingMode::ActiveStatusOnly, /*expected_goal_id*/ None, ) .await @@ -1181,7 +1181,7 @@ mod tests { thread_id, /*time_delta_seconds*/ 3, /*token_delta*/ 25, - ThreadGoalAccountingMode::ActiveOrStopped, + GoalAccountingMode::ActiveOrStopped, /*expected_goal_id*/ None, ) .await @@ -1215,7 +1215,7 @@ mod tests { thread_id, /*time_delta_seconds*/ 1, /*token_delta*/ 50, - ThreadGoalAccountingMode::ActiveOnly, + GoalAccountingMode::ActiveOnly, /*expected_goal_id*/ None, ) .await @@ -1262,7 +1262,7 @@ mod tests { thread_id, /*time_delta_seconds*/ 1, /*token_delta*/ 50, - ThreadGoalAccountingMode::ActiveOnly, + GoalAccountingMode::ActiveOnly, /*expected_goal_id*/ None, ) .await @@ -1313,7 +1313,7 @@ mod tests { thread_id, /*time_delta_seconds*/ 1, /*token_delta*/ 50, - ThreadGoalAccountingMode::ActiveOnly, + GoalAccountingMode::ActiveOnly, /*expected_goal_id*/ None, ) .await @@ -1360,7 +1360,7 @@ mod tests { thread_id, /*time_delta_seconds*/ 1, /*token_delta*/ 50, - ThreadGoalAccountingMode::ActiveOnly, + GoalAccountingMode::ActiveOnly, /*expected_goal_id*/ None, ) .await @@ -1413,7 +1413,7 @@ mod tests { thread_id, /*time_delta_seconds*/ 30, /*token_delta*/ 200, - ThreadGoalAccountingMode::ActiveOnly, + GoalAccountingMode::ActiveOnly, /*expected_goal_id*/ None, ) .await @@ -1431,7 +1431,7 @@ mod tests { thread_id, /*time_delta_seconds*/ 30, /*token_delta*/ 200, - ThreadGoalAccountingMode::ActiveOrComplete, + GoalAccountingMode::ActiveOrComplete, /*expected_goal_id*/ None, ) .await @@ -1480,7 +1480,7 @@ mod tests { thread_id, /*time_delta_seconds*/ 30, /*token_delta*/ 200, - ThreadGoalAccountingMode::ActiveOnly, + GoalAccountingMode::ActiveOnly, /*expected_goal_id*/ None, ) .await @@ -1498,7 +1498,7 @@ mod tests { thread_id, /*time_delta_seconds*/ 30, /*token_delta*/ 200, - ThreadGoalAccountingMode::ActiveOrStopped, + GoalAccountingMode::ActiveOrStopped, /*expected_goal_id*/ None, ) .await @@ -1531,14 +1531,14 @@ mod tests { thread_id, /*time_delta_seconds*/ 4, /*token_delta*/ 40, - ThreadGoalAccountingMode::ActiveOnly, + GoalAccountingMode::ActiveOnly, /*expected_goal_id*/ None, ); let second = runtime.thread_goals().account_thread_goal_usage( thread_id, /*time_delta_seconds*/ 6, /*token_delta*/ 60, - ThreadGoalAccountingMode::ActiveOnly, + GoalAccountingMode::ActiveOnly, /*expected_goal_id*/ None, ); let (first, second) = tokio::join!(first, second); From 66d5edf8258d7c07d9b904e5e4a046b8027ae45f Mon Sep 17 00:00:00 2001 From: jif-oai Date: Wed, 20 May 2026 12:07:06 +0200 Subject: [PATCH 05/17] feat: rename 3 (#23669) Just a mechanical renaming --- .../thread_goal_processor.rs | 4 +-- codex-rs/core/src/goals.rs | 4 +-- codex-rs/core/src/session/tests.rs | 2 +- codex-rs/state/src/lib.rs | 2 +- codex-rs/state/src/runtime.rs | 3 +- codex-rs/state/src/runtime/goals.rs | 32 +++++++++---------- 6 files changed, 24 insertions(+), 23 deletions(-) diff --git a/codex-rs/app-server/src/request_processors/thread_goal_processor.rs b/codex-rs/app-server/src/request_processors/thread_goal_processor.rs index cf1e58345b..c18806ea03 100644 --- a/codex-rs/app-server/src/request_processors/thread_goal_processor.rs +++ b/codex-rs/app-server/src/request_processors/thread_goal_processor.rs @@ -161,7 +161,7 @@ impl ThreadGoalRequestProcessor { .thread_goals() .update_thread_goal( thread_id, - codex_state::ThreadGoalUpdate { + codex_state::GoalUpdate { objective: Some(objective.to_string()), status, token_budget: params.token_budget, @@ -206,7 +206,7 @@ impl ThreadGoalRequestProcessor { .thread_goals() .update_thread_goal( thread_id, - codex_state::ThreadGoalUpdate { + codex_state::GoalUpdate { objective: None, status, token_budget: params.token_budget, diff --git a/codex-rs/core/src/goals.rs b/codex-rs/core/src/goals.rs index 794b14ea79..ac7d4cfb46 100644 --- a/codex-rs/core/src/goals.rs +++ b/codex-rs/core/src/goals.rs @@ -476,7 +476,7 @@ impl Session { .thread_goals() .update_thread_goal( self.conversation_id, - codex_state::ThreadGoalUpdate { + codex_state::GoalUpdate { objective: Some(objective.to_string()), status: status.map(state_goal_status_from_protocol), token_budget, @@ -516,7 +516,7 @@ impl Session { .thread_goals() .update_thread_goal( self.conversation_id, - codex_state::ThreadGoalUpdate { + codex_state::GoalUpdate { objective: None, status, token_budget, diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs index bfe3cfb3fe..175f022289 100644 --- a/codex-rs/core/src/session/tests.rs +++ b/codex-rs/core/src/session/tests.rs @@ -8620,7 +8620,7 @@ async fn external_goal_mutation_accounts_active_turn_before_status_change() -> a .thread_goals() .update_thread_goal( sess.conversation_id, - codex_state::ThreadGoalUpdate { + codex_state::GoalUpdate { objective: None, status: Some(codex_state::ThreadGoalStatus::Complete), token_budget: None, diff --git a/codex-rs/state/src/lib.rs b/codex-rs/state/src/lib.rs index f835000fa4..e18b9f8970 100644 --- a/codex-rs/state/src/lib.rs +++ b/codex-rs/state/src/lib.rs @@ -50,11 +50,11 @@ pub use model::ThreadMetadataBuilder; pub use model::ThreadsPage; pub use runtime::GoalAccountingMode; pub use runtime::GoalStore; +pub use runtime::GoalUpdate; pub use runtime::RemoteControlEnrollmentRecord; pub use runtime::RuntimeDbPath; pub use runtime::ThreadFilterOptions; pub use runtime::ThreadGoalAccountingOutcome; -pub use runtime::ThreadGoalUpdate; pub use runtime::goals_db_filename; pub use runtime::goals_db_path; pub use runtime::logs_db_filename; diff --git a/codex-rs/state/src/runtime.rs b/codex-rs/state/src/runtime.rs index 1dc349b231..0387dd3f98 100644 --- a/codex-rs/state/src/runtime.rs +++ b/codex-rs/state/src/runtime.rs @@ -68,8 +68,9 @@ mod threads; pub use goals::GoalAccountingMode; pub use goals::GoalStore; +pub use goals::GoalUpdate; +pub use goals::ThreadGoalAccountingMode; pub use goals::ThreadGoalAccountingOutcome; -pub use goals::ThreadGoalUpdate; pub use remote_control::RemoteControlEnrollmentRecord; pub use threads::ThreadFilterOptions; diff --git a/codex-rs/state/src/runtime/goals.rs b/codex-rs/state/src/runtime/goals.rs index c9a1d126bb..217feb208f 100644 --- a/codex-rs/state/src/runtime/goals.rs +++ b/codex-rs/state/src/runtime/goals.rs @@ -13,7 +13,7 @@ impl GoalStore { } } -pub struct ThreadGoalUpdate { +pub struct GoalUpdate { pub objective: Option, pub status: Option, pub token_budget: Option>, @@ -170,9 +170,9 @@ RETURNING pub async fn update_thread_goal( &self, thread_id: ThreadId, - update: ThreadGoalUpdate, + update: GoalUpdate, ) -> anyhow::Result> { - let ThreadGoalUpdate { + let GoalUpdate { objective, status, token_budget, @@ -556,7 +556,7 @@ mod tests { .thread_goals() .update_thread_goal( thread_id, - ThreadGoalUpdate { + GoalUpdate { objective: None, status: Some(crate::ThreadGoalStatus::Paused), token_budget: Some(Some(200_000)), @@ -732,7 +732,7 @@ mod tests { .thread_goals() .update_thread_goal( thread_id, - ThreadGoalUpdate { + GoalUpdate { objective: None, status: Some(crate::ThreadGoalStatus::Complete), token_budget: None, @@ -756,7 +756,7 @@ mod tests { .thread_goals() .update_thread_goal( thread_id, - ThreadGoalUpdate { + GoalUpdate { objective: None, status: Some(crate::ThreadGoalStatus::Complete), token_budget: None, @@ -853,7 +853,7 @@ mod tests { .thread_goals() .update_thread_goal( thread_id, - ThreadGoalUpdate { + GoalUpdate { objective: Some("draft the report clearly".to_string()), status: Some(crate::ThreadGoalStatus::Paused), token_budget: Some(Some(200)), @@ -891,7 +891,7 @@ mod tests { let status_update = runtime.thread_goals().update_thread_goal( thread_id, - ThreadGoalUpdate { + GoalUpdate { objective: None, status: Some(crate::ThreadGoalStatus::Paused), token_budget: None, @@ -900,7 +900,7 @@ mod tests { ); let budget_update = runtime.thread_goals().update_thread_goal( thread_id, - ThreadGoalUpdate { + GoalUpdate { objective: None, status: None, token_budget: Some(Some(200_000)), @@ -954,7 +954,7 @@ mod tests { .thread_goals() .update_thread_goal( thread_id, - ThreadGoalUpdate { + GoalUpdate { objective: None, status: Some(crate::ThreadGoalStatus::Complete), token_budget: None, @@ -1165,7 +1165,7 @@ mod tests { .thread_goals() .update_thread_goal( thread_id, - crate::ThreadGoalUpdate { + crate::GoalUpdate { objective: None, status: Some(crate::ThreadGoalStatus::Paused), token_budget: None, @@ -1225,7 +1225,7 @@ mod tests { .thread_goals() .update_thread_goal( thread_id, - ThreadGoalUpdate { + GoalUpdate { objective: None, status: None, token_budget: Some(Some(40)), @@ -1272,7 +1272,7 @@ mod tests { .thread_goals() .update_thread_goal( thread_id, - ThreadGoalUpdate { + GoalUpdate { objective: Some("stay within budget, with clearer wording".to_string()), status: Some(crate::ThreadGoalStatus::Active), token_budget: None, @@ -1323,7 +1323,7 @@ mod tests { .thread_goals() .update_thread_goal( thread_id, - ThreadGoalUpdate { + GoalUpdate { objective: None, status: Some(crate::ThreadGoalStatus::Paused), token_budget: None, @@ -1373,7 +1373,7 @@ mod tests { .thread_goals() .update_thread_goal( thread_id, - ThreadGoalUpdate { + GoalUpdate { objective: None, status: Some(crate::ThreadGoalStatus::Blocked), token_budget: None, @@ -1463,7 +1463,7 @@ mod tests { .thread_goals() .update_thread_goal( thread_id, - ThreadGoalUpdate { + GoalUpdate { objective: None, status: Some(crate::ThreadGoalStatus::Paused), token_budget: None, From 9483b09ea4b6f2c36c50235d83686e324ab9c895 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Wed, 20 May 2026 12:11:44 +0200 Subject: [PATCH 06/17] feat: rename 2 (#23668) Just a mechanical renaming --- codex-rs/core/src/goals.rs | 8 +++---- codex-rs/state/src/lib.rs | 2 +- codex-rs/state/src/runtime.rs | 2 +- codex-rs/state/src/runtime/goals.rs | 34 ++++++++++++++--------------- 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/codex-rs/core/src/goals.rs b/codex-rs/core/src/goals.rs index ac7d4cfb46..e58e82c36d 100644 --- a/codex-rs/core/src/goals.rs +++ b/codex-rs/core/src/goals.rs @@ -1039,7 +1039,7 @@ impl Session { .is_some_and(|goal_id| reported_goal_id.as_deref() == Some(goal_id)) }; let goal = match outcome { - codex_state::ThreadGoalAccountingOutcome::Updated(goal) => { + codex_state::GoalAccountingOutcome::Updated(goal) => { let clear_active_goal = match goal.status { codex_state::ThreadGoalStatus::Active => false, codex_state::ThreadGoalStatus::BudgetLimited => { @@ -1072,7 +1072,7 @@ impl Session { } goal } - codex_state::ThreadGoalAccountingOutcome::Unchanged(_) => return Ok(()), + codex_state::GoalAccountingOutcome::Unchanged(_) => return Ok(()), }; let should_steer_budget_limit = matches!(budget_limit_steering, BudgetLimitSteering::Allowed) @@ -1158,7 +1158,7 @@ impl Session { ) .await? { - codex_state::ThreadGoalAccountingOutcome::Updated(goal) => { + codex_state::GoalAccountingOutcome::Updated(goal) => { if matches!(terminal_metric_emission, TerminalMetricEmission::Emit) { self.emit_goal_terminal_metrics_if_status_changed(previous_status, &goal); } @@ -1171,7 +1171,7 @@ impl Session { let goal = protocol_goal_from_state(goal); Ok(Some(goal)) } - codex_state::ThreadGoalAccountingOutcome::Unchanged(goal) => { + codex_state::GoalAccountingOutcome::Unchanged(goal) => { { let mut accounting = self.goal_runtime.accounting.lock().await; accounting.wall_clock.reset_baseline(); diff --git a/codex-rs/state/src/lib.rs b/codex-rs/state/src/lib.rs index e18b9f8970..f35b3420f6 100644 --- a/codex-rs/state/src/lib.rs +++ b/codex-rs/state/src/lib.rs @@ -49,12 +49,12 @@ pub use model::ThreadMetadata; pub use model::ThreadMetadataBuilder; pub use model::ThreadsPage; pub use runtime::GoalAccountingMode; +pub use runtime::GoalAccountingOutcome; pub use runtime::GoalStore; pub use runtime::GoalUpdate; pub use runtime::RemoteControlEnrollmentRecord; pub use runtime::RuntimeDbPath; pub use runtime::ThreadFilterOptions; -pub use runtime::ThreadGoalAccountingOutcome; pub use runtime::goals_db_filename; pub use runtime::goals_db_path; pub use runtime::logs_db_filename; diff --git a/codex-rs/state/src/runtime.rs b/codex-rs/state/src/runtime.rs index 0387dd3f98..7041af9146 100644 --- a/codex-rs/state/src/runtime.rs +++ b/codex-rs/state/src/runtime.rs @@ -67,10 +67,10 @@ mod test_support; mod threads; pub use goals::GoalAccountingMode; +pub use goals::GoalAccountingOutcome; pub use goals::GoalStore; pub use goals::GoalUpdate; pub use goals::ThreadGoalAccountingMode; -pub use goals::ThreadGoalAccountingOutcome; pub use remote_control::RemoteControlEnrollmentRecord; pub use threads::ThreadFilterOptions; diff --git a/codex-rs/state/src/runtime/goals.rs b/codex-rs/state/src/runtime/goals.rs index 217feb208f..38a64d0845 100644 --- a/codex-rs/state/src/runtime/goals.rs +++ b/codex-rs/state/src/runtime/goals.rs @@ -20,7 +20,7 @@ pub struct GoalUpdate { pub expected_goal_id: Option, } -pub enum ThreadGoalAccountingOutcome { +pub enum GoalAccountingOutcome { Unchanged(Option), Updated(crate::ThreadGoal), } @@ -389,11 +389,11 @@ WHERE thread_id = ? token_delta: i64, mode: GoalAccountingMode, expected_goal_id: Option<&str>, - ) -> anyhow::Result { + ) -> anyhow::Result { let time_delta_seconds = time_delta_seconds.max(0); let token_delta = token_delta.max(0); if time_delta_seconds == 0 && token_delta == 0 { - return Ok(ThreadGoalAccountingOutcome::Unchanged( + return Ok(GoalAccountingOutcome::Unchanged( self.get_thread_goal(thread_id).await?, )); } @@ -464,13 +464,13 @@ RETURNING let row = query.fetch_optional(self.pool.as_ref()).await?; let Some(row) = row else { - return Ok(ThreadGoalAccountingOutcome::Unchanged( + return Ok(GoalAccountingOutcome::Unchanged( self.get_thread_goal(thread_id).await?, )); }; let updated = thread_goal_from_row(&row)?; - Ok(ThreadGoalAccountingOutcome::Updated(updated)) + Ok(GoalAccountingOutcome::Updated(updated)) } } @@ -808,7 +808,7 @@ mod tests { .await .expect("usage accounting should succeed"); - let ThreadGoalAccountingOutcome::Unchanged(Some(goal)) = outcome else { + let GoalAccountingOutcome::Unchanged(Some(goal)) = outcome else { panic!("stale goal version should not be updated"); }; assert_ne!(replacement.goal_id, original.goal_id); @@ -845,7 +845,7 @@ mod tests { ) .await .expect("usage accounting should succeed"); - let ThreadGoalAccountingOutcome::Updated(accounted) = outcome else { + let GoalAccountingOutcome::Updated(accounted) = outcome else { panic!("active goal should account usage"); }; @@ -1067,7 +1067,7 @@ mod tests { ) .await .expect("usage accounting should succeed"); - let ThreadGoalAccountingOutcome::Updated(goal) = outcome else { + let GoalAccountingOutcome::Updated(goal) = outcome else { panic!("active goal should be updated"); }; assert_eq!(crate::ThreadGoalStatus::Active, goal.status); @@ -1085,7 +1085,7 @@ mod tests { ) .await .expect("usage accounting should succeed"); - let ThreadGoalAccountingOutcome::Updated(goal) = outcome else { + let GoalAccountingOutcome::Updated(goal) = outcome else { panic!("budget crossing should update the goal"); }; assert_eq!(crate::ThreadGoalStatus::BudgetLimited, goal.status); @@ -1103,7 +1103,7 @@ mod tests { ) .await .expect("usage accounting should succeed"); - let ThreadGoalAccountingOutcome::Updated(goal) = outcome else { + let GoalAccountingOutcome::Updated(goal) = outcome else { panic!("budget-limited goal should still account in-flight active usage"); }; assert_eq!(crate::ThreadGoalStatus::BudgetLimited, goal.status); @@ -1138,7 +1138,7 @@ mod tests { ) .await .expect("usage accounting should succeed"); - let ThreadGoalAccountingOutcome::Unchanged(Some(goal)) = outcome else { + let GoalAccountingOutcome::Unchanged(Some(goal)) = outcome else { panic!("budget-limited goal should not be updated"); }; assert_eq!(crate::ThreadGoalStatus::BudgetLimited, goal.status); @@ -1186,7 +1186,7 @@ mod tests { ) .await .expect("usage accounting should succeed"); - let ThreadGoalAccountingOutcome::Updated(goal) = outcome else { + let GoalAccountingOutcome::Updated(goal) = outcome else { panic!("stopped goal should account final usage"); }; assert_eq!(crate::ThreadGoalStatus::BudgetLimited, goal.status); @@ -1365,7 +1365,7 @@ mod tests { ) .await .expect("usage accounting should succeed"); - let ThreadGoalAccountingOutcome::Updated(budget_limited) = outcome else { + let GoalAccountingOutcome::Updated(budget_limited) = outcome else { panic!("budget crossing should update the goal"); }; @@ -1418,7 +1418,7 @@ mod tests { ) .await .expect("usage accounting should succeed"); - let ThreadGoalAccountingOutcome::Unchanged(Some(goal)) = active_only else { + let GoalAccountingOutcome::Unchanged(Some(goal)) = active_only else { panic!("completed goal should not be updated by active-only accounting"); }; assert_eq!(crate::ThreadGoalStatus::Complete, goal.status); @@ -1436,7 +1436,7 @@ mod tests { ) .await .expect("usage accounting should succeed"); - let ThreadGoalAccountingOutcome::Updated(goal) = completing_turn else { + let GoalAccountingOutcome::Updated(goal) = completing_turn else { panic!("completed goal should be updated for final accounting"); }; assert_eq!(crate::ThreadGoalStatus::Complete, goal.status); @@ -1485,7 +1485,7 @@ mod tests { ) .await .expect("usage accounting should succeed"); - let ThreadGoalAccountingOutcome::Unchanged(Some(goal)) = active_only else { + let GoalAccountingOutcome::Unchanged(Some(goal)) = active_only else { panic!("paused goal should not be updated by active-only accounting"); }; assert_eq!(crate::ThreadGoalStatus::Paused, goal.status); @@ -1503,7 +1503,7 @@ mod tests { ) .await .expect("usage accounting should succeed"); - let ThreadGoalAccountingOutcome::Updated(goal) = in_flight_turn else { + let GoalAccountingOutcome::Updated(goal) = in_flight_turn else { panic!("stopped goal should be updated for in-flight accounting"); }; assert_eq!(crate::ThreadGoalStatus::Paused, goal.status); From 51d661643103d17f61ccb4e92dc88ea7fe957f7b Mon Sep 17 00:00:00 2001 From: jif-oai Date: Wed, 20 May 2026 12:27:39 +0200 Subject: [PATCH 07/17] fix: main (#23675) Fix main due to conflicting merges This is only fixing some imports and mechanics --- codex-rs/state/src/runtime.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/codex-rs/state/src/runtime.rs b/codex-rs/state/src/runtime.rs index 7041af9146..8c9c6d0267 100644 --- a/codex-rs/state/src/runtime.rs +++ b/codex-rs/state/src/runtime.rs @@ -70,7 +70,6 @@ pub use goals::GoalAccountingMode; pub use goals::GoalAccountingOutcome; pub use goals::GoalStore; pub use goals::GoalUpdate; -pub use goals::ThreadGoalAccountingMode; pub use remote_control::RemoteControlEnrollmentRecord; pub use threads::ThreadFilterOptions; From b555dd5d1d6d7c3ca7e5b00f8e305239b5a651b5 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Wed, 20 May 2026 14:44:17 +0200 Subject: [PATCH 08/17] feat: wire goal extension tools to the dedicated goal store (#23685) ## Why `ext/goal` already had the tool specs and contributor wiring for `/goal`, but the installed tools still depended on a placeholder backend that always errored. That meant the extension could not actually own goal persistence even though the dedicated `thread_goals` store already exists. This change wires the extension tools directly to the dedicated goal store so the extension can create, read, and complete goals against real state instead of falling back to host-side placeholders. ## What changed - make `install_with_backend(...)` require `Arc` so goal storage is always available when the extension is installed - remove the unused no-backend/public backend abstraction from `ext/goal` and have the tool executors talk directly to `StateRuntime` - map `thread_goals` rows into the existing protocol response shape for `get_goal`, `create_goal`, and `update_goal` - preserve current thread-list behavior by filling an empty thread preview from the goal objective when a goal is created through the extension path - add integration coverage for the installed tool surface, including successful goal creation and duplicate-create rejection ## Testing - `cargo test -p codex-goal-extension` --- codex-rs/Cargo.lock | 7 + codex-rs/ext/goal/Cargo.toml | 9 + codex-rs/ext/goal/src/extension.rs | 83 +-------- codex-rs/ext/goal/src/lib.rs | 3 - codex-rs/ext/goal/src/tool.rs | 107 +++++++++-- .../ext/goal/tests/goal_extension_backend.rs | 171 ++++++++++++++++++ 6 files changed, 286 insertions(+), 94 deletions(-) create mode 100644 codex-rs/ext/goal/tests/goal_extension_backend.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 09c5786a9c..6dfc44d844 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -2952,12 +2952,19 @@ dependencies = [ name = "codex-goal-extension" version = "0.0.0" dependencies = [ + "anyhow", "async-trait", + "chrono", "codex-extension-api", "codex-protocol", + "codex-state", "codex-tools", + "pretty_assertions", "serde", "serde_json", + "tempfile", + "tokio", + "tracing", ] [[package]] diff --git a/codex-rs/ext/goal/Cargo.toml b/codex-rs/ext/goal/Cargo.toml index 7f8b9bd308..b6590f68e6 100644 --- a/codex-rs/ext/goal/Cargo.toml +++ b/codex-rs/ext/goal/Cargo.toml @@ -17,6 +17,15 @@ workspace = true async-trait = { workspace = true } codex-extension-api = { workspace = true } codex-protocol = { workspace = true } +codex-state = { workspace = true } codex-tools = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } +tracing = { workspace = true } + +[dev-dependencies] +anyhow = { workspace = true } +chrono = { workspace = true } +pretty_assertions = { workspace = true } +tempfile = { workspace = true } +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/codex-rs/ext/goal/src/extension.rs b/codex-rs/ext/goal/src/extension.rs index a8d4f5c289..657d285f74 100644 --- a/codex-rs/ext/goal/src/extension.rs +++ b/codex-rs/ext/goal/src/extension.rs @@ -5,7 +5,6 @@ use codex_extension_api::ConfigContributor; use codex_extension_api::ExtensionData; use codex_extension_api::ExtensionEventSink; use codex_extension_api::ExtensionRegistryBuilder; -use codex_extension_api::NoopExtensionEventSink; use codex_extension_api::ThreadLifecycleContributor; use codex_extension_api::ThreadStartInput; use codex_extension_api::TokenUsageContributor; @@ -19,14 +18,12 @@ use codex_extension_api::TurnLifecycleContributor; use codex_extension_api::TurnStartInput; use codex_extension_api::TurnStopInput; use codex_protocol::ThreadId; -use codex_protocol::protocol::ThreadGoal; use codex_protocol::protocol::TokenUsageInfo; use codex_protocol::protocol::TurnAbortReason; use crate::accounting::GoalAccountingState; use crate::events::GoalEventEmitter; use crate::spec::UPDATE_GOAL_TOOL_NAME; -use crate::tool::CreateGoalRequest; use crate::tool::GoalToolExecutor; #[derive(Clone, Debug)] @@ -42,7 +39,7 @@ impl GoalExtensionConfig { #[derive(Clone)] pub struct GoalExtension { - backend: Arc, + state_dbs: Arc, event_emitter: GoalEventEmitter, goals_enabled: Arc bool + Send + Sync>, } @@ -54,70 +51,17 @@ impl std::fmt::Debug for GoalExtension { } impl GoalExtension { - pub fn new( - backend: Arc, - goals_enabled: impl Fn(&C) -> bool + Send + Sync + 'static, - ) -> Self { - Self::new_with_event_sink(backend, Arc::new(NoopExtensionEventSink), goals_enabled) - } - - pub fn new_with_event_sink( - backend: Arc, + pub(crate) fn new_with_event_sink( + state_dbs: Arc, event_sink: Arc, goals_enabled: impl Fn(&C) -> bool + Send + Sync + 'static, ) -> Self { Self { - backend, + state_dbs, event_emitter: GoalEventEmitter::new(event_sink), goals_enabled: Arc::new(goals_enabled), } } - - pub fn without_backend(goals_enabled: impl Fn(&C) -> bool + Send + Sync + 'static) -> Self { - Self::new(Arc::new(NoGoalToolBackend), goals_enabled) - } -} - -#[async_trait] -pub trait GoalToolBackend: Send + Sync { - async fn get_goal(&self, thread_id: ThreadId) -> Result, String>; - - async fn create_goal( - &self, - thread_id: ThreadId, - request: CreateGoalRequest, - ) -> Result; - - async fn complete_goal(&self, thread_id: ThreadId) -> Result; -} - -#[derive(Clone, Copy, Debug, Default)] -pub struct NoGoalToolBackend; - -#[async_trait] -impl GoalToolBackend for NoGoalToolBackend { - async fn get_goal(&self, _thread_id: ThreadId) -> Result, String> { - Err(missing_backend_message()) - } - - async fn create_goal( - &self, - _thread_id: ThreadId, - _request: CreateGoalRequest, - ) -> Result { - Err(missing_backend_message()) - } - - async fn complete_goal(&self, _thread_id: ThreadId) -> Result { - Err(missing_backend_message()) - } -} - -fn missing_backend_message() -> String { - // TODO: replace this fallback with a host-provided goal backend once - // ToolContributor invocations can reach thread-scoped goal persistence and - // the current turn context. - "goal tools are not connected to host goal persistence yet".to_string() } #[async_trait] @@ -270,41 +214,32 @@ where vec![ Arc::new(GoalToolExecutor::get( thread_id, - Arc::clone(&self.backend), + Arc::clone(&self.state_dbs), self.event_emitter.clone(), )), Arc::new(GoalToolExecutor::create( thread_id, - Arc::clone(&self.backend), + Arc::clone(&self.state_dbs), self.event_emitter.clone(), )), Arc::new(GoalToolExecutor::update( thread_id, - Arc::clone(&self.backend), + Arc::clone(&self.state_dbs), self.event_emitter.clone(), )), ] } } -pub fn install( - registry: &mut ExtensionRegistryBuilder, - goals_enabled: impl Fn(&C) -> bool + Send + Sync + 'static, -) where - C: Send + Sync + 'static, -{ - install_with_backend(registry, Arc::new(NoGoalToolBackend), goals_enabled); -} - pub fn install_with_backend( registry: &mut ExtensionRegistryBuilder, - backend: Arc, + state_dbs: Arc, goals_enabled: impl Fn(&C) -> bool + Send + Sync + 'static, ) where C: Send + Sync + 'static, { let extension = Arc::new(GoalExtension::new_with_event_sink( - backend, + state_dbs, registry.event_sink(), goals_enabled, )); diff --git a/codex-rs/ext/goal/src/lib.rs b/codex-rs/ext/goal/src/lib.rs index 38332aee41..1625aeae14 100644 --- a/codex-rs/ext/goal/src/lib.rs +++ b/codex-rs/ext/goal/src/lib.rs @@ -12,9 +12,6 @@ mod tool; pub use extension::GoalExtension; pub use extension::GoalExtensionConfig; -pub use extension::GoalToolBackend; -pub use extension::NoGoalToolBackend; -pub use extension::install; pub use extension::install_with_backend; pub use spec::CREATE_GOAL_TOOL_NAME; pub use spec::GET_GOAL_TOOL_NAME; diff --git a/codex-rs/ext/goal/src/tool.rs b/codex-rs/ext/goal/src/tool.rs index 8ad4541a3c..96b160f938 100644 --- a/codex-rs/ext/goal/src/tool.rs +++ b/codex-rs/ext/goal/src/tool.rs @@ -16,7 +16,6 @@ use serde::Deserialize; use serde::Serialize; use crate::events::GoalEventEmitter; -use crate::extension::GoalToolBackend; use crate::spec::CREATE_GOAL_TOOL_NAME; use crate::spec::GET_GOAL_TOOL_NAME; use crate::spec::UPDATE_GOAL_TOOL_NAME; @@ -28,7 +27,7 @@ use crate::spec::create_update_goal_tool; pub(crate) struct GoalToolExecutor { kind: GoalToolKind, thread_id: ThreadId, - backend: Arc, + state_db: Arc, event_emitter: GoalEventEmitter, } @@ -69,39 +68,39 @@ enum CompletionBudgetReport { impl GoalToolExecutor { pub(crate) fn get( thread_id: ThreadId, - backend: Arc, + state_db: Arc, event_emitter: GoalEventEmitter, ) -> Self { Self { kind: GoalToolKind::Get, thread_id, - backend, + state_db, event_emitter, } } pub(crate) fn create( thread_id: ThreadId, - backend: Arc, + state_db: Arc, event_emitter: GoalEventEmitter, ) -> Self { Self { kind: GoalToolKind::Create, thread_id, - backend, + state_db, event_emitter, } } pub(crate) fn update( thread_id: ThreadId, - backend: Arc, + state_db: Arc, event_emitter: GoalEventEmitter, ) -> Self { Self { kind: GoalToolKind::Update, thread_id, - backend, + state_db, event_emitter, } } @@ -141,10 +140,14 @@ impl GoalToolExecutor { ) -> Result, FunctionCallError> { let _ = invocation.function_arguments()?; let goal = self - .backend - .get_goal(self.thread_id) + .state_db + .thread_goals() + .get_thread_goal(self.thread_id) .await - .map_err(FunctionCallError::RespondToModel)?; + .map(|goal| goal.map(protocol_goal_from_state)) + .map_err(|err| { + FunctionCallError::RespondToModel(format!("failed to read goal: {err}")) + })?; goal_response(goal, CompletionBudgetReport::Omit) } @@ -159,10 +162,24 @@ impl GoalToolExecutor { validate_goal_budget(request.token_budget).map_err(FunctionCallError::RespondToModel)?; let goal = self - .backend - .create_goal(self.thread_id, request) + .state_db + .thread_goals() + .insert_thread_goal( + self.thread_id, + request.objective.as_str(), + codex_state::ThreadGoalStatus::Active, + request.token_budget, + ) .await - .map_err(FunctionCallError::RespondToModel)?; + .map_err(|err| FunctionCallError::RespondToModel(format!("failed to create goal: {err}")))? + .ok_or_else(|| { + FunctionCallError::RespondToModel( + "cannot create a new goal because this thread already has a goal; use update_goal only when the existing goal is complete" + .to_string(), + ) + })?; + fill_empty_thread_preview_if_possible(self.state_db.as_ref(), self.thread_id, &goal).await; + let goal = protocol_goal_from_state(goal); self.emit_goal_updated_from_tool_call(&invocation, goal.clone()); goal_response(Some(goal), CompletionBudgetReport::Omit) } @@ -182,10 +199,27 @@ impl GoalToolExecutor { // TODO: update_goal needs a host callback before completion to flush // final active-turn accounting with budget steering suppressed. let goal = self - .backend - .complete_goal(self.thread_id) + .state_db + .thread_goals() + .update_thread_goal( + self.thread_id, + codex_state::GoalUpdate { + objective: None, + status: Some(codex_state::ThreadGoalStatus::Complete), + token_budget: None, + expected_goal_id: None, + }, + ) .await - .map_err(FunctionCallError::RespondToModel)?; + .map_err(|err| { + FunctionCallError::RespondToModel(format!("failed to complete goal: {err}")) + })? + .map(protocol_goal_from_state) + .ok_or_else(|| { + FunctionCallError::RespondToModel( + "cannot update goal because this thread has no goal".to_string(), + ) + })?; self.emit_goal_updated_from_tool_call(&invocation, goal.clone()); goal_response(Some(goal), CompletionBudgetReport::Include) } @@ -249,6 +283,45 @@ impl GoalToolResponse { } } +async fn fill_empty_thread_preview_if_possible( + state_db: &codex_state::StateRuntime, + thread_id: ThreadId, + goal: &codex_state::ThreadGoal, +) { + if let Err(err) = state_db + .set_thread_preview_if_empty(thread_id, goal.objective.as_str()) + .await + { + tracing::warn!( + "failed to set empty thread preview from goal objective for {thread_id}: {err}" + ); + } +} + +fn protocol_goal_from_state(goal: codex_state::ThreadGoal) -> ThreadGoal { + ThreadGoal { + thread_id: goal.thread_id, + objective: goal.objective, + status: protocol_status_from_state(goal.status), + token_budget: goal.token_budget, + tokens_used: goal.tokens_used, + time_used_seconds: goal.time_used_seconds, + created_at: goal.created_at.timestamp(), + updated_at: goal.updated_at.timestamp(), + } +} + +fn protocol_status_from_state(status: codex_state::ThreadGoalStatus) -> ThreadGoalStatus { + match status { + codex_state::ThreadGoalStatus::Active => ThreadGoalStatus::Active, + codex_state::ThreadGoalStatus::Paused => ThreadGoalStatus::Paused, + codex_state::ThreadGoalStatus::Blocked => ThreadGoalStatus::Blocked, + codex_state::ThreadGoalStatus::UsageLimited => ThreadGoalStatus::UsageLimited, + codex_state::ThreadGoalStatus::BudgetLimited => ThreadGoalStatus::BudgetLimited, + codex_state::ThreadGoalStatus::Complete => ThreadGoalStatus::Complete, + } +} + fn completion_budget_report(goal: &ThreadGoal) -> Option { if goal.token_budget.is_none() && goal.time_used_seconds <= 0 { None diff --git a/codex-rs/ext/goal/tests/goal_extension_backend.rs b/codex-rs/ext/goal/tests/goal_extension_backend.rs new file mode 100644 index 0000000000..a79c69c633 --- /dev/null +++ b/codex-rs/ext/goal/tests/goal_extension_backend.rs @@ -0,0 +1,171 @@ +use std::sync::Arc; + +use codex_extension_api::ExtensionData; +use codex_extension_api::ExtensionRegistryBuilder; +use codex_extension_api::FunctionCallError; +use codex_extension_api::ThreadStartInput; +use codex_extension_api::ToolCall; +use codex_extension_api::ToolExecutor; +use codex_extension_api::ToolPayload; +use codex_goal_extension::install_with_backend; +use codex_protocol::ThreadId; +use codex_protocol::ToolName; +use codex_protocol::protocol::SessionSource; +use pretty_assertions::assert_eq; +use serde_json::json; +use tempfile::TempDir; + +#[tokio::test] +async fn installed_goal_tools_create_goal_and_fill_empty_preview() -> anyhow::Result<()> { + let runtime = test_runtime().await?; + let thread_id = test_thread_id()?; + seed_thread_metadata(runtime.as_ref(), thread_id).await?; + let tools = installed_tools(runtime.clone(), thread_id).await; + + let create_tool = tool_by_name(&tools, "create_goal"); + let invocation = ToolCall { + call_id: "call-create-goal".to_string(), + tool_name: ToolName::plain("create_goal"), + payload: ToolPayload::Function { + arguments: json!({ + "objective": "ship goal extension backend", + "token_budget": 123, + }) + .to_string(), + }, + }; + let output = create_tool.handle(invocation.clone()).await?; + let result = output.code_mode_result(&invocation.payload); + assert_eq!( + result, + json!({ + "goal": { + "threadId": thread_id, + "objective": "ship goal extension backend", + "status": "active", + "tokenBudget": 123, + "tokensUsed": 0, + "timeUsedSeconds": 0, + "createdAt": result["goal"]["createdAt"], + "updatedAt": result["goal"]["updatedAt"], + }, + "remainingTokens": 123, + "completionBudgetReport": serde_json::Value::Null, + }) + ); + + let metadata = runtime + .get_thread(thread_id) + .await? + .ok_or_else(|| anyhow::anyhow!("seeded thread metadata should exist"))?; + assert_eq!( + metadata.preview.as_deref(), + Some("ship goal extension backend") + ); + Ok(()) +} + +#[tokio::test] +async fn installed_goal_tools_reject_duplicate_goal_creation() -> anyhow::Result<()> { + let runtime = test_runtime().await?; + let thread_id = test_thread_id()?; + seed_thread_metadata(runtime.as_ref(), thread_id).await?; + let tools = installed_tools(runtime, thread_id).await; + + let create_tool = tool_by_name(&tools, "create_goal"); + let first = tool_call( + "create_goal", + "call-create-goal-1", + json!({ "objective": "first goal" }), + ); + create_tool.handle(first).await?; + + let second = tool_call( + "create_goal", + "call-create-goal-2", + json!({ "objective": "second goal" }), + ); + let err = match create_tool.handle(second).await { + Ok(_) => panic!("duplicate create should fail"), + Err(err) => err, + }; + + assert_eq!( + err, + FunctionCallError::RespondToModel( + "cannot create a new goal because this thread already has a goal; use update_goal only when the existing goal is complete" + .to_string() + ) + ); + Ok(()) +} + +async fn installed_tools( + runtime: Arc, + thread_id: ThreadId, +) -> Vec>> { + let mut builder = ExtensionRegistryBuilder::<()>::new(); + install_with_backend(&mut builder, runtime, |_| true); + let registry = builder.build(); + let session_store = ExtensionData::new("session-1"); + let thread_store = ExtensionData::new(thread_id.to_string()); + for contributor in registry.thread_lifecycle_contributors() { + contributor + .on_thread_start(ThreadStartInput { + config: &(), + session_store: &session_store, + thread_store: &thread_store, + }) + .await; + } + + registry + .tool_contributors() + .iter() + .flat_map(|contributor| contributor.tools(&session_store, &thread_store)) + .collect() +} + +fn tool_by_name<'a>( + tools: &'a [Arc>], + name: &str, +) -> &'a Arc> { + tools + .iter() + .find(|tool| tool.tool_name().namespace.is_none() && tool.tool_name().name == name) + .unwrap_or_else(|| panic!("missing tool {name}")) +} + +fn tool_call(tool_name: &str, call_id: &str, arguments: serde_json::Value) -> ToolCall { + ToolCall { + call_id: call_id.to_string(), + tool_name: ToolName::plain(tool_name), + payload: ToolPayload::Function { + arguments: arguments.to_string(), + }, + } +} + +async fn test_runtime() -> anyhow::Result> { + let tempdir = TempDir::new()?; + codex_state::StateRuntime::init(tempdir.keep(), "test-provider".to_string()).await +} + +fn test_thread_id() -> anyhow::Result { + ThreadId::from_string("11111111-1111-4111-8111-111111111111").map_err(anyhow::Error::msg) +} + +async fn seed_thread_metadata( + runtime: &codex_state::StateRuntime, + thread_id: ThreadId, +) -> anyhow::Result<()> { + let builder = codex_state::ThreadMetadataBuilder::new( + thread_id, + runtime + .codex_home() + .join(format!("rollout-{thread_id}.jsonl")), + chrono::Utc::now(), + SessionSource::Cli, + ); + runtime.upsert_thread(&builder.build("test-provider")).await +} From f64fce61b32a921968667752a947232e8eb964bf Mon Sep 17 00:00:00 2001 From: jif-oai Date: Wed, 20 May 2026 15:13:54 +0200 Subject: [PATCH 09/17] feat: async approval contrib (#23690) --- .../ext/extension-api/src/contributors.rs | 17 +++++------ codex-rs/ext/extension-api/src/lib.rs | 1 - codex-rs/ext/extension-api/src/registry.rs | 28 ++++++++++++------- 3 files changed, 25 insertions(+), 21 deletions(-) diff --git a/codex-rs/ext/extension-api/src/contributors.rs b/codex-rs/ext/extension-api/src/contributors.rs index 8968e08573..cbe8e58244 100644 --- a/codex-rs/ext/extension-api/src/contributors.rs +++ b/codex-rs/ext/extension-api/src/contributors.rs @@ -134,18 +134,15 @@ pub trait ToolLifecycleContributor: Send + Sync { } } -/// Future returned by one claimed approval-review contribution. -pub type ApprovalReviewFuture<'a> = - std::pin::Pin + Send + 'a>>; - /// Extension contribution that can claim rendered approval-review prompts. +#[async_trait::async_trait] pub trait ApprovalReviewContributor: Send + Sync { - fn contribute<'a>( - &'a self, - session_store: &'a ExtensionData, - thread_store: &'a ExtensionData, - prompt: &'a str, - ) -> Option>; + async fn contribute( + &self, + session_store: &ExtensionData, + thread_store: &ExtensionData, + prompt: &str, + ) -> Option; } /// Future returned by one ordered turn-item contribution. diff --git a/codex-rs/ext/extension-api/src/lib.rs b/codex-rs/ext/extension-api/src/lib.rs index 373f3735a4..f9af83d38d 100644 --- a/codex-rs/ext/extension-api/src/lib.rs +++ b/codex-rs/ext/extension-api/src/lib.rs @@ -18,7 +18,6 @@ pub use codex_tools::ToolPayload; pub use codex_tools::ToolSpec; pub use codex_tools::parse_tool_input_schema; pub use contributors::ApprovalReviewContributor; -pub use contributors::ApprovalReviewFuture; pub use contributors::ConfigContributor; pub use contributors::ContextContributor; pub use contributors::PromptFragment; diff --git a/codex-rs/ext/extension-api/src/registry.rs b/codex-rs/ext/extension-api/src/registry.rs index 4577ddc048..a359726257 100644 --- a/codex-rs/ext/extension-api/src/registry.rs +++ b/codex-rs/ext/extension-api/src/registry.rs @@ -1,7 +1,8 @@ use std::sync::Arc; +use codex_protocol::protocol::ReviewDecision; + use crate::ApprovalReviewContributor; -use crate::ApprovalReviewFuture; use crate::ConfigContributor; use crate::ContextContributor; use crate::ExtensionData; @@ -171,15 +172,22 @@ impl ExtensionRegistry { /// Claims the first rendered approval-review prompt accepted by an /// installed contributor. - pub fn approval_review<'a>( - &'a self, - session_store: &'a ExtensionData, - thread_store: &'a ExtensionData, - prompt: &'a str, - ) -> Option> { - self.approval_review_contributors - .iter() - .find_map(|contributor| contributor.contribute(session_store, thread_store, prompt)) + pub async fn approval_review( + &self, + session_store: &ExtensionData, + thread_store: &ExtensionData, + prompt: &str, + ) -> Option { + for contributor in &self.approval_review_contributors { + if let Some(decision) = contributor + .contribute(session_store, thread_store, prompt) + .await + { + return Some(decision); + } + } + + None } /// Returns the registered prompt contributors. From 1392a2a770e57334f7ec859524230a273a425ab9 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Wed, 20 May 2026 15:30:01 +0200 Subject: [PATCH 10/17] feat: async turn item process (#23692) Mechanical change --- codex-rs/core/src/session/turn_tests.rs | 28 ++++----- .../core/src/stream_events_utils_tests.rs | 59 +++++++++---------- .../ext/extension-api/src/contributors.rs | 17 +++--- codex-rs/ext/extension-api/src/lib.rs | 1 - 4 files changed, 48 insertions(+), 57 deletions(-) diff --git a/codex-rs/core/src/session/turn_tests.rs b/codex-rs/core/src/session/turn_tests.rs index f1cf86b46c..81a43ff5a9 100644 --- a/codex-rs/core/src/session/turn_tests.rs +++ b/codex-rs/core/src/session/turn_tests.rs @@ -1,6 +1,5 @@ use super::*; use codex_extension_api::ExtensionData; -use codex_extension_api::TurnItemContributionFuture; use codex_extension_api::TurnItemContributor; use codex_protocol::items::AgentMessageContent; use pretty_assertions::assert_eq; @@ -8,21 +7,20 @@ use std::sync::Arc; struct RewriteAgentMessageContributor; +#[async_trait::async_trait] impl TurnItemContributor for RewriteAgentMessageContributor { - fn contribute<'a>( - &'a self, - _thread_store: &'a ExtensionData, - _turn_store: &'a ExtensionData, - item: &'a mut TurnItem, - ) -> TurnItemContributionFuture<'a> { - Box::pin(async move { - if let TurnItem::AgentMessage(agent_message) = item { - agent_message.content = vec![AgentMessageContent::Text { - text: "plan contributed assistant text".to_string(), - }]; - } - Ok(()) - }) + async fn contribute( + &self, + _thread_store: &ExtensionData, + _turn_store: &ExtensionData, + item: &mut TurnItem, + ) -> Result<(), String> { + if let TurnItem::AgentMessage(agent_message) = item { + agent_message.content = vec![AgentMessageContent::Text { + text: "plan contributed assistant text".to_string(), + }]; + } + Ok(()) } } diff --git a/codex-rs/core/src/stream_events_utils_tests.rs b/codex-rs/core/src/stream_events_utils_tests.rs index 28ffbb7a6b..9c4be94576 100644 --- a/codex-rs/core/src/stream_events_utils_tests.rs +++ b/codex-rs/core/src/stream_events_utils_tests.rs @@ -13,7 +13,6 @@ use crate::tools::ToolRouter; use crate::tools::parallel::ToolCallRuntime; use crate::turn_diff_tracker::TurnDiffTracker; use codex_extension_api::ExtensionData; -use codex_extension_api::TurnItemContributionFuture; use codex_extension_api::TurnItemContributor; use codex_protocol::error::CodexErr; use codex_protocol::items::AgentMessageContent; @@ -168,43 +167,41 @@ struct TestTurnItemContributor; #[derive(Debug)] struct TurnItemContributorRan; +#[async_trait::async_trait] impl TurnItemContributor for TestTurnItemContributor { - fn contribute<'a>( - &'a self, - _thread_store: &'a ExtensionData, - turn_store: &'a ExtensionData, - item: &'a mut TurnItem, - ) -> TurnItemContributionFuture<'a> { - Box::pin(async move { - turn_store.insert(TurnItemContributorRan); - if let TurnItem::AgentMessage(agent_message) = item { - agent_message.memory_citation = Some(MemoryCitation { - entries: Vec::new(), - rollout_ids: Vec::new(), - }); - } - Ok(()) - }) + async fn contribute( + &self, + _thread_store: &ExtensionData, + turn_store: &ExtensionData, + item: &mut TurnItem, + ) -> Result<(), String> { + turn_store.insert(TurnItemContributorRan); + if let TurnItem::AgentMessage(agent_message) = item { + agent_message.memory_citation = Some(MemoryCitation { + entries: Vec::new(), + rollout_ids: Vec::new(), + }); + } + Ok(()) } } struct RewriteAgentMessageContributor; +#[async_trait::async_trait] impl TurnItemContributor for RewriteAgentMessageContributor { - fn contribute<'a>( - &'a self, - _thread_store: &'a ExtensionData, - _turn_store: &'a ExtensionData, - item: &'a mut TurnItem, - ) -> TurnItemContributionFuture<'a> { - Box::pin(async move { - if let TurnItem::AgentMessage(agent_message) = item { - agent_message.content = vec![AgentMessageContent::Text { - text: "contributed assistant text".to_string(), - }]; - } - Ok(()) - }) + async fn contribute( + &self, + _thread_store: &ExtensionData, + _turn_store: &ExtensionData, + item: &mut TurnItem, + ) -> Result<(), String> { + if let TurnItem::AgentMessage(agent_message) = item { + agent_message.content = vec![AgentMessageContent::Text { + text: "contributed assistant text".to_string(), + }]; + } + Ok(()) } } diff --git a/codex-rs/ext/extension-api/src/contributors.rs b/codex-rs/ext/extension-api/src/contributors.rs index cbe8e58244..2113fdef8e 100644 --- a/codex-rs/ext/extension-api/src/contributors.rs +++ b/codex-rs/ext/extension-api/src/contributors.rs @@ -145,20 +145,17 @@ pub trait ApprovalReviewContributor: Send + Sync { ) -> Option; } -/// Future returned by one ordered turn-item contribution. -pub type TurnItemContributionFuture<'a> = - std::pin::Pin> + Send + 'a>>; - /// Ordered post-processing contribution for one parsed turn item. /// /// Implementations may mutate the item before it is emitted and may use the /// explicitly exposed thread- and turn-lifetime stores when they need durable /// extension-private state. +#[async_trait::async_trait] pub trait TurnItemContributor: Send + Sync { - fn contribute<'a>( - &'a self, - thread_store: &'a ExtensionData, - turn_store: &'a ExtensionData, - item: &'a mut TurnItem, - ) -> TurnItemContributionFuture<'a>; + async fn contribute( + &self, + thread_store: &ExtensionData, + turn_store: &ExtensionData, + item: &mut TurnItem, + ) -> Result<(), String>; } diff --git a/codex-rs/ext/extension-api/src/lib.rs b/codex-rs/ext/extension-api/src/lib.rs index f9af83d38d..2ec132ef86 100644 --- a/codex-rs/ext/extension-api/src/lib.rs +++ b/codex-rs/ext/extension-api/src/lib.rs @@ -35,7 +35,6 @@ pub use contributors::ToolLifecycleContributor; pub use contributors::ToolLifecycleFuture; pub use contributors::ToolStartInput; pub use contributors::TurnAbortInput; -pub use contributors::TurnItemContributionFuture; pub use contributors::TurnItemContributor; pub use contributors::TurnLifecycleContributor; pub use contributors::TurnStartInput; From 59507b849126f598ed7c624bbaf75d7ebc5588c2 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Wed, 20 May 2026 15:54:29 +0200 Subject: [PATCH 11/17] feat: expose turn-start metadata to extensions (#23688) ## Why The goal extension needs more context when a turn starts than `turn_store` alone provides. In particular, goal accounting needs the stable turn id, the effective collaboration mode, and the cumulative token-usage baseline captured at turn start so it can: - suppress goal accounting for plan-mode turns - compute exact per-turn deltas from cumulative `total_token_usage` snapshots instead of relying on the most recent usage event alone - keep the extension-owned accounting path aligned with the host turn lifecycle ## What - extend `codex_extension_api::TurnStartInput` to expose `turn_id`, `collaboration_mode`, and `token_usage_at_turn_start` - pass the full `TurnContext` plus the captured token-usage baseline through the turn-start lifecycle emission path - initialize goal turn accounting from the turn-start baseline and collaboration mode - switch goal token accounting to compute deltas from cumulative `total_token_usage` snapshots - add coverage for the new turn-start lifecycle fields and for goal-accounting baseline behavior ## Testing - added `turn_start_lifecycle_exposes_turn_metadata_and_token_baseline` in `codex-rs/core/src/session/tests.rs` - added `ext/goal/tests/accounting.rs` coverage for baseline-aware goal accounting and plan-mode suppression --- codex-rs/core/src/session/tests.rs | 102 ++++++++++++++++++ codex-rs/core/src/tasks/lifecycle.rs | 13 ++- codex-rs/core/src/tasks/mod.rs | 4 +- .../src/contributors/turn_lifecycle.rs | 8 ++ codex-rs/ext/goal/BUILD.bazel | 1 + codex-rs/ext/goal/src/accounting.rs | 52 +++++++-- codex-rs/ext/goal/src/extension.rs | 11 +- codex-rs/ext/goal/tests/accounting.rs | 68 ++++++++++++ 8 files changed, 242 insertions(+), 17 deletions(-) create mode 100644 codex-rs/ext/goal/tests/accounting.rs diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs index 175f022289..5318b72151 100644 --- a/codex-rs/core/src/session/tests.rs +++ b/codex-rs/core/src/session/tests.rs @@ -1911,6 +1911,108 @@ async fn record_token_usage_info_notifies_extension_contributors() { assert_eq!(expected, actual); } +#[tokio::test] +async fn turn_start_lifecycle_exposes_turn_metadata_and_token_baseline() { + struct SessionTurnStartMarker; + struct ThreadTurnStartMarker; + + #[derive(Debug, PartialEq, Eq)] + struct RecordedTurnStart { + turn_id: String, + collaboration_mode: CollaborationMode, + token_usage_at_turn_start: TokenUsage, + saw_session_store: bool, + saw_thread_store: bool, + } + + struct TurnStartRecorder { + records: Arc>>, + } + + #[async_trait::async_trait] + impl codex_extension_api::TurnLifecycleContributor for TurnStartRecorder { + async fn on_turn_start(&self, input: codex_extension_api::TurnStartInput<'_>) { + self.records + .lock() + .expect("turn start records lock") + .push(RecordedTurnStart { + turn_id: input.turn_id.to_string(), + collaboration_mode: input.collaboration_mode.clone(), + token_usage_at_turn_start: input.token_usage_at_turn_start.clone(), + saw_session_store: input + .session_store + .get::() + .is_some(), + saw_thread_store: input.thread_store.get::().is_some(), + }); + } + } + + let (mut session, turn_context) = make_session_and_context().await; + let records = Arc::new(std::sync::Mutex::new(Vec::new())); + let mut builder = codex_extension_api::ExtensionRegistryBuilder::::new(); + builder.turn_lifecycle_contributor(Arc::new(TurnStartRecorder { + records: Arc::clone(&records), + })); + session.services.extensions = Arc::new(builder.build()); + session + .services + .session_extension_data + .insert(SessionTurnStartMarker); + session + .services + .thread_extension_data + .insert(ThreadTurnStartMarker); + + let token_usage_at_turn_start = TokenUsage { + input_tokens: 120, + cached_input_tokens: 15, + output_tokens: 40, + reasoning_output_tokens: 9, + total_tokens: 169, + }; + session + .state + .lock() + .await + .set_token_info(Some(TokenUsageInfo { + total_token_usage: token_usage_at_turn_start.clone(), + last_token_usage: TokenUsage::default(), + model_context_window: turn_context.model_context_window(), + })); + + let turn_context = Arc::new(turn_context); + let session = Arc::new(session); + session + .spawn_task( + Arc::clone(&turn_context), + Vec::new(), + NeverEndingTask { + kind: TaskKind::Regular, + listen_to_cancellation_token: true, + }, + ) + .await; + + session.abort_all_tasks(TurnAbortReason::Interrupted).await; + + let actual = records + .lock() + .expect("turn start records lock") + .drain(..) + .collect::>(); + assert_eq!( + vec![RecordedTurnStart { + turn_id: turn_context.sub_id.clone(), + collaboration_mode: turn_context.collaboration_mode.clone(), + token_usage_at_turn_start, + saw_session_store: true, + saw_thread_store: true, + }], + actual + ); +} + #[tokio::test] async fn config_change_contributor_observes_effective_config_changes() { struct SessionConfigMarker; diff --git a/codex-rs/core/src/tasks/lifecycle.rs b/codex-rs/core/src/tasks/lifecycle.rs index 94b51647c9..8b934175cf 100644 --- a/codex-rs/core/src/tasks/lifecycle.rs +++ b/codex-rs/core/src/tasks/lifecycle.rs @@ -1,16 +1,25 @@ use codex_extension_api::ExtensionData; +use codex_protocol::protocol::TokenUsage; use codex_protocol::protocol::TurnAbortReason; use crate::session::session::Session; +use crate::session::turn_context::TurnContext; impl Session { - pub(super) async fn emit_turn_start_lifecycle(&self, turn_store: &ExtensionData) { + pub(super) async fn emit_turn_start_lifecycle( + &self, + turn_context: &TurnContext, + token_usage_at_turn_start: &TokenUsage, + ) { for contributor in self.services.extensions.turn_lifecycle_contributors() { contributor .on_turn_start(codex_extension_api::TurnStartInput { + turn_id: turn_context.sub_id.as_str(), + collaboration_mode: &turn_context.collaboration_mode, + token_usage_at_turn_start, session_store: &self.services.session_extension_data, thread_store: &self.services.thread_extension_data, - turn_store, + turn_store: turn_context.extension_data.as_ref(), }) .await; } diff --git a/codex-rs/core/src/tasks/mod.rs b/codex-rs/core/src/tasks/mod.rs index 164e65f07c..80ddfee897 100644 --- a/codex-rs/core/src/tasks/mod.rs +++ b/codex-rs/core/src/tasks/mod.rs @@ -357,7 +357,7 @@ impl Session { debug_assert!(turn.tasks.is_empty()); Arc::clone(&turn.turn_state) }; - turn_state.lock().await.token_usage_at_turn_start = token_usage_at_turn_start; + turn_state.lock().await.token_usage_at_turn_start = token_usage_at_turn_start.clone(); let mut pending_items = queued_response_items .into_iter() .map(TurnInput::ResponseInputItem) @@ -366,7 +366,7 @@ impl Session { self.input_queue .extend_pending_input_for_turn_state(turn_state.as_ref(), pending_items) .await; - self.emit_turn_start_lifecycle(turn_context.extension_data.as_ref()) + self.emit_turn_start_lifecycle(turn_context.as_ref(), &token_usage_at_turn_start) .await; let turn_extension_data = Arc::clone(&turn_context.extension_data); diff --git a/codex-rs/ext/extension-api/src/contributors/turn_lifecycle.rs b/codex-rs/ext/extension-api/src/contributors/turn_lifecycle.rs index 0b53183e28..bbd3ae8f39 100644 --- a/codex-rs/ext/extension-api/src/contributors/turn_lifecycle.rs +++ b/codex-rs/ext/extension-api/src/contributors/turn_lifecycle.rs @@ -1,9 +1,17 @@ +use codex_protocol::config_types::CollaborationMode; +use codex_protocol::protocol::TokenUsage; use codex_protocol::protocol::TurnAbortReason; use crate::ExtensionData; /// Input supplied when the host starts a turn. pub struct TurnStartInput<'a> { + /// Stable host-owned turn identifier. + pub turn_id: &'a str, + /// Effective collaboration mode for this turn. + pub collaboration_mode: &'a CollaborationMode, + /// Total token usage snapshot captured when the turn started. + pub token_usage_at_turn_start: &'a TokenUsage, /// Store scoped to the host session runtime. pub session_store: &'a ExtensionData, /// Store scoped to this thread runtime. diff --git a/codex-rs/ext/goal/BUILD.bazel b/codex-rs/ext/goal/BUILD.bazel index 037313da37..05b80c3706 100644 --- a/codex-rs/ext/goal/BUILD.bazel +++ b/codex-rs/ext/goal/BUILD.bazel @@ -3,4 +3,5 @@ load("//:defs.bzl", "codex_rust_crate") codex_rust_crate( name = "goal", crate_name = "codex_goal_extension", + integration_compile_data_extra = ["src/accounting.rs"], ) diff --git a/codex-rs/ext/goal/src/accounting.rs b/codex-rs/ext/goal/src/accounting.rs index 712e325849..cc04684169 100644 --- a/codex-rs/ext/goal/src/accounting.rs +++ b/codex-rs/ext/goal/src/accounting.rs @@ -1,3 +1,4 @@ +use codex_protocol::config_types::ModeKind; use codex_protocol::protocol::TokenUsage; use std::collections::HashMap; use std::sync::Mutex; @@ -17,6 +18,8 @@ struct GoalAccountingInner { #[derive(Debug, Default)] struct GoalTurnAccounting { token_delta: i64, + last_accounted_token_usage: TokenUsage, + account_tokens: bool, stopped: bool, } @@ -27,24 +30,42 @@ pub(crate) struct RecordedTokenDelta { } impl GoalAccountingState { - pub(crate) fn start_turn(&self, turn_id: impl Into) { + pub(crate) fn start_turn( + &self, + turn_id: impl Into, + collaboration_mode: ModeKind, + token_usage_at_turn_start: &TokenUsage, + ) { let turn_id = turn_id.into(); - self.inner().turns.entry(turn_id).or_default().stopped = false; + self.inner().turns.insert( + turn_id, + GoalTurnAccounting { + token_delta: 0, + last_accounted_token_usage: token_usage_at_turn_start.clone(), + account_tokens: !matches!(collaboration_mode, ModeKind::Plan), + stopped: false, + }, + ); } pub(crate) fn record_token_usage( &self, turn_id: impl Into, - usage: &TokenUsage, + total_usage: &TokenUsage, ) -> Option { - let delta = goal_token_delta_for_usage(usage); - if delta <= 0 { + let turn_id = turn_id.into(); + let mut inner = self.inner(); + let turn = inner.turns.get_mut(&turn_id)?; + if turn.stopped || !turn.account_tokens { return None; } - let turn_id = turn_id.into(); - let mut inner = self.inner(); - let turn = inner.turns.entry(turn_id).or_default(); + let delta = + token_delta_since_last_accounting(&turn.last_accounted_token_usage, total_usage); + turn.last_accounted_token_usage = total_usage.clone(); + if delta <= 0 { + return None; + } turn.token_delta = turn.token_delta.saturating_add(delta); let turn_delta = turn.token_delta; inner.unflushed_token_delta = inner.unflushed_token_delta.saturating_add(delta); @@ -65,6 +86,21 @@ impl GoalAccountingState { } } +fn token_delta_since_last_accounting(last: &TokenUsage, current: &TokenUsage) -> i64 { + let delta = TokenUsage { + input_tokens: current.input_tokens.saturating_sub(last.input_tokens), + cached_input_tokens: current + .cached_input_tokens + .saturating_sub(last.cached_input_tokens), + output_tokens: current.output_tokens.saturating_sub(last.output_tokens), + reasoning_output_tokens: current + .reasoning_output_tokens + .saturating_sub(last.reasoning_output_tokens), + total_tokens: current.total_tokens.saturating_sub(last.total_tokens), + }; + goal_token_delta_for_usage(&delta) +} + pub(crate) fn goal_token_delta_for_usage(usage: &TokenUsage) -> i64 { usage .input_tokens diff --git a/codex-rs/ext/goal/src/extension.rs b/codex-rs/ext/goal/src/extension.rs index 657d285f74..6d8eb42f0e 100644 --- a/codex-rs/ext/goal/src/extension.rs +++ b/codex-rs/ext/goal/src/extension.rs @@ -108,10 +108,11 @@ where return; } - // TODO: TurnStartInput should expose collaboration mode and token usage - // at turn start. Goals need mode to suppress plan-mode accounting and - // the token baseline to account deltas exactly. - accounting_state(input.thread_store).start_turn(input.turn_store.level_id()); + accounting_state(input.thread_store).start_turn( + input.turn_id, + input.collaboration_mode.mode, + input.token_usage_at_turn_start, + ); } async fn on_turn_stop(&self, input: TurnStopInput<'_>) { @@ -158,7 +159,7 @@ where } let Some(_recorded) = accounting_state(thread_store) - .record_token_usage(turn_store.level_id(), &token_usage.last_token_usage) + .record_token_usage(turn_store.level_id(), &token_usage.total_token_usage) else { return; }; diff --git a/codex-rs/ext/goal/tests/accounting.rs b/codex-rs/ext/goal/tests/accounting.rs new file mode 100644 index 0000000000..99e9c93005 --- /dev/null +++ b/codex-rs/ext/goal/tests/accounting.rs @@ -0,0 +1,68 @@ +#![allow(dead_code)] + +#[path = "../src/accounting.rs"] +mod accounting; + +use accounting::GoalAccountingState; +use codex_protocol::config_types::ModeKind; +use codex_protocol::protocol::TokenUsage; +use pretty_assertions::assert_eq; + +#[test] +fn goal_accounting_uses_turn_start_baseline_for_exact_deltas() { + let state = GoalAccountingState::default(); + state.start_turn( + "turn-1", + ModeKind::Default, + &token_usage( + /*input_tokens*/ 100, /*cached_input_tokens*/ 10, /*output_tokens*/ 30, + /*reasoning_output_tokens*/ 5, /*total_tokens*/ 135, + ), + ); + + let recorded = state + .record_token_usage( + "turn-1", + &token_usage( + /*input_tokens*/ 120, /*cached_input_tokens*/ 14, + /*output_tokens*/ 42, /*reasoning_output_tokens*/ 8, + /*total_tokens*/ 162, + ), + ) + .expect("token delta should be recorded"); + + assert_eq!(28, recorded.turn_delta); + assert_eq!(28, recorded.thread_unflushed_delta); +} + +#[test] +fn goal_accounting_ignores_plan_mode_turns() { + let state = GoalAccountingState::default(); + state.start_turn("turn-1", ModeKind::Plan, &TokenUsage::default()); + + let recorded = state.record_token_usage( + "turn-1", + &token_usage( + /*input_tokens*/ 20, /*cached_input_tokens*/ 5, /*output_tokens*/ 8, + /*reasoning_output_tokens*/ 2, /*total_tokens*/ 30, + ), + ); + + assert_eq!(None, recorded); +} + +fn token_usage( + input_tokens: i64, + cached_input_tokens: i64, + output_tokens: i64, + reasoning_output_tokens: i64, + total_tokens: i64, +) -> TokenUsage { + TokenUsage { + input_tokens, + cached_input_tokens, + output_tokens, + reasoning_output_tokens, + total_tokens, + } +} From a52c91d8b59601d1b01e8ccfffc2ae0d5dde31df Mon Sep 17 00:00:00 2001 From: pakrym-oai Date: Wed, 20 May 2026 08:09:45 -0700 Subject: [PATCH 12/17] [codex] Hide deferred tools from code mode prompt (#23605) ## Why `code_mode_only_guides_all_tools_search_and_calls_deferred_app_tools` was failing because code-mode prompt generation used the same nested tool spec list for both the model-visible `exec` guide and the runtime `ALL_TOOLS` surface. That allowed deferred MCP/app tools, such as `calendar_timezone_option_99`, to leak into the `exec` description even though they should only be discoverable through `ALL_TOOLS` at runtime. ## What changed Split code-mode nested tool planning into two sets in `core/src/tools/spec_plan.rs`: - runtime nested tool specs still include deferred tools, so `tools[...]` and `ALL_TOOLS` can call them - `exec` prompt docs only render non-deferred tools, so deferred app tools stay out of the model-visible guide ## Validation - `cargo test -p codex-core --test all code_mode_only_guides_all_tools_search_and_calls_deferred_app_tools -- --nocapture` - looped the same focused test 5 additional times with `cargo test -q -p codex-core --test all code_mode_only_guides_all_tools_search_and_calls_deferred_app_tools` --- codex-rs/core/src/tools/spec_plan.rs | 30 ++++++++++++++++++---------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/codex-rs/core/src/tools/spec_plan.rs b/codex-rs/core/src/tools/spec_plan.rs index ab9d02f97c..a47ed9da34 100644 --- a/codex-rs/core/src/tools/spec_plan.rs +++ b/codex-rs/core/src/tools/spec_plan.rs @@ -367,19 +367,27 @@ fn build_code_mode_executors( return vec![]; } - let code_mode_nested_tool_specs = executors - .iter() - .filter_map(|executor| { - if executor.exposure() == ToolExposure::DirectModelOnly { - return None; - } + let mut code_mode_nested_tool_specs = Vec::new(); + let mut exec_prompt_tool_specs = Vec::new(); + for executor in executors { + let exposure = executor.exposure(); + if exposure == ToolExposure::DirectModelOnly { + continue; + } - executor.spec() - }) - .collect::>(); - let namespace_descriptions = code_mode_namespace_descriptions(&code_mode_nested_tool_specs); + let Some(spec) = executor.spec() else { + continue; + }; + + if exposure != ToolExposure::Deferred { + exec_prompt_tool_specs.push(spec.clone()); + } + code_mode_nested_tool_specs.push(spec); + } + + let namespace_descriptions = code_mode_namespace_descriptions(&exec_prompt_tool_specs); let mut enabled_tools = - collect_code_mode_exec_prompt_tool_definitions(code_mode_nested_tool_specs.iter()); + collect_code_mode_exec_prompt_tool_definitions(exec_prompt_tool_specs.iter()); enabled_tools .sort_by(|left, right| compare_code_mode_tools(left, right, &namespace_descriptions)); From b0b383bea3ba7830e6c9af4bbb6224d5ab1a2fe5 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Wed, 20 May 2026 08:24:43 -0700 Subject: [PATCH 13/17] runtime: use install context for bundled bwrap (#23634) ## Summary The Linux sandbox should find bundled `bwrap` through the same package-layout abstraction as the rest of the runtime, instead of maintaining a separate standalone-specific lookup path. This adds an `InstallContext` helper for bundled resources and updates `codex-linux-sandbox` to ask the current install context for `codex-resources/bwrap` before falling back to the old executable-relative probes. The tests cover npm-style, standalone, and canonical package layouts so `bwrap` lookup follows the package structure introduced earlier in the stack. ## Test plan - `cargo test -p codex-install-context` - `cargo test -p codex-linux-sandbox --lib` - `just fix -p codex-install-context -p codex-linux-sandbox` - `just bazel-lock-check` --- [//]: # (BEGIN SAPLING FOOTER) Stack created with [Sapling](https://sapling-scm.com). Best reviewed with [ReviewStack](https://reviewstack.dev/openai/codex/pull/23634). * #23638 * #23637 * #23636 * #23635 * __->__ #23634 --- codex-rs/Cargo.lock | 1 + codex-rs/install-context/src/lib.rs | 76 +++++++++++++++++++-- codex-rs/linux-sandbox/Cargo.toml | 1 + codex-rs/linux-sandbox/src/bundled_bwrap.rs | 56 ++++++++++++--- 4 files changed, 121 insertions(+), 13 deletions(-) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 6dfc44d844..7ea7dce4d4 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -3024,6 +3024,7 @@ version = "0.0.0" dependencies = [ "clap", "codex-core", + "codex-install-context", "codex-process-hardening", "codex-protocol", "codex-sandboxing", diff --git a/codex-rs/install-context/src/lib.rs b/codex-rs/install-context/src/lib.rs index d1f8d3839b..ec5c5217ca 100644 --- a/codex-rs/install-context/src/lib.rs +++ b/codex-rs/install-context/src/lib.rs @@ -124,7 +124,7 @@ impl InstallContext { && let Some(path_dir) = &package_layout.path_dir { let bundled_rg = path_dir.join(default_rg_command()); - if bundled_rg.exists() { + if bundled_rg.is_file() { return bundled_rg.into_path_buf(); } } @@ -135,13 +135,37 @@ impl InstallContext { } = &self.method { let bundled_rg = resources_dir.join(default_rg_command()); - if bundled_rg.exists() { + if bundled_rg.is_file() { return bundled_rg.into_path_buf(); } } default_rg_command() } + + pub fn bundled_resource(&self, file_name: impl AsRef) -> Option { + if let Some(package_layout) = &self.package_layout + && let Some(resources_dir) = &package_layout.resources_dir + { + let resource = resources_dir.join(file_name.as_ref()); + if resource.is_file() { + return Some(resource); + } + } + + if let InstallMethod::Standalone { + resources_dir: Some(resources_dir), + .. + } = &self.method + { + let resource = resources_dir.join(file_name); + if resource.is_file() { + return Some(resource); + } + } + + None + } } impl CodexPackageLayout { @@ -242,6 +266,8 @@ mod tests { use pretty_assertions::assert_eq; use std::fs; + const TEST_RESOURCE_NAME: &str = "codex-test-helper"; + #[test] fn detects_standalone_install_from_release_layout() -> std::io::Result<()> { let codex_home = tempfile::tempdir()?; @@ -253,6 +279,7 @@ mod tests { let exe_path = release_dir.join(if cfg!(windows) { "codex.exe" } else { "codex" }); fs::write(&exe_path, "")?; fs::write(resources_dir.join(default_rg_command()), "")?; + fs::write(resources_dir.join(TEST_RESOURCE_NAME), "")?; let canonical_release_dir = AbsolutePathBuf::from_absolute_path(release_dir.canonicalize()?)?; let canonical_resources_dir = @@ -270,12 +297,16 @@ mod tests { InstallContext { method: InstallMethod::Standalone { release_dir: canonical_release_dir, - resources_dir: Some(canonical_resources_dir), + resources_dir: Some(canonical_resources_dir.clone()), platform: standalone_platform(), }, package_layout: None, } ); + assert_eq!( + context.bundled_resource(TEST_RESOURCE_NAME), + Some(canonical_resources_dir.join(TEST_RESOURCE_NAME)) + ); Ok(()) } @@ -312,6 +343,7 @@ mod tests { fs::write(package_dir.path().join(PACKAGE_METADATA_FILENAME), "{}")?; let exe_path = bin_dir.join(if cfg!(windows) { "codex.exe" } else { "codex" }); fs::write(&exe_path, "")?; + fs::write(resources_dir.join(TEST_RESOURCE_NAME), "")?; fs::write(path_dir.join(default_rg_command()), "")?; let canonical_package_dir = AbsolutePathBuf::from_absolute_path(package_dir.path().canonicalize()?)?; @@ -322,7 +354,7 @@ mod tests { let package_layout = CodexPackageLayout { package_dir: canonical_package_dir, bin_dir: canonical_bin_dir, - resources_dir: Some(canonical_resources_dir), + resources_dir: Some(canonical_resources_dir.clone()), path_dir: Some(canonical_path_dir.clone()), }; @@ -346,6 +378,10 @@ mod tests { .join(default_rg_command()) .into_path_buf() ); + assert_eq!( + context.bundled_resource(TEST_RESOURCE_NAME), + Some(canonical_resources_dir.join(TEST_RESOURCE_NAME)) + ); Ok(()) } @@ -364,6 +400,7 @@ mod tests { fs::write(package_dir.join(PACKAGE_METADATA_FILENAME), "{}")?; let exe_path = bin_dir.join(if cfg!(windows) { "codex.exe" } else { "codex" }); fs::write(&exe_path, "")?; + fs::write(resources_dir.join(TEST_RESOURCE_NAME), "")?; fs::write(path_dir.join(default_rg_command()), "")?; let canonical_package_dir = AbsolutePathBuf::from_absolute_path(package_dir.canonicalize()?)?; @@ -390,7 +427,7 @@ mod tests { package_layout: Some(CodexPackageLayout { package_dir: canonical_package_dir, bin_dir: canonical_bin_dir, - resources_dir: Some(canonical_resources_dir), + resources_dir: Some(canonical_resources_dir.clone()), path_dir: Some(canonical_path_dir.clone()), }), } @@ -401,6 +438,10 @@ mod tests { .join(default_rg_command()) .into_path_buf() ); + assert_eq!( + context.bundled_resource(TEST_RESOURCE_NAME), + Some(canonical_resources_dir.join(TEST_RESOURCE_NAME)) + ); Ok(()) } @@ -455,6 +496,31 @@ mod tests { Ok(()) } + #[test] + fn bundled_file_lookups_ignore_directories() -> std::io::Result<()> { + let package_dir = tempfile::tempdir()?; + let bin_dir = package_dir.path().join(BIN_DIRNAME); + let resources_dir = package_dir.path().join(RESOURCES_DIRNAME); + let path_dir = package_dir.path().join(PATH_DIRNAME); + fs::create_dir_all(&bin_dir)?; + fs::create_dir_all(resources_dir.join(TEST_RESOURCE_NAME))?; + fs::create_dir_all(path_dir.join(default_rg_command()))?; + fs::write(package_dir.path().join(PACKAGE_METADATA_FILENAME), "{}")?; + let exe_path = bin_dir.join(if cfg!(windows) { "codex.exe" } else { "codex" }); + fs::write(&exe_path, "")?; + + let context = InstallContext::from_exe_with_codex_home( + /*is_macos*/ false, + /*current_exe*/ Some(&exe_path), + /*managed_by_npm*/ false, + /*managed_by_bun*/ false, + /*codex_home*/ None, + ); + assert_eq!(context.rg_command(), default_rg_command()); + assert_eq!(context.bundled_resource(TEST_RESOURCE_NAME), None); + Ok(()) + } + #[test] fn npm_and_bun_take_precedence() { let npm_context = InstallContext::from_exe_with_codex_home( diff --git a/codex-rs/linux-sandbox/Cargo.toml b/codex-rs/linux-sandbox/Cargo.toml index 1ae2e6b5f5..fc7937536c 100644 --- a/codex-rs/linux-sandbox/Cargo.toml +++ b/codex-rs/linux-sandbox/Cargo.toml @@ -18,6 +18,7 @@ workspace = true [target.'cfg(target_os = "linux")'.dependencies] clap = { workspace = true, features = ["derive"] } +codex-install-context = { workspace = true } codex-process-hardening = { workspace = true } codex-protocol = { workspace = true } codex-sandboxing = { workspace = true } diff --git a/codex-rs/linux-sandbox/src/bundled_bwrap.rs b/codex-rs/linux-sandbox/src/bundled_bwrap.rs index 505377907f..972be9dac3 100644 --- a/codex-rs/linux-sandbox/src/bundled_bwrap.rs +++ b/codex-rs/linux-sandbox/src/bundled_bwrap.rs @@ -12,6 +12,7 @@ use std::sync::OnceLock; use crate::bazel_bwrap; use crate::exec_util::argv_to_cstrings; use crate::exec_util::make_files_inheritable; +use codex_install_context::InstallContext; use codex_utils_absolute_path::AbsolutePathBuf; use sha2::Digest as _; use sha2::Sha256; @@ -26,7 +27,9 @@ pub(crate) struct BundledBwrapLauncher { pub(crate) fn launcher() -> Option { let current_exe = std::env::current_exe().ok()?; - find_for_exe(¤t_exe).map(|program| BundledBwrapLauncher { program }) + find_for_install_context(InstallContext::current()) + .or_else(|| find_legacy_for_exe(¤t_exe)) + .map(|program| BundledBwrapLauncher { program }) } impl BundledBwrapLauncher { @@ -66,8 +69,14 @@ impl BundledBwrapLauncher { } } -fn find_for_exe(exe: &Path) -> Option { - candidates_for_exe(exe) +fn find_for_install_context(context: &InstallContext) -> Option { + context + .bundled_resource("bwrap") + .filter(|path| is_executable_file(path)) +} + +fn find_legacy_for_exe(exe: &Path) -> Option { + legacy_candidates_for_exe(exe) .into_iter() .find(|candidate| is_executable_file(candidate)) .map(|path| { @@ -80,7 +89,7 @@ fn find_for_exe(exe: &Path) -> Option { }) } -fn candidates_for_exe(exe: &Path) -> Vec { +fn legacy_candidates_for_exe(exe: &Path) -> Vec { let Some(exe_dir) = exe.parent() else { return Vec::new(); }; @@ -180,13 +189,44 @@ fn bytes_to_hex(bytes: &[u8; 32]) -> String { #[cfg(test)] mod tests { use super::*; + use codex_install_context::CodexPackageLayout; + use codex_install_context::InstallContext; + use codex_install_context::InstallMethod; use pretty_assertions::assert_eq; use std::fs; use tempfile::NamedTempFile; use tempfile::tempdir; #[test] - fn finds_standalone_bundled_bwrap_next_to_exe_resources() { + fn finds_package_layout_bwrap_from_install_context() { + let temp_dir = tempdir().expect("temp dir"); + let package_dir = temp_dir.path(); + let bin_dir = package_dir.join("bin"); + let resources_dir = package_dir.join("codex-resources"); + let expected_bwrap = resources_dir.join("bwrap"); + fs::create_dir_all(&bin_dir).expect("create bin dir"); + write_executable(&expected_bwrap); + + let context = InstallContext { + method: InstallMethod::Other, + package_layout: Some(CodexPackageLayout { + package_dir: AbsolutePathBuf::from_absolute_path(package_dir).expect("absolute"), + bin_dir: AbsolutePathBuf::from_absolute_path(&bin_dir).expect("absolute"), + resources_dir: Some( + AbsolutePathBuf::from_absolute_path(&resources_dir).expect("absolute"), + ), + path_dir: None, + }), + }; + + assert_eq!( + find_for_install_context(&context), + Some(AbsolutePathBuf::from_absolute_path(&expected_bwrap).expect("absolute")) + ); + } + + #[test] + fn finds_legacy_standalone_bundled_bwrap_next_to_exe_resources() { let temp_dir = tempdir().expect("temp dir"); let exe = temp_dir.path().join("codex"); let expected_bwrap = temp_dir.path().join("codex-resources").join("bwrap"); @@ -194,7 +234,7 @@ mod tests { write_executable(&expected_bwrap); assert_eq!( - find_for_exe(&exe), + find_legacy_for_exe(&exe), Some(AbsolutePathBuf::from_absolute_path(&expected_bwrap).expect("absolute")) ); } @@ -209,7 +249,7 @@ mod tests { write_executable(&expected_bwrap); assert_eq!( - find_for_exe(&exe), + find_legacy_for_exe(&exe), Some(AbsolutePathBuf::from_absolute_path(&expected_bwrap).expect("absolute")) ); } @@ -223,7 +263,7 @@ mod tests { write_executable(&expected_bwrap); assert_eq!( - find_for_exe(&exe), + find_legacy_for_exe(&exe), Some(AbsolutePathBuf::from_absolute_path(&expected_bwrap).expect("absolute")) ); } From 1d3e8423631d7fa81539deecd719a816823ab47c Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Wed, 20 May 2026 08:24:55 -0700 Subject: [PATCH 14/17] release: publish Codex package archive checksums ## Summary Standalone installers and other downstream package consumers need a stable checksum source for the canonical package archives. Relying on per-asset metadata makes that harder to consume uniformly, especially when several package archives are produced in the same release. This keeps the `codex-package-*.tar.gz` and `codex-app-server-package-*.tar.gz` assets in the GitHub Release upload set and adds `codex-package_SHA256SUMS` to `dist/` before the release is created. The manifest contains one SHA-256 line per package archive and fails the release job if no package archives are present. ## Test plan - `ruby -e 'require "yaml"; YAML.load_file(".github/workflows/rust-release.yml"); puts "ok"'` --- .github/workflows/rust-release.yml | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/.github/workflows/rust-release.yml b/.github/workflows/rust-release.yml index c55337ecfe..93e23b8fd7 100644 --- a/.github/workflows/rust-release.yml +++ b/.github/workflows/rust-release.yml @@ -1107,15 +1107,33 @@ jobs: # If included in files: dist/**, release upload races on duplicate # asset names and can fail with 404s. find dist -type f -name 'cargo-timing.html' -delete - # Keep package-builder sidecar archives as workflow artifacts only - # until distribution channels are ready to consume them. - find dist -type f \ - \( -name 'codex-package-*' -o -name 'codex-app-server-package-*' \) \ - -delete find dist -type d -empty -delete ls -R dist/ + - name: Add Codex package checksum manifest + run: | + set -euo pipefail + + manifest="dist/codex-package_SHA256SUMS" + tmp_manifest="$(mktemp)" + find dist -type f \ + \( -name 'codex-package-*.tar.gz' -o -name 'codex-app-server-package-*.tar.gz' \) \ + -print | + sort | + while IFS= read -r archive; do + sha256sum "$archive" | + awk -v name="$(basename "$archive")" '{ print $1 " " name }' + done > "$tmp_manifest" + + if [[ ! -s "$tmp_manifest" ]]; then + echo "No Codex package archives found for checksum manifest" + exit 1 + fi + + mv "$tmp_manifest" "$manifest" + cat "$manifest" + - name: Add config schema release asset run: | cp codex-rs/core/config.schema.json dist/config-schema.json From e7812d48ed679ab3c071195c718e6d0b4d8dcd43 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Wed, 20 May 2026 08:24:55 -0700 Subject: [PATCH 15/17] install: consume Codex package archives ## Summary Standalone installs should exercise the same canonical package archive layout that release builds produce, rather than unpacking npm platform packages and reconstructing a parallel install tree. This updates `install.sh` and `install.ps1` to download `codex-package-.tar.gz` plus `codex-package_SHA256SUMS`, verify the selected archive against the manifest, install the archive directly, and point visible command shims at `bin/codex` inside the package layout. Existing installs are still version-detected through the old root-level `codex` path as a fallback during migration. ## Test plan - `sh -n scripts/install/install.sh` - `pwsh -NoProfile -Command ''` --- scripts/install/install.ps1 | 102 ++++++++++++++++++++++------------- scripts/install/install.sh | 104 ++++++++++++++---------------------- 2 files changed, 103 insertions(+), 103 deletions(-) diff --git a/scripts/install/install.ps1 b/scripts/install/install.ps1 index ed4a3e1a32..fdb26868fc 100644 --- a/scripts/install/install.ps1 +++ b/scripts/install/install.ps1 @@ -86,10 +86,27 @@ function Test-ArchiveDigest { $actualDigest = (Get-FileHash -LiteralPath $ArchivePath -Algorithm SHA256).Hash.ToLowerInvariant() if ($actualDigest -ne $ExpectedDigest) { - throw "Downloaded Codex archive checksum did not match release metadata. Expected $ExpectedDigest but got $actualDigest." + throw "Downloaded Codex archive checksum did not match expected digest. Expected $ExpectedDigest but got $actualDigest." } } +function Get-PackageArchiveDigest { + param( + [string]$ManifestPath, + [string]$AssetName + ) + + $escapedAssetName = [regex]::Escape($AssetName) + foreach ($line in Get-Content -LiteralPath $ManifestPath) { + $match = [regex]::Match($line, "^\s*([0-9a-fA-F]{64})\s+$escapedAssetName\s*$") + if ($match.Success) { + return $match.Groups[1].Value.ToLowerInvariant() + } + } + + throw "Could not find SHA-256 digest for $AssetName in codex-package_SHA256SUMS." +} + function Path-Contains { param( [string]$PathValue, @@ -190,6 +207,11 @@ function Get-CurrentInstalledVersion { [string]$StandaloneCurrentDir ) + $standaloneVersion = Get-VersionFromBinary -CodexPath (Join-Path $StandaloneCurrentDir "bin\codex.exe") + if (-not [string]::IsNullOrWhiteSpace($standaloneVersion)) { + return $standaloneVersion + } + $standaloneVersion = Get-VersionFromBinary -CodexPath (Join-Path $StandaloneCurrentDir "codex.exe") if (-not [string]::IsNullOrWhiteSpace($standaloneVersion)) { return $standaloneVersion @@ -449,6 +471,31 @@ function Ensure-Junction { throw "Refusing to replace file at $LinkPath with a junction." } +function Test-PackageContentsAreComplete { + param( + [string]$PackageDir + ) + + if (-not (Test-Path -LiteralPath $PackageDir -PathType Container)) { + return $false + } + + $expectedFiles = @( + "codex-package.json", + "bin\codex.exe", + "codex-path\rg.exe", + "codex-resources\codex-command-runner.exe", + "codex-resources\codex-windows-sandbox-setup.exe" + ) + foreach ($name in $expectedFiles) { + if (-not (Test-Path -LiteralPath (Join-Path $PackageDir $name) -PathType Leaf)) { + return $false + } + } + + return $true +} + function Test-ReleaseIsComplete { param( [string]$ReleaseDir, @@ -456,22 +503,10 @@ function Test-ReleaseIsComplete { [string]$ExpectedTarget ) - if (-not (Test-Path -LiteralPath $ReleaseDir -PathType Container)) { + if (-not (Test-PackageContentsAreComplete -PackageDir $ReleaseDir)) { return $false } - $expectedFiles = @( - "codex.exe", - "codex-resources\codex-command-runner.exe", - "codex-resources\codex-windows-sandbox-setup.exe", - "codex-resources\rg.exe" - ) - foreach ($name in $expectedFiles) { - if (-not (Test-Path -LiteralPath (Join-Path $ReleaseDir $name) -PathType Leaf)) { - return $false - } - } - return (Split-Path -Leaf $ReleaseDir) -eq "$ExpectedVersion-$ExpectedTarget" } @@ -584,17 +619,14 @@ if (-not [Environment]::Is64BitOperatingSystem) { $architecture = [System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture $target = $null $platformLabel = $null -$npmTag = $null switch ($architecture) { "Arm64" { $target = "aarch64-pc-windows-msvc" $platformLabel = "Windows (ARM64)" - $npmTag = "win32-arm64" } "X64" { $target = "x86_64-pc-windows-msvc" $platformLabel = "Windows (x64)" - $npmTag = "win32-x64" } default { Write-Error "Unsupported architecture: $architecture" @@ -637,7 +669,8 @@ Write-Step "Resolved version: $resolvedVersion" $conflictingInstall = Get-ConflictingInstall -VisibleBinDir $visibleBinDir $oldStandaloneBackup = $null -$packageAsset = "codex-npm-$npmTag-$resolvedVersion.tgz" +$packageAsset = "codex-package-$target.tar.gz" +$checksumAsset = "codex-package_SHA256SUMS" $tempDir = Join-Path ([System.IO.Path]::GetTempPath()) ("codex-install-" + [System.Guid]::NewGuid().ToString("N")) New-Item -ItemType Directory -Force -Path $tempDir | Out-Null @@ -651,34 +684,26 @@ try { } $archivePath = Join-Path $tempDir $packageAsset - $extractDir = Join-Path $tempDir "extract" + $checksumPath = Join-Path $tempDir $checksumAsset $stagingDir = Join-Path $releasesDir ".staging.$releaseName.$PID" - $assetMetadata = Get-ReleaseAssetMetadata -AssetName $packageAsset -ResolvedVersion $resolvedVersion + $packageMetadata = Get-ReleaseAssetMetadata -AssetName $packageAsset -ResolvedVersion $resolvedVersion + $checksumMetadata = Get-ReleaseAssetMetadata -AssetName $checksumAsset -ResolvedVersion $resolvedVersion Write-Step "Downloading Codex CLI" - Invoke-WebRequest -Uri $assetMetadata.Url -OutFile $archivePath - Test-ArchiveDigest -ArchivePath $archivePath -ExpectedDigest $assetMetadata.Sha256 + Invoke-WebRequest -Uri $checksumMetadata.Url -OutFile $checksumPath + Test-ArchiveDigest -ArchivePath $checksumPath -ExpectedDigest $checksumMetadata.Sha256 + $expectedPackageDigest = Get-PackageArchiveDigest -ManifestPath $checksumPath -AssetName $packageAsset + Invoke-WebRequest -Uri $packageMetadata.Url -OutFile $archivePath + Test-ArchiveDigest -ArchivePath $archivePath -ExpectedDigest $expectedPackageDigest - New-Item -ItemType Directory -Force -Path $extractDir | Out-Null New-Item -ItemType Directory -Force -Path $releasesDir | Out-Null if (Test-Path -LiteralPath $stagingDir) { Remove-Item -LiteralPath $stagingDir -Recurse -Force } New-Item -ItemType Directory -Force -Path $stagingDir | Out-Null - tar -xzf $archivePath -C $extractDir - - $vendorRoot = Join-Path $extractDir "package/vendor/$target" - $resourcesDir = Join-Path $stagingDir "codex-resources" - New-Item -ItemType Directory -Force -Path $resourcesDir | Out-Null - $copyMap = @{ - "codex/codex.exe" = "codex.exe" - "codex/codex-command-runner.exe" = "codex-resources\codex-command-runner.exe" - "codex/codex-windows-sandbox-setup.exe" = "codex-resources\codex-windows-sandbox-setup.exe" - "path/rg.exe" = "codex-resources\rg.exe" - } - - foreach ($relativeSource in $copyMap.Keys) { - Copy-Item -LiteralPath (Join-Path $vendorRoot $relativeSource) -Destination (Join-Path $stagingDir $copyMap[$relativeSource]) + tar -xzf $archivePath -C $stagingDir + if (-not (Test-PackageContentsAreComplete -PackageDir $stagingDir)) { + throw "Downloaded Codex package archive did not contain the expected package layout." } if (Test-Path -LiteralPath $releaseDir) { @@ -691,10 +716,11 @@ try { Ensure-Junction -LinkPath $currentDir -TargetPath $releaseDir -InstallerOwnedTargetPrefix $releasesDir $visibleParent = Split-Path -Parent $visibleBinDir + $currentBinDir = Join-Path $currentDir "bin" New-Item -ItemType Directory -Force -Path $visibleParent | Out-Null $oldStandaloneBackup = Move-OldStandaloneBinIfApproved -VisibleBinDir $visibleBinDir -DefaultVisibleBinDir $defaultVisibleBinDir try { - Ensure-Junction -LinkPath $visibleBinDir -TargetPath $currentDir -InstallerOwnedTargetPrefix $standaloneRoot + Ensure-Junction -LinkPath $visibleBinDir -TargetPath $currentBinDir -InstallerOwnedTargetPrefix $standaloneRoot Test-VisibleCodexCommand -VisibleBinDir $visibleBinDir } catch { if ($null -ne $oldStandaloneBackup -and (Test-Path -LiteralPath $oldStandaloneBackup)) { diff --git a/scripts/install/install.sh b/scripts/install/install.sh index 2fc585d7e9..0119b79ff2 100755 --- a/scripts/install/install.sh +++ b/scripts/install/install.sh @@ -114,55 +114,29 @@ release_url_for_asset() { printf 'https://github.com/openai/codex/releases/download/rust-v%s/%s\n' "$resolved_version" "$asset" } -release_metadata_url() { - resolved_version="$1" - - printf 'https://api.github.com/repos/openai/codex/releases/tags/rust-v%s\n' "$resolved_version" -} - -release_asset_digest() { +package_archive_digest() { asset="$1" - resolved_version="$2" - release_json="$(download_text "$(release_metadata_url "$resolved_version")")" + manifest_path="$2" - digest="$(printf '%s\n' "$release_json" | awk -v asset="$asset" ' - { - if ($0 ~ "\"name\":[[:space:]]*\"" asset "\"") { - in_asset = 1 - asset_depth = depth - } - - if (in_asset && /"digest":[[:space:]]*"[^"]+"/) { - sub(/^.*"digest":[[:space:]]*"/, "") - sub(/".*$/, "") - digest = $0 - } - - line = $0 - opens = gsub(/\{/, "{", line) - closes = gsub(/\}/, "}", line) - depth += opens - closes - - if (in_asset && depth < asset_depth) { - in_asset = 0 - } + digest="$(awk -v asset="$asset" ' + $2 == asset && $1 ~ /^[0-9a-fA-F]{64}$/ { + print tolower($1) + found = 1 + exit } END { - if (digest != "") { - print digest + if (!found) { + exit 1 } } - ')" + ' "$manifest_path" 2>/dev/null || true)" - case "$digest" in - sha256:????????????????????????????????????????????????????????????????) - printf '%s\n' "${digest#sha256:}" - ;; - *) - echo "Could not find SHA-256 digest for release asset $asset." >&2 - exit 1 - ;; - esac + if [ -z "$digest" ]; then + echo "Could not find SHA-256 digest for $asset in codex-package_SHA256SUMS." >&2 + exit 1 + fi + + printf '%s\n' "$digest" } file_sha256() { @@ -193,7 +167,7 @@ verify_archive_digest() { actual_digest="$(file_sha256 "$archive_path")" if [ "$actual_digest" != "$expected_digest" ]; then - echo "Downloaded Codex archive checksum did not match release metadata." >&2 + echo "Downloaded Codex archive checksum did not match release checksum manifest." >&2 echo "expected: $expected_digest" >&2 echo "actual: $actual_digest" >&2 exit 1 @@ -441,6 +415,12 @@ version_from_binary() { } current_installed_version() { + version="$(version_from_binary "$CURRENT_LINK/bin/codex" || true)" + if [ -n "$version" ]; then + printf '%s\n' "$version" + return 0 + fi + version="$(version_from_binary "$CURRENT_LINK/codex" || true)" if [ -n "$version" ]; then printf '%s\n' "$version" @@ -586,18 +566,15 @@ handle_conflicting_install() { install_release() { release_dir="$1" - vendor_root="$2" + archive_path="$2" stage_release="$RELEASES_DIR/.staging.$(basename "$release_dir").$$" mkdir -p "$RELEASES_DIR" rm -rf "$stage_release" - mkdir -p "$stage_release/codex-resources" - cp "$vendor_root/codex/codex" "$stage_release/codex" - cp "$vendor_root/path/rg" "$stage_release/codex-resources/rg" - chmod 0755 "$stage_release/codex" - chmod 0755 "$stage_release/codex-resources/rg" - if [ -f "$vendor_root/codex-resources/bwrap" ]; then - cp "$vendor_root/codex-resources/bwrap" "$stage_release/codex-resources/bwrap" + mkdir -p "$stage_release" + tar -xzf "$archive_path" -C "$stage_release" + chmod 0755 "$stage_release/bin/codex" "$stage_release/codex-path/rg" + if [ -f "$stage_release/codex-resources/bwrap" ]; then chmod 0755 "$stage_release/codex-resources/bwrap" fi @@ -613,8 +590,9 @@ release_dir_is_complete() { expected_target="$3" [ -d "$release_dir" ] && - [ -x "$release_dir/codex" ] && - [ -x "$release_dir/codex-resources/rg" ] && + [ -f "$release_dir/codex-package.json" ] && + [ -x "$release_dir/bin/codex" ] && + [ -x "$release_dir/codex-path/rg" ] && [ "$(basename "$release_dir")" = "$expected_version-$expected_target" ] && case "$expected_target" in *linux*) [ -x "$release_dir/codex-resources/bwrap" ] ;; @@ -633,7 +611,7 @@ update_visible_command() { mkdir -p "$BIN_DIR" tmp_link="$BIN_DIR/.codex.$$" - replace_path_with_symlink "$BIN_PATH" "$CURRENT_LINK/codex" "$tmp_link" + replace_path_with_symlink "$BIN_PATH" "$CURRENT_LINK/bin/codex" "$tmp_link" } verify_visible_command() { @@ -679,29 +657,27 @@ fi if [ "$os" = "darwin" ]; then if [ "$arch" = "aarch64" ]; then - npm_tag="darwin-arm64" vendor_target="aarch64-apple-darwin" platform_label="macOS (Apple Silicon)" else - npm_tag="darwin-x64" vendor_target="x86_64-apple-darwin" platform_label="macOS (Intel)" fi else if [ "$arch" = "aarch64" ]; then - npm_tag="linux-arm64" vendor_target="aarch64-unknown-linux-musl" platform_label="Linux (ARM64)" else - npm_tag="linux-x64" vendor_target="x86_64-unknown-linux-musl" platform_label="Linux (x64)" fi fi resolved_version="$(resolve_version)" -asset="codex-npm-$npm_tag-$resolved_version.tgz" +asset="codex-package-$vendor_target.tar.gz" +checksum_asset="codex-package_SHA256SUMS" download_url="$(release_url_for_asset "$asset" "$resolved_version")" +checksum_url="$(release_url_for_asset "$checksum_asset" "$resolved_version")" release_name="$resolved_version-$vendor_target" release_dir="$RELEASES_DIR/$release_name" current_version="$(current_installed_version)" @@ -736,18 +712,16 @@ if ! release_dir_is_complete "$release_dir" "$resolved_version" "$vendor_target" fi archive_path="$tmp_dir/$asset" - extract_dir="$tmp_dir/extract" + checksum_path="$tmp_dir/$checksum_asset" step "Downloading Codex CLI" - expected_digest="$(release_asset_digest "$asset" "$resolved_version")" + download_file "$checksum_url" "$checksum_path" + expected_digest="$(package_archive_digest "$asset" "$checksum_path")" download_file "$download_url" "$archive_path" verify_archive_digest "$archive_path" "$expected_digest" - mkdir -p "$extract_dir" - tar -xzf "$archive_path" -C "$extract_dir" - step "Installing standalone package to $release_dir" - install_release "$release_dir" "$extract_dir/package/vendor/$vendor_target" + install_release "$release_dir" "$archive_path" fi update_current_link "$release_dir" update_visible_command From bea33335779b4e134447e8b5bb24e761aa596075 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Wed, 20 May 2026 08:24:55 -0700 Subject: [PATCH 16/17] npm: ship platform packages in Codex package layout ## Summary The npm platform packages should stop carrying a bespoke native layout now that the release workflow builds canonical Codex package archives. Keeping npm on the same `bin/`, `codex-resources/`, and `codex-path/` structure lets the Rust package-layout detection behave consistently across standalone, npm, and future DotSlash installs. This changes platform npm packages to stage the `codex-package` artifact for each target under `vendor/`. The Node launcher now resolves `bin/codex` and prepends `codex-path`, while retaining legacy `vendor//codex` and `vendor//path` fallback support for local development and migration. The npm staging helper downloads `codex-package` archives instead of rebuilding the CLI payload from individual `codex`, `rg`, `bwrap`, and sandbox helper artifacts. CI still needs to stage npm packages from historical rust-release workflow artifacts that predate package archives, so the staging scripts now expose an explicit `--allow-legacy-codex-package` fallback. That fallback synthesizes the canonical package layout from legacy per-binary artifacts and is wired only into the CI smoke path; release staging remains strict and continues to require real package archives. ## Test plan - `python3 -m py_compile codex-cli/scripts/build_npm_package.py codex-cli/scripts/install_native_deps.py scripts/stage_npm_packages.py scripts/codex_package/cli.py` - `node --check codex-cli/bin/codex.js` - `ruby -e 'require "yaml"; YAML.load_file(".github/workflows/rust-release.yml"); YAML.load_file(".github/workflows/ci.yml"); puts "ok"'` - Staged a synthetic `codex-linux-x64` platform package from a canonical vendor tree and verified it copied only `bin/`, `codex-path/`, `codex-resources/`, and `codex-package.json`. - Imported `install_native_deps.py` and extracted a synthetic `codex-package-x86_64-unknown-linux-musl.tar.gz` into `vendor/`. - Ran legacy-layout conversion smokes for Linux, Windows, and unsigned macOS artifact naming. - `NPM_CONFIG_CACHE="$tmp_dir/npm-cache" python3 ./scripts/stage_npm_packages.py --release-version 0.125.0 --workflow-url https://github.com/openai/codex/actions/runs/26131514935 --package codex --allow-legacy-codex-package --output-dir "$tmp_dir"` - `node codex-cli/bin/codex.js --version` --- .github/workflows/ci.yml | 8 +- .github/workflows/rust-release.yml | 2 - codex-cli/bin/codex.js | 60 ++++--- codex-cli/scripts/README.md | 8 +- codex-cli/scripts/build_npm_package.py | 70 ++++++-- codex-cli/scripts/install_native_deps.py | 201 +++++++++++++++++++++-- scripts/stage_npm_packages.py | 22 ++- 7 files changed, 307 insertions(+), 64 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a1c60acc26..97591d4c19 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,14 +53,16 @@ jobs: # Passing the workflow URL directly avoids relying on old rust-v* # branches remaining discoverable via `gh run list --branch ...`. CODEX_VERSION=0.125.0 - WORKFLOW_URL="https://github.com/openai/codex/actions/runs/24901475298" + WORKFLOW_URL="https://github.com/openai/codex/actions/runs/26131514935" OUTPUT_DIR="${RUNNER_TEMP}" - # This reused workflow predates the standalone bwrap artifact. + # This reused workflow predates codex-package archive artifacts, so + # CI synthesizes the package layout from the older per-binary + # artifacts. Release staging must use real package archives. python3 ./scripts/stage_npm_packages.py \ --release-version "$CODEX_VERSION" \ --workflow-url "$WORKFLOW_URL" \ --package codex \ - --allow-missing-native-component bwrap \ + --allow-legacy-codex-package \ --output-dir "$OUTPUT_DIR" PACK_OUTPUT="${OUTPUT_DIR}/codex-npm-${CODEX_VERSION}.tgz" echo "pack_output=$PACK_OUTPUT" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/rust-release.yml b/.github/workflows/rust-release.yml index 93e23b8fd7..7268a24ad8 100644 --- a/.github/workflows/rust-release.yml +++ b/.github/workflows/rust-release.yml @@ -1208,8 +1208,6 @@ jobs: if: ${{ env.SIGN_MACOS == 'true' }} run: pnpm install --frozen-lockfile - # stage_npm_packages.py requires DotSlash when staging releases. - - uses: facebook/install-dotslash@1e4e7b3e07eaca387acb98f1d4720e0bee8dbb6a # v2 - name: Stage npm packages if: ${{ env.SIGN_MACOS == 'true' }} env: diff --git a/codex-cli/bin/codex.js b/codex-cli/bin/codex.js index 475239549a..1a43ce7e1f 100755 --- a/codex-cli/bin/codex.js +++ b/codex-cli/bin/codex.js @@ -77,33 +77,43 @@ if (!platformPackage) { const codexBinaryName = process.platform === "win32" ? "codex.exe" : "codex"; const localVendorRoot = path.join(__dirname, "..", "vendor"); -const localBinaryPath = path.join( - localVendorRoot, - targetTriple, - "codex", - codexBinaryName, -); +const packageBinaryPath = (vendorRoot) => + path.join(vendorRoot, targetTriple, "bin", codexBinaryName); +const legacyBinaryPath = (vendorRoot) => + path.join(vendorRoot, targetTriple, "codex", codexBinaryName); -let vendorRoot; -try { - const packageJsonPath = require.resolve(`${platformPackage}/package.json`); - vendorRoot = path.join(path.dirname(packageJsonPath), "vendor"); -} catch { - if (existsSync(localBinaryPath)) { - vendorRoot = localVendorRoot; - } else { - const packageManager = detectPackageManager(); - const updateCommand = - packageManager === "bun" - ? "bun install -g @openai/codex@latest" - : "npm install -g @openai/codex@latest"; - throw new Error( - `Missing optional dependency ${platformPackage}. Reinstall Codex: ${updateCommand}`, - ); +function resolveNativePackage(vendorRoot) { + const packageRoot = path.join(vendorRoot, targetTriple); + const binaryPath = packageBinaryPath(vendorRoot); + if (existsSync(binaryPath)) { + return { + binaryPath, + pathDir: path.join(packageRoot, "codex-path"), + }; } + + const legacyPath = legacyBinaryPath(vendorRoot); + if (existsSync(legacyPath)) { + return { + binaryPath: legacyPath, + pathDir: path.join(packageRoot, "path"), + }; + } + + return null; } -if (!vendorRoot) { +let nativePackage; +try { + const packageJsonPath = require.resolve(`${platformPackage}/package.json`); + nativePackage = resolveNativePackage( + path.join(path.dirname(packageJsonPath), "vendor"), + ); +} catch { + nativePackage = resolveNativePackage(localVendorRoot); +} + +if (!nativePackage) { const packageManager = detectPackageManager(); const updateCommand = packageManager === "bun" @@ -114,8 +124,7 @@ if (!vendorRoot) { ); } -const archRoot = path.join(vendorRoot, targetTriple); -const binaryPath = path.join(archRoot, "codex", codexBinaryName); +const { binaryPath, pathDir } = nativePackage; // Use an asynchronous spawn instead of spawnSync so that Node is able to // respond to signals (e.g. Ctrl-C / SIGINT) while the native binary is @@ -159,7 +168,6 @@ function detectPackageManager() { } const additionalDirs = []; -const pathDir = path.join(archRoot, "path"); if (existsSync(pathDir)) { additionalDirs.push(pathDir); } diff --git a/codex-cli/scripts/README.md b/codex-cli/scripts/README.md index ca0d54b544..ce58097db5 100644 --- a/codex-cli/scripts/README.md +++ b/codex-cli/scripts/README.md @@ -11,13 +11,13 @@ example, to stage the CLI, responses proxy, and SDK packages for version `0.6.0` --package codex-sdk ``` -This downloads the native artifacts once, hydrates `vendor/` for each package, and writes -tarballs to `dist/npm/`. +This downloads the native package archive artifacts once, hydrates `vendor/` for each +package, and writes tarballs to `dist/npm/`. When `--package codex` is provided, the staging helper builds the lightweight `@openai/codex` meta package plus all platform-native `@openai/codex` variants that are later published under platform-specific dist-tags. If you need to invoke `build_npm_package.py` directly, run -`codex-cli/scripts/install_native_deps.py` first and pass `--vendor-src` pointing to the -directory that contains the populated `vendor/` tree. +`codex-cli/scripts/install_native_deps.py --component codex-package` first and pass +`--vendor-src` pointing to the directory that contains the populated `vendor/` tree. diff --git a/codex-cli/scripts/build_npm_package.py b/codex-cli/scripts/build_npm_package.py index c38fed68d9..5d4bc99c4e 100755 --- a/codex-cli/scripts/build_npm_package.py +++ b/codex-cli/scripts/build_npm_package.py @@ -15,6 +15,8 @@ REPO_ROOT = CODEX_CLI_ROOT.parent RESPONSES_API_PROXY_NPM_ROOT = REPO_ROOT / "codex-rs" / "responses-api-proxy" / "npm" CODEX_SDK_ROOT = REPO_ROOT / "sdk" / "typescript" CODEX_NPM_NAME = "@openai/codex" +CODEX_PACKAGE_COMPONENT = "codex-package" +CODEX_PACKAGE_ENTRIES = ("codex-package.json", "bin", "codex-resources", "codex-path") # `npm_name` is the local optional-dependency alias consumed by `bin/codex.js`. # The underlying package published to npm is always `@openai/codex`. @@ -69,12 +71,12 @@ PACKAGE_EXPANSIONS: dict[str, list[str]] = { PACKAGE_NATIVE_COMPONENTS: dict[str, list[str]] = { "codex": [], - "codex-linux-x64": ["bwrap", "codex", "rg"], - "codex-linux-arm64": ["bwrap", "codex", "rg"], - "codex-darwin-x64": ["codex", "rg"], - "codex-darwin-arm64": ["codex", "rg"], - "codex-win32-x64": ["codex", "rg", "codex-windows-sandbox-setup", "codex-command-runner"], - "codex-win32-arm64": ["codex", "rg", "codex-windows-sandbox-setup", "codex-command-runner"], + "codex-linux-x64": [CODEX_PACKAGE_COMPONENT], + "codex-linux-arm64": [CODEX_PACKAGE_COMPONENT], + "codex-darwin-x64": [CODEX_PACKAGE_COMPONENT], + "codex-darwin-arm64": [CODEX_PACKAGE_COMPONENT], + "codex-win32-x64": [CODEX_PACKAGE_COMPONENT], + "codex-win32-arm64": [CODEX_PACKAGE_COMPONENT], "codex-responses-api-proxy": ["codex-responses-api-proxy"], "codex-sdk": [], } @@ -383,7 +385,11 @@ def copy_native_binaries( if not vendor_src.exists(): raise RuntimeError(f"Vendor source directory not found: {vendor_src}") - components_set = {component for component in components if component in COMPONENT_DEST_DIR} + components_set = { + component + for component in components + if component == CODEX_PACKAGE_COMPONENT or component in COMPONENT_DEST_DIR + } allow_missing_components = allow_missing_components or set() if not components_set: return @@ -402,11 +408,26 @@ def copy_native_binaries( if target_filter is not None and target_dir.name not in target_filter: continue - dest_target_dir = vendor_dest / target_dir.name - dest_target_dir.mkdir(parents=True, exist_ok=True) copied_targets.add(target_dir.name) - for component in components_set: + dest_target_dir = vendor_dest / target_dir.name + + if CODEX_PACKAGE_COMPONENT in components_set: + validate_codex_package_dir(target_dir) + if dest_target_dir.exists(): + shutil.rmtree(dest_target_dir) + dest_target_dir.mkdir(parents=True, exist_ok=True) + for entry in CODEX_PACKAGE_ENTRIES: + src = target_dir / entry + dest = dest_target_dir / entry + if src.is_dir(): + shutil.copytree(src, dest) + else: + shutil.copy2(src, dest) + else: + dest_target_dir.mkdir(parents=True, exist_ok=True) + + for component in components_set - {CODEX_PACKAGE_COMPONENT}: dest_dir_name = COMPONENT_DEST_DIR.get(component) if dest_dir_name is None: continue @@ -431,6 +452,35 @@ def copy_native_binaries( raise RuntimeError(f"Missing target directories in vendor source: {missing_list}") +def validate_codex_package_dir(package_dir: Path) -> None: + is_windows = "windows" in package_dir.name + required_files = [ + Path("codex-package.json"), + Path("bin") / ("codex.exe" if is_windows else "codex"), + Path("codex-path") / ("rg.exe" if is_windows else "rg"), + ] + + if "linux" in package_dir.name: + required_files.append(Path("codex-resources") / "bwrap") + + if is_windows: + required_files.extend( + [ + Path("codex-resources") / "codex-command-runner.exe", + Path("codex-resources") / "codex-windows-sandbox-setup.exe", + ] + ) + + missing_files = [ + str(relative_path) + for relative_path in required_files + if not (package_dir / relative_path).is_file() + ] + if missing_files: + missing = ", ".join(missing_files) + raise RuntimeError(f"Missing files in Codex package directory {package_dir}: {missing}") + + def run_npm_pack(staging_dir: Path, output_path: Path) -> Path: output_path = output_path.resolve() output_path.parent.mkdir(parents=True, exist_ok=True) diff --git a/codex-cli/scripts/install_native_deps.py b/codex-cli/scripts/install_native_deps.py index 79f596fbaf..73f5134f09 100755 --- a/codex-cli/scripts/install_native_deps.py +++ b/codex-cli/scripts/install_native_deps.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Install Codex native binaries (Rust CLI, bwrap, and ripgrep helpers).""" +"""Install Codex package archives and native helper binaries.""" import argparse from contextlib import contextmanager @@ -31,6 +31,7 @@ BINARY_TARGETS = ( "x86_64-pc-windows-msvc", "aarch64-pc-windows-msvc", ) +CODEX_PACKAGE_COMPONENT = "codex-package" @dataclass(frozen=True) @@ -139,11 +140,19 @@ def parse_args() -> argparse.Namespace: "--component", dest="components", action="append", - choices=tuple(list(BINARY_COMPONENTS) + ["rg"]), + choices=tuple([CODEX_PACKAGE_COMPONENT, *BINARY_COMPONENTS, "rg"]), help=( "Limit installation to the specified components." - " May be repeated. Defaults to bwrap, codex, codex-windows-sandbox-setup," - " codex-command-runner, and rg." + " May be repeated. Defaults to codex-package and codex-responses-api-proxy." + ), + ) + parser.add_argument( + "--allow-legacy-codex-package", + action="store_true", + help=( + "Allow codex-package to be synthesized from legacy per-binary artifacts " + "when package archives are missing. Intended for CI compatibility only; " + "release staging should not use this." ), ) parser.add_argument( @@ -165,13 +174,7 @@ def main() -> int: vendor_dir = codex_cli_root / VENDOR_DIR_NAME vendor_dir.mkdir(parents=True, exist_ok=True) - components = args.components or [ - "bwrap", - "codex", - "codex-windows-sandbox-setup", - "codex-command-runner", - "rg", - ] + components = args.components or [CODEX_PACKAGE_COMPONENT, "codex-responses-api-proxy"] workflow_url = (args.workflow_url or DEFAULT_WORKFLOW_URL).strip() if not workflow_url: @@ -184,6 +187,18 @@ def main() -> int: with tempfile.TemporaryDirectory(prefix="codex-native-artifacts-") as artifacts_dir_str: artifacts_dir = Path(artifacts_dir_str) _download_artifacts(workflow_id, artifacts_dir) + if CODEX_PACKAGE_COMPONENT in components: + try: + install_codex_package_archives(artifacts_dir, vendor_dir, BINARY_TARGETS) + except FileNotFoundError: + if not args.allow_legacy_codex_package: + raise + install_legacy_codex_package_layouts( + artifacts_dir, + vendor_dir, + BINARY_TARGETS, + manifest_path=RG_MANIFEST, + ) install_binary_components( artifacts_dir, vendor_dir, @@ -199,6 +214,135 @@ def main() -> int: return 0 +def install_codex_package_archives( + artifacts_dir: Path, + vendor_dir: Path, + targets: Sequence[str], +) -> None: + targets = list(targets) + if not targets: + return + + print("Installing Codex package archives for targets: " + ", ".join(targets)) + max_workers = min(len(targets), max(1, (os.cpu_count() or 1))) + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = { + executor.submit( + _install_single_codex_package_archive, + artifacts_dir, + vendor_dir, + target, + ): target + for target in targets + } + for future in as_completed(futures): + installed_path = future.result() + print(f" installed {installed_path}") + + +def _install_single_codex_package_archive( + artifacts_dir: Path, + vendor_dir: Path, + target: str, +) -> Path: + artifact_subdir = artifact_dir_for_target(artifacts_dir, target) + archive_path = artifact_subdir / f"codex-package-{target}.tar.gz" + if not archive_path.exists(): + raise FileNotFoundError(f"Expected package archive not found: {archive_path}") + + dest_dir = vendor_dir / target + if dest_dir.exists(): + shutil.rmtree(dest_dir) + dest_dir.mkdir(parents=True, exist_ok=True) + + with tarfile.open(archive_path, "r:gz") as archive: + archive.extractall(dest_dir, filter="data") + + return dest_dir + + +def install_legacy_codex_package_layouts( + artifacts_dir: Path, + vendor_dir: Path, + targets: Sequence[str], + *, + manifest_path: Path, +) -> None: + targets = list(targets) + print( + "Synthesizing Codex package layouts from legacy artifacts for targets: " + + ", ".join(targets) + ) + with tempfile.TemporaryDirectory(prefix="codex-legacy-package-") as legacy_vendor_dir_str: + legacy_vendor_dir = Path(legacy_vendor_dir_str) + install_binary_components( + artifacts_dir, + legacy_vendor_dir, + [ + BINARY_COMPONENTS["codex"], + BINARY_COMPONENTS["bwrap"], + BINARY_COMPONENTS["codex-windows-sandbox-setup"], + BINARY_COMPONENTS["codex-command-runner"], + ], + ) + fetch_rg(legacy_vendor_dir, targets, manifest_path=manifest_path) + + for target in targets: + dest_dir = vendor_dir / target + if dest_dir.exists(): + shutil.rmtree(dest_dir) + _build_legacy_codex_package_layout(legacy_vendor_dir / target, dest_dir, target) + print(f" synthesized {dest_dir}") + + +def _build_legacy_codex_package_layout( + legacy_target_dir: Path, + package_dir: Path, + target: str, +) -> None: + is_windows = "windows" in target + exe_suffix = ".exe" if is_windows else "" + package_dir.mkdir(parents=True) + + bin_dir = package_dir / "bin" + resources_dir = package_dir / "codex-resources" + path_dir = package_dir / "codex-path" + bin_dir.mkdir() + resources_dir.mkdir() + path_dir.mkdir() + + shutil.copy2( + legacy_target_dir / "codex" / f"codex{exe_suffix}", + bin_dir / f"codex{exe_suffix}", + ) + shutil.copy2( + legacy_target_dir / "path" / f"rg{exe_suffix}", + path_dir / f"rg{exe_suffix}", + ) + + if is_windows: + for helper in [ + "codex-command-runner.exe", + "codex-windows-sandbox-setup.exe", + ]: + shutil.copy2(legacy_target_dir / "codex" / helper, resources_dir / helper) + elif "linux" in target: + shutil.copy2(legacy_target_dir / "codex-resources" / "bwrap", resources_dir / "bwrap") + + write_json( + package_dir / "codex-package.json", + { + "layoutVersion": 1, + "version": "unknown", + "target": target, + "variant": "codex", + "entrypoint": f"bin/codex{exe_suffix}", + "resourcesDir": "codex-resources", + "pathDir": "codex-path", + }, + ) + + def fetch_rg( vendor_dir: Path, targets: Sequence[str] | None = None, @@ -319,11 +463,8 @@ def _install_single_binary( target: str, component: BinaryComponent, ) -> Path: - artifact_subdir = artifacts_dir / target - archive_name = _archive_name_for_target(component.artifact_prefix, target) - archive_path = artifact_subdir / archive_name - if not archive_path.exists(): - raise FileNotFoundError(f"Expected artifact not found: {archive_path}") + artifact_subdir = artifact_dir_for_target(artifacts_dir, target) + archive_path = legacy_binary_archive_path(artifact_subdir, component.artifact_prefix, target) dest_dir = vendor_dir / target / component.dest_dir dest_dir.mkdir(parents=True, exist_ok=True) @@ -345,6 +486,28 @@ def _archive_name_for_target(artifact_prefix: str, target: str) -> str: return f"{artifact_prefix}-{target}.zst" +def legacy_binary_archive_path(artifact_dir: Path, artifact_prefix: str, target: str) -> Path: + archive_names = [_archive_name_for_target(artifact_prefix, target)] + if artifact_dir.name == f"{target}-unsigned": + archive_names.append(_archive_name_for_target(artifact_prefix, f"{target}-unsigned")) + + for archive_name in archive_names: + archive_path = artifact_dir / archive_name + if archive_path.exists(): + return archive_path + + raise FileNotFoundError(f"Expected artifact not found: {artifact_dir / archive_names[0]}") + + +def artifact_dir_for_target(artifacts_dir: Path, target: str) -> Path: + for artifact_name in [target, f"{target}-unsigned"]: + artifact_dir = artifacts_dir / artifact_name + if artifact_dir.is_dir(): + return artifact_dir + + return artifacts_dir / target + + def _fetch_single_rg( vendor_dir: Path, target: str, @@ -477,6 +640,12 @@ def _load_manifest(manifest_path: Path) -> dict: return manifest +def write_json(path: Path, value: object) -> None: + with open(path, "w", encoding="utf-8") as out: + json.dump(value, out, indent=2) + out.write("\n") + + if __name__ == "__main__": import sys diff --git a/scripts/stage_npm_packages.py b/scripts/stage_npm_packages.py index 2a0cd0028d..4eb69053eb 100755 --- a/scripts/stage_npm_packages.py +++ b/scripts/stage_npm_packages.py @@ -1,8 +1,6 @@ #!/usr/bin/env python3 """Stage one or more Codex npm packages for release.""" -from __future__ import annotations - import argparse import importlib.util import json @@ -68,6 +66,15 @@ def parse_args() -> argparse.Namespace: "Intended for CI compatibility only; release staging should not use this." ), ) + parser.add_argument( + "--allow-legacy-codex-package", + action="store_true", + help=( + "Allow codex-package layouts to be synthesized from legacy per-binary " + "workflow artifacts. Intended for CI compatibility only; release staging " + "should not use this." + ), + ) return parser.parse_args() @@ -124,11 +131,15 @@ def install_native_components( workflow_url: str, components: set[str], vendor_root: Path, + *, + allow_legacy_codex_package: bool, ) -> None: if not components: return cmd = [str(INSTALL_NATIVE_DEPS), "--workflow-url", workflow_url] + if allow_legacy_codex_package: + cmd.append("--allow-legacy-codex-package") for component in sorted(components): cmd.extend(["--component", component]) cmd.append(str(vendor_root)) @@ -172,7 +183,12 @@ def main() -> int: args.release_version, args.workflow_url ) vendor_temp_root = Path(tempfile.mkdtemp(prefix="npm-native-", dir=runner_temp)) - install_native_components(workflow_url, native_components_to_install, vendor_temp_root) + install_native_components( + workflow_url, + native_components_to_install, + vendor_temp_root, + allow_legacy_codex_package=args.allow_legacy_codex_package, + ) vendor_src = vendor_temp_root / "vendor" if resolved_head_sha: From 0689625c702d3ca06604de5e5e63f990c159f3d5 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Wed, 20 May 2026 08:24:55 -0700 Subject: [PATCH 17/17] dotslash: publish Codex entrypoints from package archives ## Summary DotSlash should resolve the same canonical package layout used by standalone installers and npm platform packages, but it should not take the size regression of downloading gzip-compressed package archives. Installers continue to use `.tar.gz` because that format is broadly available, while DotSlash can use zstd-compressed tar archives. This updates the package archive release helper to emit both `*.tar.gz` and `*.tar.zst` package archives from the same package directory. The Codex CLI and `codex-app-server` DotSlash release config entries now match `codex-package-.tar.zst` and `codex-app-server-package-.tar.zst`, with paths that select `bin/codex` or `bin/codex-app-server` inside the extracted package. The Windows release compression loop also skips prebuilt `.tar.zst` archives so it does not try to recompress them as ordinary files. The other helper outputs stay on their existing per-binary artifacts for now. ## Test plan - `python3 -m json.tool .github/dotslash-config.json > /dev/null` - `bash -n .github/scripts/build-codex-package-archive.sh` - `python3 -m py_compile scripts/codex_package/cli.py` - `ruby -e 'require "yaml"; YAML.load_file(".github/workflows/rust-release-windows.yml"); puts "ok"'` - Ran a Python regex smoke test that checked every updated `codex` and `codex-app-server` platform entry against the `.tar.zst` archive names emitted by `.github/scripts/build-codex-package-archive.sh`. - Ran `scripts/build_codex_package.py` with synthetic macOS entrypoint and `rg` binaries and repeated `--archive-output` values for `.tar.gz` and `.tar.zst`, verifying both archives were written. --- .github/dotslash-config.json | 48 +++++++++---------- .../scripts/build-codex-package-archive.sh | 10 +++- .github/workflows/rust-release-windows.yml | 2 +- scripts/codex_package/cli.py | 9 ++-- 4 files changed, 38 insertions(+), 31 deletions(-) diff --git a/.github/dotslash-config.json b/.github/dotslash-config.json index a0297c269a..78ea6b37e6 100644 --- a/.github/dotslash-config.json +++ b/.github/dotslash-config.json @@ -3,56 +3,56 @@ "codex": { "platforms": { "macos-aarch64": { - "regex": "^codex-aarch64-apple-darwin\\.zst$", - "path": "codex" + "regex": "^codex-package-aarch64-apple-darwin\\.tar\\.zst$", + "path": "bin/codex" }, "macos-x86_64": { - "regex": "^codex-x86_64-apple-darwin\\.zst$", - "path": "codex" + "regex": "^codex-package-x86_64-apple-darwin\\.tar\\.zst$", + "path": "bin/codex" }, "linux-x86_64": { - "regex": "^codex-x86_64-unknown-linux-musl-bundle\\.tar\\.zst$", - "path": "codex" + "regex": "^codex-package-x86_64-unknown-linux-musl\\.tar\\.zst$", + "path": "bin/codex" }, "linux-aarch64": { - "regex": "^codex-aarch64-unknown-linux-musl-bundle\\.tar\\.zst$", - "path": "codex" + "regex": "^codex-package-aarch64-unknown-linux-musl\\.tar\\.zst$", + "path": "bin/codex" }, "windows-x86_64": { - "regex": "^codex-x86_64-pc-windows-msvc\\.exe\\.zst$", - "path": "codex.exe" + "regex": "^codex-package-x86_64-pc-windows-msvc\\.tar\\.zst$", + "path": "bin/codex.exe" }, "windows-aarch64": { - "regex": "^codex-aarch64-pc-windows-msvc\\.exe\\.zst$", - "path": "codex.exe" + "regex": "^codex-package-aarch64-pc-windows-msvc\\.tar\\.zst$", + "path": "bin/codex.exe" } } }, "codex-app-server": { "platforms": { "macos-aarch64": { - "regex": "^codex-app-server-aarch64-apple-darwin\\.zst$", - "path": "codex-app-server" + "regex": "^codex-app-server-package-aarch64-apple-darwin\\.tar\\.zst$", + "path": "bin/codex-app-server" }, "macos-x86_64": { - "regex": "^codex-app-server-x86_64-apple-darwin\\.zst$", - "path": "codex-app-server" + "regex": "^codex-app-server-package-x86_64-apple-darwin\\.tar\\.zst$", + "path": "bin/codex-app-server" }, "linux-x86_64": { - "regex": "^codex-app-server-x86_64-unknown-linux-musl\\.zst$", - "path": "codex-app-server" + "regex": "^codex-app-server-package-x86_64-unknown-linux-musl\\.tar\\.zst$", + "path": "bin/codex-app-server" }, "linux-aarch64": { - "regex": "^codex-app-server-aarch64-unknown-linux-musl\\.zst$", - "path": "codex-app-server" + "regex": "^codex-app-server-package-aarch64-unknown-linux-musl\\.tar\\.zst$", + "path": "bin/codex-app-server" }, "windows-x86_64": { - "regex": "^codex-app-server-x86_64-pc-windows-msvc\\.exe\\.zst$", - "path": "codex-app-server.exe" + "regex": "^codex-app-server-package-x86_64-pc-windows-msvc\\.tar\\.zst$", + "path": "bin/codex-app-server.exe" }, "windows-aarch64": { - "regex": "^codex-app-server-aarch64-pc-windows-msvc\\.exe\\.zst$", - "path": "codex-app-server.exe" + "regex": "^codex-app-server-package-aarch64-pc-windows-msvc\\.tar\\.zst$", + "path": "bin/codex-app-server.exe" } } }, diff --git a/.github/scripts/build-codex-package-archive.sh b/.github/scripts/build-codex-package-archive.sh index 90eae12ef0..c475ffe952 100644 --- a/.github/scripts/build-codex-package-archive.sh +++ b/.github/scripts/build-codex-package-archive.sh @@ -97,9 +97,14 @@ else python_bin="python" fi +if ! command -v zstd >/dev/null 2>&1 && [[ -x "${repo_root}/.github/workflows/zstd" ]]; then + export PATH="${repo_root}/.github/workflows:${PATH}" +fi + mkdir -p "$archive_dir" package_dir="${RUNNER_TEMP:-/tmp}/${archive_stem}-${target}" -archive_path="${archive_dir}/${archive_stem}-${target}.tar.gz" +gzip_archive_path="${archive_dir}/${archive_stem}-${target}.tar.gz" +zstd_archive_path="${archive_dir}/${archive_stem}-${target}.tar.zst" rm -rf "$package_dir" "$python_bin" "${repo_root}/scripts/build_codex_package.py" \ @@ -108,5 +113,6 @@ rm -rf "$package_dir" --entrypoint-bin "${entrypoint_dir%/}/${entrypoint_name}${exe_suffix}" \ --cargo-profile release \ --package-dir "$package_dir" \ - --archive-output "$archive_path" \ + --archive-output "$gzip_archive_path" \ + --archive-output "$zstd_archive_path" \ --force diff --git a/.github/workflows/rust-release-windows.yml b/.github/workflows/rust-release-windows.yml index ac28b7855a..81b5a1e51b 100644 --- a/.github/workflows/rust-release-windows.yml +++ b/.github/workflows/rust-release-windows.yml @@ -295,7 +295,7 @@ jobs: base="$(basename "$f")" # Skip files that are already archives (shouldn't happen, but be # safe). - if [[ "$base" == *.tar.gz || "$base" == *.zip || "$base" == *.dmg ]]; then + if [[ "$base" == *.tar.gz || "$base" == *.tar.zst || "$base" == *.zip || "$base" == *.dmg ]]; then continue fi diff --git a/scripts/codex_package/cli.py b/scripts/codex_package/cli.py index 16324cf44d..0ca9f5d35e 100644 --- a/scripts/codex_package/cli.py +++ b/scripts/codex_package/cli.py @@ -50,9 +50,11 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--archive-output", type=Path, + action="append", + default=[], help=( - "Optional archive output path. Supported suffixes: .tar.gz, .tgz, " - ".tar.zst, .zip." + "Optional archive output path. May be repeated. Supported suffixes: " + ".tar.gz, .tgz, .tar.zst, .zip." ), ) parser.add_argument( @@ -130,8 +132,7 @@ def main() -> int: build_package_dir(package_dir, version, variant, spec, inputs) validate_package_dir(package_dir, variant, spec) - archive_output = args.archive_output - if archive_output is not None: + for archive_output in args.archive_output: archive_path = archive_output.resolve() write_archive(package_dir, archive_path, force=args.force) print(f"Built Codex package archive at {archive_path}")