From d84b824d539c131d90fa4e1ff6345b924a87bfa1 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Wed, 20 May 2026 20:37:27 +0200 Subject: [PATCH 1/5] [codex] Preserve failed goal accounting flushes (#23717) ## What - Preserve database accounting failures from the goal extension instead of collapsing them into `None` - Warn with turn/tool context when a flush fails - Keep stop/abort accounting snapshots alive when the final flush did not persist ## Why PR #23696 can finish and discard a turn snapshot after `account_thread_goal_usage` fails. That loses the final accumulated accounting state silently. This follow-up keeps that failure explicit and avoids deleting the local snapshot in the failing path. ## Testing - `just fmt` - `cargo test -p codex-goal-extension` --- codex-rs/ext/goal/src/extension.rs | 83 +++++++++++++++++++----------- 1 file changed, 53 insertions(+), 30 deletions(-) diff --git a/codex-rs/ext/goal/src/extension.rs b/codex-rs/ext/goal/src/extension.rs index 61114bf57e..361147a512 100644 --- a/codex-rs/ext/goal/src/extension.rs +++ b/codex-rs/ext/goal/src/extension.rs @@ -151,14 +151,21 @@ where } let turn_id = input.turn_store.level_id(); - self.account_active_goal_progress( - input.thread_store, - turn_id, - &format!("{turn_id}:turn-stop"), - codex_state::GoalAccountingMode::ActiveOnly, - BudgetLimitedGoalDisposition::ClearActive, - ) - .await; + if let Err(err) = self + .account_active_goal_progress( + input.thread_store, + turn_id, + &format!("{turn_id}:turn-stop"), + codex_state::GoalAccountingMode::ActiveOnly, + BudgetLimitedGoalDisposition::ClearActive, + ) + .await + { + tracing::warn!( + "failed to account active goal progress at turn stop for {turn_id}: {err}" + ); + return; + } accounting_state(input.thread_store).finish_turn(turn_id); } @@ -168,14 +175,21 @@ where } let turn_id = input.turn_store.level_id(); - self.account_active_goal_progress( - input.thread_store, - turn_id, - &format!("{turn_id}:turn-abort"), - codex_state::GoalAccountingMode::ActiveOnly, - BudgetLimitedGoalDisposition::ClearActive, - ) - .await; + if let Err(err) = self + .account_active_goal_progress( + input.thread_store, + turn_id, + &format!("{turn_id}:turn-abort"), + codex_state::GoalAccountingMode::ActiveOnly, + BudgetLimitedGoalDisposition::ClearActive, + ) + .await + { + tracing::warn!( + "failed to account active goal progress after turn abort for {turn_id}: {err}" + ); + return; + } accounting_state(input.thread_store).finish_turn(turn_id); } } @@ -217,14 +231,21 @@ where if !should_count_for_goal_progress { return; } - self.account_active_goal_progress( - input.thread_store, - input.turn_id, - input.call_id, - codex_state::GoalAccountingMode::ActiveOnly, - BudgetLimitedGoalDisposition::KeepActive, - ) - .await; + let turn_id = input.turn_id; + if let Err(err) = self + .account_active_goal_progress( + input.thread_store, + turn_id, + input.call_id, + codex_state::GoalAccountingMode::ActiveOnly, + BudgetLimitedGoalDisposition::KeepActive, + ) + .await + { + tracing::warn!( + "failed to account active goal progress after tool finish for {turn_id}: {err}" + ); + } }) } } @@ -326,12 +347,14 @@ impl GoalExtension { event_id: &str, mode: codex_state::GoalAccountingMode, budget_limited_goal_disposition: BudgetLimitedGoalDisposition, - ) -> Option { + ) -> Result, String> { let Ok(thread_id) = ThreadId::from_string(thread_store.level_id()) else { - return None; + return Ok(None); }; let accounting = accounting_state(thread_store); - let snapshot = accounting.progress_snapshot(turn_id)?; + let Some(snapshot) = accounting.progress_snapshot(turn_id) else { + return Ok(None); + }; let outcome = self .state_dbs .thread_goals() @@ -343,8 +366,8 @@ impl GoalExtension { Some(snapshot.expected_goal_id.as_str()), ) .await - .ok()?; - match outcome { + .map_err(|err| err.to_string())?; + Ok(match outcome { codex_state::GoalAccountingOutcome::Updated(goal) => { accounting.mark_progress_accounted_for_status( turn_id, @@ -361,6 +384,6 @@ impl GoalExtension { Some(goal) } codex_state::GoalAccountingOutcome::Unchanged(_) => None, - } + }) } } From ed6d73b3b94fd436db87eca3d5984628fa8c7a57 Mon Sep 17 00:00:00 2001 From: sayan-oai Date: Wed, 20 May 2026 11:38:21 -0700 Subject: [PATCH 2/5] add standalone websearch api client (#23655) add standalone web search request types and a `codex-api` client ahead of the extension-contributed search tool. this adds typed commands/settings and opaque encrypted output handling for the new standalone search flow. the endpoint types are close to finalized but may still shift slightly as that API settles. --- codex-rs/codex-api/src/endpoint/mod.rs | 2 + codex-rs/codex-api/src/endpoint/search.rs | 264 ++++++++++++++++++++++ codex-rs/codex-api/src/lib.rs | 27 +++ codex-rs/codex-api/src/search.rs | 246 ++++++++++++++++++++ 4 files changed, 539 insertions(+) create mode 100644 codex-rs/codex-api/src/endpoint/search.rs create mode 100644 codex-rs/codex-api/src/search.rs diff --git a/codex-rs/codex-api/src/endpoint/mod.rs b/codex-rs/codex-api/src/endpoint/mod.rs index 70c52b9580..21ebf372a1 100644 --- a/codex-rs/codex-api/src/endpoint/mod.rs +++ b/codex-rs/codex-api/src/endpoint/mod.rs @@ -5,6 +5,7 @@ pub(crate) mod realtime_call; pub(crate) mod realtime_websocket; pub(crate) mod responses; pub(crate) mod responses_websocket; +pub(crate) mod search; mod session; pub use compact::CompactClient; @@ -27,3 +28,4 @@ pub use responses_websocket::ResponsesWebsocketClient; pub use responses_websocket::ResponsesWebsocketClose; pub use responses_websocket::ResponsesWebsocketConnection; pub use responses_websocket::ResponsesWebsocketProbe; +pub use search::SearchClient; diff --git a/codex-rs/codex-api/src/endpoint/search.rs b/codex-rs/codex-api/src/endpoint/search.rs new file mode 100644 index 0000000000..2940f872d8 --- /dev/null +++ b/codex-rs/codex-api/src/endpoint/search.rs @@ -0,0 +1,264 @@ +use crate::auth::SharedAuthProvider; +use crate::endpoint::session::EndpointSession; +use crate::error::ApiError; +use crate::provider::Provider; +use crate::search::SearchRequest; +use crate::search::SearchResponse; +use codex_client::HttpTransport; +use codex_client::RequestTelemetry; +use http::HeaderMap; +use http::Method; +use serde_json::to_value; +use std::sync::Arc; + +pub struct SearchClient { + session: EndpointSession, +} + +impl SearchClient { + pub fn new(transport: T, provider: Provider, auth: SharedAuthProvider) -> Self { + Self { + session: EndpointSession::new(transport, provider, auth), + } + } + + pub fn with_telemetry(self, request: Option>) -> Self { + Self { + session: self.session.with_request_telemetry(request), + } + } + + fn path() -> &'static str { + "alpha/search" + } + + pub async fn search( + &self, + request: &SearchRequest, + extra_headers: HeaderMap, + ) -> Result { + let body = to_value(request) + .map_err(|e| ApiError::Stream(format!("failed to encode search request: {e}")))?; + let resp = self + .session + .execute(Method::POST, Self::path(), extra_headers, Some(body)) + .await?; + serde_json::from_slice(&resp.body) + .map_err(|e| ApiError::Stream(format!("failed to decode search response: {e}"))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::auth::AuthProvider; + use crate::provider::RetryConfig; + use crate::search::AllowedCaller; + use crate::search::ApproximateLocation; + use crate::search::LocationType; + use crate::search::OpenOperation; + use crate::search::SearchCommands; + use crate::search::SearchContextSize; + use crate::search::SearchFilters; + use crate::search::SearchImageSettings; + use crate::search::SearchInput; + use crate::search::SearchQuery; + use crate::search::SearchSettings; + use async_trait::async_trait; + use codex_client::Request; + use codex_client::RequestBody; + use codex_client::Response; + use codex_client::StreamResponse; + use codex_client::TransportError; + use codex_protocol::models::ContentItem; + use codex_protocol::models::ResponseItem; + use http::StatusCode; + use pretty_assertions::assert_eq; + use serde_json::json; + use std::sync::Mutex; + use std::time::Duration; + + #[derive(Clone, Default)] + struct DummyAuth; + + impl AuthProvider for DummyAuth { + fn add_auth_headers(&self, _headers: &mut HeaderMap) {} + } + + #[derive(Clone)] + struct CapturingTransport { + last_request: Arc>>, + response_body: Arc>, + } + + impl CapturingTransport { + fn new(response_body: Vec) -> Self { + Self { + last_request: Arc::new(Mutex::new(None)), + response_body: Arc::new(response_body), + } + } + } + + #[async_trait] + impl HttpTransport for CapturingTransport { + async fn execute(&self, req: Request) -> Result { + *self.last_request.lock().expect("lock request store") = Some(req); + Ok(Response { + status: StatusCode::OK, + headers: HeaderMap::new(), + body: self.response_body.as_ref().clone().into(), + }) + } + + async fn stream(&self, _req: Request) -> Result { + Err(TransportError::Build("stream should not run".to_string())) + } + } + + fn provider() -> Provider { + Provider { + name: "test".to_string(), + base_url: "https://example.com/v1".to_string(), + query_params: None, + headers: HeaderMap::new(), + retry: RetryConfig { + max_attempts: 1, + base_delay: Duration::from_millis(1), + retry_429: false, + retry_5xx: true, + retry_transport: true, + }, + stream_idle_timeout: Duration::from_secs(1), + } + } + + #[tokio::test] + async fn search_posts_typed_request_and_parses_encrypted_output() { + let transport = CapturingTransport::new( + serde_json::to_vec(&json!({"encrypted_output": "ciphertext"})) + .expect("serialize response"), + ); + let client = SearchClient::new(transport.clone(), provider(), Arc::new(DummyAuth)); + + let response = client + .search( + &SearchRequest { + id: "search-session".to_string(), + model: Some("gpt-test".to_string()), + reasoning: None, + input: Some(SearchInput::Items(vec![ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ + ContentItem::InputText { + text: "find this".to_string(), + }, + ContentItem::InputImage { + image_url: "https://example.com/image.png".to_string(), + detail: None, + }, + ], + phase: None, + }])), + commands: Some(SearchCommands { + search_query: Some(vec![SearchQuery { + q: "OpenAI news".to_string(), + recency: Some(7), + domains: Some(vec!["openai.com".to_string()]), + }]), + open: Some(vec![OpenOperation { + ref_id: "https://openai.com".to_string(), + lineno: Some(12), + }]), + ..Default::default() + }), + settings: Some(SearchSettings { + user_location: Some(ApproximateLocation { + r#type: LocationType::Approximate, + country: Some("US".to_string()), + region: None, + city: Some("San Francisco".to_string()), + timezone: None, + }), + search_context_size: Some(SearchContextSize::Low), + filters: Some(SearchFilters { + allowed_domains: Some(vec!["openai.com".to_string()]), + blocked_domains: Some(vec!["example.com".to_string()]), + }), + image_settings: Some(SearchImageSettings { + max_results: Some(4), + caption: Some(true), + }), + allowed_callers: Some(vec![AllowedCaller::Direct]), + external_web_access: Some(true), + }), + max_output_tokens: Some(2500), + }, + HeaderMap::new(), + ) + .await + .expect("search request should succeed"); + + assert_eq!( + response, + SearchResponse { + encrypted_output: "ciphertext".to_string(), + } + ); + + let request = transport + .last_request + .lock() + .expect("lock request store") + .clone() + .expect("request should be captured"); + let body = request + .body + .as_ref() + .and_then(RequestBody::json) + .expect("request body should be JSON"); + assert_eq!( + body, + &json!({ + "id": "search-session", + "model": "gpt-test", + "input": [{ + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": "find this"}, + { + "type": "input_image", + "image_url": "https://example.com/image.png" + } + ] + }], + "commands": { + "search_query": [{ + "q": "OpenAI news", + "recency": 7, + "domains": ["openai.com"] + }], + "open": [{"ref_id": "https://openai.com", "lineno": 12}] + }, + "settings": { + "user_location": { + "type": "approximate", + "country": "US", + "city": "San Francisco" + }, + "search_context_size": "low", + "filters": { + "allowed_domains": ["openai.com"], + "blocked_domains": ["example.com"] + }, + "image_settings": {"max_results": 4, "caption": true}, + "allowed_callers": ["direct"], + "external_web_access": true + }, + "max_output_tokens": 2500 + }) + ); + } +} diff --git a/codex-rs/codex-api/src/lib.rs b/codex-rs/codex-api/src/lib.rs index 99470cac59..f47791b799 100644 --- a/codex-rs/codex-api/src/lib.rs +++ b/codex-rs/codex-api/src/lib.rs @@ -7,6 +7,7 @@ pub(crate) mod files; pub(crate) mod provider; pub(crate) mod rate_limits; pub(crate) mod requests; +pub(crate) mod search; pub(crate) mod sse; pub(crate) mod telemetry; @@ -58,6 +59,7 @@ pub use crate::endpoint::ResponsesWebsocketClient; pub use crate::endpoint::ResponsesWebsocketClose; pub use crate::endpoint::ResponsesWebsocketConnection; pub use crate::endpoint::ResponsesWebsocketProbe; +pub use crate::endpoint::SearchClient; pub use crate::endpoint::session_update_session_json; pub use crate::error::ApiError; pub use crate::files::upload_local_file; @@ -65,6 +67,31 @@ pub use crate::provider::Provider; pub use crate::provider::RetryConfig; pub use crate::provider::is_azure_responses_provider; pub use crate::requests::Compression; +pub use crate::search::AllowedCaller; +pub use crate::search::ApproximateLocation; +pub use crate::search::ClickOperation; +pub use crate::search::FinanceAssetType; +pub use crate::search::FinanceOperation; +pub use crate::search::FindOperation; +pub use crate::search::LocationType; +pub use crate::search::OpenOperation; +pub use crate::search::ScreenshotOperation; +pub use crate::search::SearchCommands; +pub use crate::search::SearchContextSize; +pub use crate::search::SearchFilters; +pub use crate::search::SearchImageSettings; +pub use crate::search::SearchInput; +pub use crate::search::SearchQuery; +pub use crate::search::SearchRequest; +pub use crate::search::SearchResponse; +pub use crate::search::SearchResponseLength; +pub use crate::search::SearchSettings; +pub use crate::search::SportsFunction; +pub use crate::search::SportsLeague; +pub use crate::search::SportsOperation; +pub use crate::search::SportsToolName; +pub use crate::search::TimeOperation; +pub use crate::search::WeatherOperation; pub use crate::telemetry::SseTelemetry; pub use crate::telemetry::WebsocketTelemetry; pub use codex_protocol::protocol::RealtimeAudioFrame; diff --git a/codex-rs/codex-api/src/search.rs b/codex-rs/codex-api/src/search.rs new file mode 100644 index 0000000000..b841d06a30 --- /dev/null +++ b/codex-rs/codex-api/src/search.rs @@ -0,0 +1,246 @@ +use crate::common::Reasoning; +use codex_protocol::models::ResponseItem; +use serde::Deserialize; +use serde::Serialize; + +#[derive(Debug, Clone, Serialize, PartialEq)] +pub struct SearchRequest { + pub id: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub model: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub reasoning: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub input: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub commands: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub settings: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub max_output_tokens: Option, +} + +#[derive(Debug, Clone, Serialize, PartialEq)] +#[serde(untagged)] +pub enum SearchInput { + Text(String), + Items(Vec), +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq)] +pub struct SearchCommands { + #[serde(skip_serializing_if = "Option::is_none")] + pub search_query: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub image_query: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub open: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub click: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub find: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub screenshot: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub finance: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub weather: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub sports: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub time: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub response_length: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct SearchQuery { + pub q: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub recency: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub domains: Option>, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct OpenOperation { + pub ref_id: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub lineno: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ClickOperation { + pub ref_id: String, + pub id: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct FindOperation { + pub ref_id: String, + pub pattern: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ScreenshotOperation { + pub ref_id: String, + pub pageno: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct FinanceOperation { + pub ticker: String, + pub r#type: FinanceAssetType, + #[serde(skip_serializing_if = "Option::is_none")] + pub market: Option, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum FinanceAssetType { + Equity, + Fund, + Crypto, + Index, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct WeatherOperation { + pub location: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub start: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub duration: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct SportsOperation { + #[serde(skip_serializing_if = "Option::is_none")] + pub tool: Option, + pub r#fn: SportsFunction, + pub league: SportsLeague, + #[serde(skip_serializing_if = "Option::is_none")] + pub team: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub opponent: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub date_from: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub date_to: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub num_games: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub locale: Option, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum SportsToolName { + Sports, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum SportsFunction { + Schedule, + Standings, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum SportsLeague { + Nba, + Wnba, + Nfl, + Nhl, + Mlb, + Epl, + Ncaamb, + Ncaawb, + Ipl, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct TimeOperation { + pub utc_offset: String, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum SearchResponseLength { + Short, + Medium, + Long, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq)] +pub struct SearchSettings { + #[serde(skip_serializing_if = "Option::is_none")] + pub user_location: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub search_context_size: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub filters: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub image_settings: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub allowed_callers: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub external_web_access: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct ApproximateLocation { + pub r#type: LocationType, + #[serde(skip_serializing_if = "Option::is_none")] + pub country: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub region: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub city: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub timezone: Option, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum LocationType { + Approximate, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum SearchContextSize { + Low, + Medium, + High, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)] +pub struct SearchFilters { + #[serde(skip_serializing_if = "Option::is_none")] + pub allowed_domains: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub blocked_domains: Option>, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)] +pub struct SearchImageSettings { + #[serde(skip_serializing_if = "Option::is_none")] + pub max_results: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub caption: Option, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum AllowedCaller { + Direct, + Shell, + CodeInterpreter, +} + +#[derive(Debug, Clone, Deserialize, PartialEq, Eq)] +pub struct SearchResponse { + pub encrypted_output: String, +} From 7c3cc1db81a46a4f3ceeca1d7d50df0f7d10d09c Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Wed, 20 May 2026 11:58:23 -0700 Subject: [PATCH 3/5] Fix thread settings clippy failure (#23724) ## Why `main` picked up two small Rust build failures after nearby merges: - #23507 added a real handler for `ServerNotification::ThreadSettingsUpdated`, but the same variant was still listed in the ignored-notification match arm. Full Clippy runs treat the resulting unreachable-pattern warning as an error. - #23666 added `turn_id` and `truncation_policy` to `codex_tools::ToolCall`, while the goal extension backend test fixtures from the goal-extension work still used the old shape. That left `codex-goal-extension` tests unable to compile once the branches met on `main`. ## What changed Removed the duplicate `ThreadSettingsUpdated` match pattern from `tui/src/chatwidget/protocol.rs`. Updated the goal extension test `tool_call` helper to populate the new `ToolCall` fields, and reused that helper for the one direct literal that still had the old field list. ## Verification - `just fix -p codex-tui` - `cargo test -p codex-goal-extension` --- .../ext/goal/tests/goal_extension_backend.rs | 23 +++++++++---------- codex-rs/tui/src/chatwidget/protocol.rs | 1 - 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/codex-rs/ext/goal/tests/goal_extension_backend.rs b/codex-rs/ext/goal/tests/goal_extension_backend.rs index d04dbe6021..c2c0a2306c 100644 --- a/codex-rs/ext/goal/tests/goal_extension_backend.rs +++ b/codex-rs/ext/goal/tests/goal_extension_backend.rs @@ -12,7 +12,6 @@ use codex_extension_api::ToolCallOutcome; use codex_extension_api::ToolCallSource; use codex_extension_api::ToolExecutor; use codex_extension_api::ToolFinishInput; -use codex_extension_api::ToolName; use codex_extension_api::ToolPayload; use codex_extension_api::TurnStartInput; use codex_extension_api::TurnStopInput; @@ -27,6 +26,7 @@ use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::ThreadGoalStatus; use codex_protocol::protocol::TokenUsage; use codex_protocol::protocol::TokenUsageInfo; +use codex_protocol::protocol::TruncationPolicy; use pretty_assertions::assert_eq; use serde_json::json; use tempfile::TempDir; @@ -39,17 +39,14 @@ async fn installed_goal_tools_create_goal_and_fill_empty_preview() -> anyhow::Re let tools = installed_tools(runtime.clone(), thread_id).await; let create_tool = tool_by_name(&tools, "create_goal"); - let invocation = ToolCall { - call_id: "call-create-goal".to_string(), - tool_name: ToolName::plain("create_goal"), - payload: ToolPayload::Function { - arguments: json!({ - "objective": "ship goal extension backend", - "token_budget": 123, - }) - .to_string(), - }, - }; + let invocation = tool_call( + "create_goal", + "call-create-goal", + json!({ + "objective": "ship goal extension backend", + "token_budget": 123, + }), + ); let output = create_tool.handle(invocation.clone()).await?; let result = output.code_mode_result(&invocation.payload); assert_eq!( @@ -534,8 +531,10 @@ fn tool_by_name<'a>( fn tool_call(tool_name: &str, call_id: &str, arguments: serde_json::Value) -> ToolCall { ToolCall { + turn_id: "turn-1".to_string(), call_id: call_id.to_string(), tool_name: codex_extension_api::ToolName::plain(tool_name), + truncation_policy: TruncationPolicy::Bytes(1024), payload: ToolPayload::Function { arguments: arguments.to_string(), }, diff --git a/codex-rs/tui/src/chatwidget/protocol.rs b/codex-rs/tui/src/chatwidget/protocol.rs index 6a980e347c..b3b58cbc82 100644 --- a/codex-rs/tui/src/chatwidget/protocol.rs +++ b/codex-rs/tui/src/chatwidget/protocol.rs @@ -220,7 +220,6 @@ impl ChatWidget { | ServerNotification::AccountRateLimitsUpdated(_) | ServerNotification::ThreadStarted(_) | ServerNotification::ThreadStatusChanged(_) - | ServerNotification::ThreadSettingsUpdated(_) | ServerNotification::ThreadArchived(_) | ServerNotification::ThreadUnarchived(_) | ServerNotification::RawResponseItemCompleted(_) From e389e01f8347a4861ed139b01956fe64aa0c0fda Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Wed, 20 May 2026 12:02:32 -0700 Subject: [PATCH 4/5] npm: ship platform packages in Codex package layout (#23637) ## Summary The npm platform packages should stop carrying a bespoke native layout now that the release workflow builds canonical Codex package archives. Keeping npm on the same `bin/`, `codex-resources/`, and `codex-path/` structure lets the Rust package-layout detection behave consistently across standalone, npm, and future DotSlash installs. This changes platform npm packages to stage the `codex-package` artifact for each target under `vendor/`. The Node launcher now resolves `bin/codex` and prepends `codex-path`, while retaining legacy `vendor//codex` and `vendor//path` fallback support for local development and migration. The npm staging helper downloads `codex-package` archives instead of rebuilding the CLI payload from individual `codex`, `rg`, `bwrap`, and sandbox helper artifacts. CI still needs to stage npm packages from historical rust-release workflow artifacts that predate package archives, so the staging scripts expose an explicit `--allow-legacy-codex-package` fallback. That fallback synthesizes the canonical package layout from legacy per-binary artifacts and is wired only into the CI smoke path; release staging remains strict and continues to require real package archives. For direct local use, `install_native_deps.py` now points its built-in default workflow at the same recent artifact run used by CI and automatically enables legacy package synthesis only when `--workflow-url` is omitted. Explicit workflow URLs remain strict unless callers opt in with `--allow-legacy-codex-package`. ## Test plan - `python3 -m py_compile codex-cli/scripts/build_npm_package.py codex-cli/scripts/install_native_deps.py scripts/stage_npm_packages.py scripts/codex_package/cli.py` - `node --check codex-cli/bin/codex.js` - `ruby -e 'require "yaml"; YAML.load_file(".github/workflows/rust-release.yml"); YAML.load_file(".github/workflows/ci.yml"); puts "ok"'` - Staged a synthetic `codex-linux-x64` platform package from a canonical vendor tree and verified it copied only `bin/`, `codex-path/`, `codex-resources/`, and `codex-package.json`. - Imported `install_native_deps.py` and extracted a synthetic `codex-package-x86_64-unknown-linux-musl.tar.gz` into `vendor/`. - Ran legacy-layout conversion smokes for Linux, Windows, and unsigned macOS artifact naming. - Ran a synthetic `install_native_deps.py` default-workflow smoke that verifies legacy package synthesis is automatic only when `--workflow-url` is omitted. - `NPM_CONFIG_CACHE="$tmp_dir/npm-cache" python3 ./scripts/stage_npm_packages.py --release-version 0.125.0 --workflow-url https://github.com/openai/codex/actions/runs/26131514935 --package codex --allow-legacy-codex-package --output-dir "$tmp_dir"` - `node codex-cli/bin/codex.js --version` --- [//]: # (BEGIN SAPLING FOOTER) Stack created with [Sapling](https://sapling-scm.com). Best reviewed with [ReviewStack](https://reviewstack.dev/openai/codex/pull/23637). * #23638 * __->__ #23637 --- .github/workflows/ci.yml | 8 +- .github/workflows/rust-release.yml | 2 - codex-cli/bin/codex.js | 60 ++++--- codex-cli/scripts/README.md | 8 +- codex-cli/scripts/build_npm_package.py | 70 ++++++-- codex-cli/scripts/install_native_deps.py | 210 ++++++++++++++++++++--- scripts/stage_npm_packages.py | 22 ++- 7 files changed, 312 insertions(+), 68 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a1c60acc26..97591d4c19 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,14 +53,16 @@ jobs: # Passing the workflow URL directly avoids relying on old rust-v* # branches remaining discoverable via `gh run list --branch ...`. CODEX_VERSION=0.125.0 - WORKFLOW_URL="https://github.com/openai/codex/actions/runs/24901475298" + WORKFLOW_URL="https://github.com/openai/codex/actions/runs/26131514935" OUTPUT_DIR="${RUNNER_TEMP}" - # This reused workflow predates the standalone bwrap artifact. + # This reused workflow predates codex-package archive artifacts, so + # CI synthesizes the package layout from the older per-binary + # artifacts. Release staging must use real package archives. python3 ./scripts/stage_npm_packages.py \ --release-version "$CODEX_VERSION" \ --workflow-url "$WORKFLOW_URL" \ --package codex \ - --allow-missing-native-component bwrap \ + --allow-legacy-codex-package \ --output-dir "$OUTPUT_DIR" PACK_OUTPUT="${OUTPUT_DIR}/codex-npm-${CODEX_VERSION}.tgz" echo "pack_output=$PACK_OUTPUT" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/rust-release.yml b/.github/workflows/rust-release.yml index 93e23b8fd7..7268a24ad8 100644 --- a/.github/workflows/rust-release.yml +++ b/.github/workflows/rust-release.yml @@ -1208,8 +1208,6 @@ jobs: if: ${{ env.SIGN_MACOS == 'true' }} run: pnpm install --frozen-lockfile - # stage_npm_packages.py requires DotSlash when staging releases. - - uses: facebook/install-dotslash@1e4e7b3e07eaca387acb98f1d4720e0bee8dbb6a # v2 - name: Stage npm packages if: ${{ env.SIGN_MACOS == 'true' }} env: diff --git a/codex-cli/bin/codex.js b/codex-cli/bin/codex.js index 475239549a..1a43ce7e1f 100755 --- a/codex-cli/bin/codex.js +++ b/codex-cli/bin/codex.js @@ -77,33 +77,43 @@ if (!platformPackage) { const codexBinaryName = process.platform === "win32" ? "codex.exe" : "codex"; const localVendorRoot = path.join(__dirname, "..", "vendor"); -const localBinaryPath = path.join( - localVendorRoot, - targetTriple, - "codex", - codexBinaryName, -); +const packageBinaryPath = (vendorRoot) => + path.join(vendorRoot, targetTriple, "bin", codexBinaryName); +const legacyBinaryPath = (vendorRoot) => + path.join(vendorRoot, targetTriple, "codex", codexBinaryName); -let vendorRoot; -try { - const packageJsonPath = require.resolve(`${platformPackage}/package.json`); - vendorRoot = path.join(path.dirname(packageJsonPath), "vendor"); -} catch { - if (existsSync(localBinaryPath)) { - vendorRoot = localVendorRoot; - } else { - const packageManager = detectPackageManager(); - const updateCommand = - packageManager === "bun" - ? "bun install -g @openai/codex@latest" - : "npm install -g @openai/codex@latest"; - throw new Error( - `Missing optional dependency ${platformPackage}. Reinstall Codex: ${updateCommand}`, - ); +function resolveNativePackage(vendorRoot) { + const packageRoot = path.join(vendorRoot, targetTriple); + const binaryPath = packageBinaryPath(vendorRoot); + if (existsSync(binaryPath)) { + return { + binaryPath, + pathDir: path.join(packageRoot, "codex-path"), + }; } + + const legacyPath = legacyBinaryPath(vendorRoot); + if (existsSync(legacyPath)) { + return { + binaryPath: legacyPath, + pathDir: path.join(packageRoot, "path"), + }; + } + + return null; } -if (!vendorRoot) { +let nativePackage; +try { + const packageJsonPath = require.resolve(`${platformPackage}/package.json`); + nativePackage = resolveNativePackage( + path.join(path.dirname(packageJsonPath), "vendor"), + ); +} catch { + nativePackage = resolveNativePackage(localVendorRoot); +} + +if (!nativePackage) { const packageManager = detectPackageManager(); const updateCommand = packageManager === "bun" @@ -114,8 +124,7 @@ if (!vendorRoot) { ); } -const archRoot = path.join(vendorRoot, targetTriple); -const binaryPath = path.join(archRoot, "codex", codexBinaryName); +const { binaryPath, pathDir } = nativePackage; // Use an asynchronous spawn instead of spawnSync so that Node is able to // respond to signals (e.g. Ctrl-C / SIGINT) while the native binary is @@ -159,7 +168,6 @@ function detectPackageManager() { } const additionalDirs = []; -const pathDir = path.join(archRoot, "path"); if (existsSync(pathDir)) { additionalDirs.push(pathDir); } diff --git a/codex-cli/scripts/README.md b/codex-cli/scripts/README.md index ca0d54b544..ce58097db5 100644 --- a/codex-cli/scripts/README.md +++ b/codex-cli/scripts/README.md @@ -11,13 +11,13 @@ example, to stage the CLI, responses proxy, and SDK packages for version `0.6.0` --package codex-sdk ``` -This downloads the native artifacts once, hydrates `vendor/` for each package, and writes -tarballs to `dist/npm/`. +This downloads the native package archive artifacts once, hydrates `vendor/` for each +package, and writes tarballs to `dist/npm/`. When `--package codex` is provided, the staging helper builds the lightweight `@openai/codex` meta package plus all platform-native `@openai/codex` variants that are later published under platform-specific dist-tags. If you need to invoke `build_npm_package.py` directly, run -`codex-cli/scripts/install_native_deps.py` first and pass `--vendor-src` pointing to the -directory that contains the populated `vendor/` tree. +`codex-cli/scripts/install_native_deps.py --component codex-package` first and pass +`--vendor-src` pointing to the directory that contains the populated `vendor/` tree. diff --git a/codex-cli/scripts/build_npm_package.py b/codex-cli/scripts/build_npm_package.py index c38fed68d9..5d4bc99c4e 100755 --- a/codex-cli/scripts/build_npm_package.py +++ b/codex-cli/scripts/build_npm_package.py @@ -15,6 +15,8 @@ REPO_ROOT = CODEX_CLI_ROOT.parent RESPONSES_API_PROXY_NPM_ROOT = REPO_ROOT / "codex-rs" / "responses-api-proxy" / "npm" CODEX_SDK_ROOT = REPO_ROOT / "sdk" / "typescript" CODEX_NPM_NAME = "@openai/codex" +CODEX_PACKAGE_COMPONENT = "codex-package" +CODEX_PACKAGE_ENTRIES = ("codex-package.json", "bin", "codex-resources", "codex-path") # `npm_name` is the local optional-dependency alias consumed by `bin/codex.js`. # The underlying package published to npm is always `@openai/codex`. @@ -69,12 +71,12 @@ PACKAGE_EXPANSIONS: dict[str, list[str]] = { PACKAGE_NATIVE_COMPONENTS: dict[str, list[str]] = { "codex": [], - "codex-linux-x64": ["bwrap", "codex", "rg"], - "codex-linux-arm64": ["bwrap", "codex", "rg"], - "codex-darwin-x64": ["codex", "rg"], - "codex-darwin-arm64": ["codex", "rg"], - "codex-win32-x64": ["codex", "rg", "codex-windows-sandbox-setup", "codex-command-runner"], - "codex-win32-arm64": ["codex", "rg", "codex-windows-sandbox-setup", "codex-command-runner"], + "codex-linux-x64": [CODEX_PACKAGE_COMPONENT], + "codex-linux-arm64": [CODEX_PACKAGE_COMPONENT], + "codex-darwin-x64": [CODEX_PACKAGE_COMPONENT], + "codex-darwin-arm64": [CODEX_PACKAGE_COMPONENT], + "codex-win32-x64": [CODEX_PACKAGE_COMPONENT], + "codex-win32-arm64": [CODEX_PACKAGE_COMPONENT], "codex-responses-api-proxy": ["codex-responses-api-proxy"], "codex-sdk": [], } @@ -383,7 +385,11 @@ def copy_native_binaries( if not vendor_src.exists(): raise RuntimeError(f"Vendor source directory not found: {vendor_src}") - components_set = {component for component in components if component in COMPONENT_DEST_DIR} + components_set = { + component + for component in components + if component == CODEX_PACKAGE_COMPONENT or component in COMPONENT_DEST_DIR + } allow_missing_components = allow_missing_components or set() if not components_set: return @@ -402,11 +408,26 @@ def copy_native_binaries( if target_filter is not None and target_dir.name not in target_filter: continue - dest_target_dir = vendor_dest / target_dir.name - dest_target_dir.mkdir(parents=True, exist_ok=True) copied_targets.add(target_dir.name) - for component in components_set: + dest_target_dir = vendor_dest / target_dir.name + + if CODEX_PACKAGE_COMPONENT in components_set: + validate_codex_package_dir(target_dir) + if dest_target_dir.exists(): + shutil.rmtree(dest_target_dir) + dest_target_dir.mkdir(parents=True, exist_ok=True) + for entry in CODEX_PACKAGE_ENTRIES: + src = target_dir / entry + dest = dest_target_dir / entry + if src.is_dir(): + shutil.copytree(src, dest) + else: + shutil.copy2(src, dest) + else: + dest_target_dir.mkdir(parents=True, exist_ok=True) + + for component in components_set - {CODEX_PACKAGE_COMPONENT}: dest_dir_name = COMPONENT_DEST_DIR.get(component) if dest_dir_name is None: continue @@ -431,6 +452,35 @@ def copy_native_binaries( raise RuntimeError(f"Missing target directories in vendor source: {missing_list}") +def validate_codex_package_dir(package_dir: Path) -> None: + is_windows = "windows" in package_dir.name + required_files = [ + Path("codex-package.json"), + Path("bin") / ("codex.exe" if is_windows else "codex"), + Path("codex-path") / ("rg.exe" if is_windows else "rg"), + ] + + if "linux" in package_dir.name: + required_files.append(Path("codex-resources") / "bwrap") + + if is_windows: + required_files.extend( + [ + Path("codex-resources") / "codex-command-runner.exe", + Path("codex-resources") / "codex-windows-sandbox-setup.exe", + ] + ) + + missing_files = [ + str(relative_path) + for relative_path in required_files + if not (package_dir / relative_path).is_file() + ] + if missing_files: + missing = ", ".join(missing_files) + raise RuntimeError(f"Missing files in Codex package directory {package_dir}: {missing}") + + def run_npm_pack(staging_dir: Path, output_path: Path) -> Path: output_path = output_path.resolve() output_path.parent.mkdir(parents=True, exist_ok=True) diff --git a/codex-cli/scripts/install_native_deps.py b/codex-cli/scripts/install_native_deps.py index 79f596fbaf..b785b2a877 100755 --- a/codex-cli/scripts/install_native_deps.py +++ b/codex-cli/scripts/install_native_deps.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Install Codex native binaries (Rust CLI, bwrap, and ripgrep helpers).""" +"""Install Codex package archives and native helper binaries.""" import argparse from contextlib import contextmanager @@ -20,7 +20,7 @@ from urllib.request import urlopen SCRIPT_DIR = Path(__file__).resolve().parent CODEX_CLI_ROOT = SCRIPT_DIR.parent -DEFAULT_WORKFLOW_URL = "https://github.com/openai/codex/actions/runs/17952349351" # rust-v0.40.0 +DEFAULT_WORKFLOW_URL = "https://github.com/openai/codex/actions/runs/26131514935" # rust-v0.132.0 VENDOR_DIR_NAME = "vendor" RG_MANIFEST = CODEX_CLI_ROOT / "bin" / "rg" BINARY_TARGETS = ( @@ -31,6 +31,7 @@ BINARY_TARGETS = ( "x86_64-pc-windows-msvc", "aarch64-pc-windows-msvc", ) +CODEX_PACKAGE_COMPONENT = "codex-package" @dataclass(frozen=True) @@ -139,11 +140,20 @@ def parse_args() -> argparse.Namespace: "--component", dest="components", action="append", - choices=tuple(list(BINARY_COMPONENTS) + ["rg"]), + choices=tuple([CODEX_PACKAGE_COMPONENT, *BINARY_COMPONENTS, "rg"]), help=( "Limit installation to the specified components." - " May be repeated. Defaults to bwrap, codex, codex-windows-sandbox-setup," - " codex-command-runner, and rg." + " May be repeated. Defaults to codex-package and codex-responses-api-proxy." + ), + ) + parser.add_argument( + "--allow-legacy-codex-package", + action="store_true", + help=( + "Allow codex-package to be synthesized from legacy per-binary artifacts " + "when package archives are missing. Intended for CI compatibility only; " + "release staging should not use this. Automatically enabled for the " + "built-in default workflow." ), ) parser.add_argument( @@ -165,17 +175,11 @@ def main() -> int: vendor_dir = codex_cli_root / VENDOR_DIR_NAME vendor_dir.mkdir(parents=True, exist_ok=True) - components = args.components or [ - "bwrap", - "codex", - "codex-windows-sandbox-setup", - "codex-command-runner", - "rg", - ] + components = args.components or [CODEX_PACKAGE_COMPONENT, "codex-responses-api-proxy"] - workflow_url = (args.workflow_url or DEFAULT_WORKFLOW_URL).strip() - if not workflow_url: - workflow_url = DEFAULT_WORKFLOW_URL + workflow_override = (args.workflow_url or "").strip() + use_default_workflow = not workflow_override + workflow_url = workflow_override or DEFAULT_WORKFLOW_URL workflow_id = workflow_url.rstrip("/").split("/")[-1] print(f"Downloading native artifacts from workflow {workflow_id}...") @@ -184,6 +188,18 @@ def main() -> int: with tempfile.TemporaryDirectory(prefix="codex-native-artifacts-") as artifacts_dir_str: artifacts_dir = Path(artifacts_dir_str) _download_artifacts(workflow_id, artifacts_dir) + if CODEX_PACKAGE_COMPONENT in components: + try: + install_codex_package_archives(artifacts_dir, vendor_dir, BINARY_TARGETS) + except FileNotFoundError: + if not (args.allow_legacy_codex_package or use_default_workflow): + raise + install_legacy_codex_package_layouts( + artifacts_dir, + vendor_dir, + BINARY_TARGETS, + manifest_path=RG_MANIFEST, + ) install_binary_components( artifacts_dir, vendor_dir, @@ -199,6 +215,135 @@ def main() -> int: return 0 +def install_codex_package_archives( + artifacts_dir: Path, + vendor_dir: Path, + targets: Sequence[str], +) -> None: + targets = list(targets) + if not targets: + return + + print("Installing Codex package archives for targets: " + ", ".join(targets)) + max_workers = min(len(targets), max(1, (os.cpu_count() or 1))) + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = { + executor.submit( + _install_single_codex_package_archive, + artifacts_dir, + vendor_dir, + target, + ): target + for target in targets + } + for future in as_completed(futures): + installed_path = future.result() + print(f" installed {installed_path}") + + +def _install_single_codex_package_archive( + artifacts_dir: Path, + vendor_dir: Path, + target: str, +) -> Path: + artifact_subdir = artifact_dir_for_target(artifacts_dir, target) + archive_path = artifact_subdir / f"codex-package-{target}.tar.gz" + if not archive_path.exists(): + raise FileNotFoundError(f"Expected package archive not found: {archive_path}") + + dest_dir = vendor_dir / target + if dest_dir.exists(): + shutil.rmtree(dest_dir) + dest_dir.mkdir(parents=True, exist_ok=True) + + with tarfile.open(archive_path, "r:gz") as archive: + archive.extractall(dest_dir, filter="data") + + return dest_dir + + +def install_legacy_codex_package_layouts( + artifacts_dir: Path, + vendor_dir: Path, + targets: Sequence[str], + *, + manifest_path: Path, +) -> None: + targets = list(targets) + print( + "Synthesizing Codex package layouts from legacy artifacts for targets: " + + ", ".join(targets) + ) + with tempfile.TemporaryDirectory(prefix="codex-legacy-package-") as legacy_vendor_dir_str: + legacy_vendor_dir = Path(legacy_vendor_dir_str) + install_binary_components( + artifacts_dir, + legacy_vendor_dir, + [ + BINARY_COMPONENTS["codex"], + BINARY_COMPONENTS["bwrap"], + BINARY_COMPONENTS["codex-windows-sandbox-setup"], + BINARY_COMPONENTS["codex-command-runner"], + ], + ) + fetch_rg(legacy_vendor_dir, targets, manifest_path=manifest_path) + + for target in targets: + dest_dir = vendor_dir / target + if dest_dir.exists(): + shutil.rmtree(dest_dir) + _build_legacy_codex_package_layout(legacy_vendor_dir / target, dest_dir, target) + print(f" synthesized {dest_dir}") + + +def _build_legacy_codex_package_layout( + legacy_target_dir: Path, + package_dir: Path, + target: str, +) -> None: + is_windows = "windows" in target + exe_suffix = ".exe" if is_windows else "" + package_dir.mkdir(parents=True) + + bin_dir = package_dir / "bin" + resources_dir = package_dir / "codex-resources" + path_dir = package_dir / "codex-path" + bin_dir.mkdir() + resources_dir.mkdir() + path_dir.mkdir() + + shutil.copy2( + legacy_target_dir / "codex" / f"codex{exe_suffix}", + bin_dir / f"codex{exe_suffix}", + ) + shutil.copy2( + legacy_target_dir / "path" / f"rg{exe_suffix}", + path_dir / f"rg{exe_suffix}", + ) + + if is_windows: + for helper in [ + "codex-command-runner.exe", + "codex-windows-sandbox-setup.exe", + ]: + shutil.copy2(legacy_target_dir / "codex" / helper, resources_dir / helper) + elif "linux" in target: + shutil.copy2(legacy_target_dir / "codex-resources" / "bwrap", resources_dir / "bwrap") + + write_json( + package_dir / "codex-package.json", + { + "layoutVersion": 1, + "version": "unknown", + "target": target, + "variant": "codex", + "entrypoint": f"bin/codex{exe_suffix}", + "resourcesDir": "codex-resources", + "pathDir": "codex-path", + }, + ) + + def fetch_rg( vendor_dir: Path, targets: Sequence[str] | None = None, @@ -319,11 +464,8 @@ def _install_single_binary( target: str, component: BinaryComponent, ) -> Path: - artifact_subdir = artifacts_dir / target - archive_name = _archive_name_for_target(component.artifact_prefix, target) - archive_path = artifact_subdir / archive_name - if not archive_path.exists(): - raise FileNotFoundError(f"Expected artifact not found: {archive_path}") + artifact_subdir = artifact_dir_for_target(artifacts_dir, target) + archive_path = legacy_binary_archive_path(artifact_subdir, component.artifact_prefix, target) dest_dir = vendor_dir / target / component.dest_dir dest_dir.mkdir(parents=True, exist_ok=True) @@ -345,6 +487,28 @@ def _archive_name_for_target(artifact_prefix: str, target: str) -> str: return f"{artifact_prefix}-{target}.zst" +def legacy_binary_archive_path(artifact_dir: Path, artifact_prefix: str, target: str) -> Path: + archive_names = [_archive_name_for_target(artifact_prefix, target)] + if artifact_dir.name == f"{target}-unsigned": + archive_names.append(_archive_name_for_target(artifact_prefix, f"{target}-unsigned")) + + for archive_name in archive_names: + archive_path = artifact_dir / archive_name + if archive_path.exists(): + return archive_path + + raise FileNotFoundError(f"Expected artifact not found: {artifact_dir / archive_names[0]}") + + +def artifact_dir_for_target(artifacts_dir: Path, target: str) -> Path: + for artifact_name in [target, f"{target}-unsigned"]: + artifact_dir = artifacts_dir / artifact_name + if artifact_dir.is_dir(): + return artifact_dir + + return artifacts_dir / target + + def _fetch_single_rg( vendor_dir: Path, target: str, @@ -477,6 +641,12 @@ def _load_manifest(manifest_path: Path) -> dict: return manifest +def write_json(path: Path, value: object) -> None: + with open(path, "w", encoding="utf-8") as out: + json.dump(value, out, indent=2) + out.write("\n") + + if __name__ == "__main__": import sys diff --git a/scripts/stage_npm_packages.py b/scripts/stage_npm_packages.py index 2a0cd0028d..4eb69053eb 100755 --- a/scripts/stage_npm_packages.py +++ b/scripts/stage_npm_packages.py @@ -1,8 +1,6 @@ #!/usr/bin/env python3 """Stage one or more Codex npm packages for release.""" -from __future__ import annotations - import argparse import importlib.util import json @@ -68,6 +66,15 @@ def parse_args() -> argparse.Namespace: "Intended for CI compatibility only; release staging should not use this." ), ) + parser.add_argument( + "--allow-legacy-codex-package", + action="store_true", + help=( + "Allow codex-package layouts to be synthesized from legacy per-binary " + "workflow artifacts. Intended for CI compatibility only; release staging " + "should not use this." + ), + ) return parser.parse_args() @@ -124,11 +131,15 @@ def install_native_components( workflow_url: str, components: set[str], vendor_root: Path, + *, + allow_legacy_codex_package: bool, ) -> None: if not components: return cmd = [str(INSTALL_NATIVE_DEPS), "--workflow-url", workflow_url] + if allow_legacy_codex_package: + cmd.append("--allow-legacy-codex-package") for component in sorted(components): cmd.extend(["--component", component]) cmd.append(str(vendor_root)) @@ -172,7 +183,12 @@ def main() -> int: args.release_version, args.workflow_url ) vendor_temp_root = Path(tempfile.mkdtemp(prefix="npm-native-", dir=runner_temp)) - install_native_components(workflow_url, native_components_to_install, vendor_temp_root) + install_native_components( + workflow_url, + native_components_to_install, + vendor_temp_root, + allow_legacy_codex_package=args.allow_legacy_codex_package, + ) vendor_src = vendor_temp_root / "vendor" if resolved_head_sha: From f858dff81677684148b7a1a4597147504ae799df Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Wed, 20 May 2026 12:02:38 -0700 Subject: [PATCH 5/5] dotslash: publish Codex entrypoints from package archives ## Summary DotSlash should resolve the same canonical package layout used by standalone installers and npm platform packages, but it should not take the size regression of downloading gzip-compressed package archives. Installers continue to use `.tar.gz` because that format is broadly available, while DotSlash can use zstd-compressed tar archives. This updates the package archive release helper to emit both `*.tar.gz` and `*.tar.zst` package archives from the same package directory. The Codex CLI and `codex-app-server` DotSlash release config entries now match `codex-package-.tar.zst` and `codex-app-server-package-.tar.zst`, with paths that select `bin/codex` or `bin/codex-app-server` inside the extracted package. The Windows release compression loop also skips prebuilt `.tar.zst` archives so it does not try to recompress them as ordinary files. The other helper outputs stay on their existing per-binary artifacts for now. ## Test plan - `python3 -m json.tool .github/dotslash-config.json > /dev/null` - `bash -n .github/scripts/build-codex-package-archive.sh` - `python3 -m py_compile scripts/codex_package/cli.py` - `ruby -e 'require "yaml"; YAML.load_file(".github/workflows/rust-release-windows.yml"); puts "ok"'` - Ran a Python regex smoke test that checked every updated `codex` and `codex-app-server` platform entry against the `.tar.zst` archive names emitted by `.github/scripts/build-codex-package-archive.sh`. - Ran `scripts/build_codex_package.py` with synthetic macOS entrypoint and `rg` binaries and repeated `--archive-output` values for `.tar.gz` and `.tar.zst`, verifying both archives were written. --- .github/dotslash-config.json | 48 +++++++++---------- .../scripts/build-codex-package-archive.sh | 10 +++- .github/workflows/rust-release-windows.yml | 2 +- scripts/codex_package/cli.py | 9 ++-- 4 files changed, 38 insertions(+), 31 deletions(-) diff --git a/.github/dotslash-config.json b/.github/dotslash-config.json index a0297c269a..78ea6b37e6 100644 --- a/.github/dotslash-config.json +++ b/.github/dotslash-config.json @@ -3,56 +3,56 @@ "codex": { "platforms": { "macos-aarch64": { - "regex": "^codex-aarch64-apple-darwin\\.zst$", - "path": "codex" + "regex": "^codex-package-aarch64-apple-darwin\\.tar\\.zst$", + "path": "bin/codex" }, "macos-x86_64": { - "regex": "^codex-x86_64-apple-darwin\\.zst$", - "path": "codex" + "regex": "^codex-package-x86_64-apple-darwin\\.tar\\.zst$", + "path": "bin/codex" }, "linux-x86_64": { - "regex": "^codex-x86_64-unknown-linux-musl-bundle\\.tar\\.zst$", - "path": "codex" + "regex": "^codex-package-x86_64-unknown-linux-musl\\.tar\\.zst$", + "path": "bin/codex" }, "linux-aarch64": { - "regex": "^codex-aarch64-unknown-linux-musl-bundle\\.tar\\.zst$", - "path": "codex" + "regex": "^codex-package-aarch64-unknown-linux-musl\\.tar\\.zst$", + "path": "bin/codex" }, "windows-x86_64": { - "regex": "^codex-x86_64-pc-windows-msvc\\.exe\\.zst$", - "path": "codex.exe" + "regex": "^codex-package-x86_64-pc-windows-msvc\\.tar\\.zst$", + "path": "bin/codex.exe" }, "windows-aarch64": { - "regex": "^codex-aarch64-pc-windows-msvc\\.exe\\.zst$", - "path": "codex.exe" + "regex": "^codex-package-aarch64-pc-windows-msvc\\.tar\\.zst$", + "path": "bin/codex.exe" } } }, "codex-app-server": { "platforms": { "macos-aarch64": { - "regex": "^codex-app-server-aarch64-apple-darwin\\.zst$", - "path": "codex-app-server" + "regex": "^codex-app-server-package-aarch64-apple-darwin\\.tar\\.zst$", + "path": "bin/codex-app-server" }, "macos-x86_64": { - "regex": "^codex-app-server-x86_64-apple-darwin\\.zst$", - "path": "codex-app-server" + "regex": "^codex-app-server-package-x86_64-apple-darwin\\.tar\\.zst$", + "path": "bin/codex-app-server" }, "linux-x86_64": { - "regex": "^codex-app-server-x86_64-unknown-linux-musl\\.zst$", - "path": "codex-app-server" + "regex": "^codex-app-server-package-x86_64-unknown-linux-musl\\.tar\\.zst$", + "path": "bin/codex-app-server" }, "linux-aarch64": { - "regex": "^codex-app-server-aarch64-unknown-linux-musl\\.zst$", - "path": "codex-app-server" + "regex": "^codex-app-server-package-aarch64-unknown-linux-musl\\.tar\\.zst$", + "path": "bin/codex-app-server" }, "windows-x86_64": { - "regex": "^codex-app-server-x86_64-pc-windows-msvc\\.exe\\.zst$", - "path": "codex-app-server.exe" + "regex": "^codex-app-server-package-x86_64-pc-windows-msvc\\.tar\\.zst$", + "path": "bin/codex-app-server.exe" }, "windows-aarch64": { - "regex": "^codex-app-server-aarch64-pc-windows-msvc\\.exe\\.zst$", - "path": "codex-app-server.exe" + "regex": "^codex-app-server-package-aarch64-pc-windows-msvc\\.tar\\.zst$", + "path": "bin/codex-app-server.exe" } } }, diff --git a/.github/scripts/build-codex-package-archive.sh b/.github/scripts/build-codex-package-archive.sh index 90eae12ef0..c475ffe952 100644 --- a/.github/scripts/build-codex-package-archive.sh +++ b/.github/scripts/build-codex-package-archive.sh @@ -97,9 +97,14 @@ else python_bin="python" fi +if ! command -v zstd >/dev/null 2>&1 && [[ -x "${repo_root}/.github/workflows/zstd" ]]; then + export PATH="${repo_root}/.github/workflows:${PATH}" +fi + mkdir -p "$archive_dir" package_dir="${RUNNER_TEMP:-/tmp}/${archive_stem}-${target}" -archive_path="${archive_dir}/${archive_stem}-${target}.tar.gz" +gzip_archive_path="${archive_dir}/${archive_stem}-${target}.tar.gz" +zstd_archive_path="${archive_dir}/${archive_stem}-${target}.tar.zst" rm -rf "$package_dir" "$python_bin" "${repo_root}/scripts/build_codex_package.py" \ @@ -108,5 +113,6 @@ rm -rf "$package_dir" --entrypoint-bin "${entrypoint_dir%/}/${entrypoint_name}${exe_suffix}" \ --cargo-profile release \ --package-dir "$package_dir" \ - --archive-output "$archive_path" \ + --archive-output "$gzip_archive_path" \ + --archive-output "$zstd_archive_path" \ --force diff --git a/.github/workflows/rust-release-windows.yml b/.github/workflows/rust-release-windows.yml index ac28b7855a..81b5a1e51b 100644 --- a/.github/workflows/rust-release-windows.yml +++ b/.github/workflows/rust-release-windows.yml @@ -295,7 +295,7 @@ jobs: base="$(basename "$f")" # Skip files that are already archives (shouldn't happen, but be # safe). - if [[ "$base" == *.tar.gz || "$base" == *.zip || "$base" == *.dmg ]]; then + if [[ "$base" == *.tar.gz || "$base" == *.tar.zst || "$base" == *.zip || "$base" == *.dmg ]]; then continue fi diff --git a/scripts/codex_package/cli.py b/scripts/codex_package/cli.py index 16324cf44d..0ca9f5d35e 100644 --- a/scripts/codex_package/cli.py +++ b/scripts/codex_package/cli.py @@ -50,9 +50,11 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--archive-output", type=Path, + action="append", + default=[], help=( - "Optional archive output path. Supported suffixes: .tar.gz, .tgz, " - ".tar.zst, .zip." + "Optional archive output path. May be repeated. Supported suffixes: " + ".tar.gz, .tgz, .tar.zst, .zip." ), ) parser.add_argument( @@ -130,8 +132,7 @@ def main() -> int: build_package_dir(package_dir, version, variant, spec, inputs) validate_package_dir(package_dir, variant, spec) - archive_output = args.archive_output - if archive_output is not None: + for archive_output in args.archive_output: archive_path = archive_output.resolve() write_archive(package_dir, archive_path, force=args.force) print(f"Built Codex package archive at {archive_path}")