From b519267d05459095eccb9767c28550a2952d8a2f Mon Sep 17 00:00:00 2001 From: Ahmed Ibrahim Date: Fri, 21 Nov 2025 19:06:45 -0800 Subject: [PATCH 01/61] Account for encrypted reasoning for auto compaction (#7113) - The total token used returned from the api doesn't account for the reasoning items before the assistant message - Account for those for auto compaction - Add the encrypted reasoning effort in the common tests utils - Add a test to make sure it works as expected --- codex-rs/Cargo.lock | 1 + codex-rs/core/src/codex.rs | 35 ++--- codex-rs/core/src/context_manager/history.rs | 49 +++++++ .../core/src/context_manager/history_tests.rs | 33 +++++ codex-rs/core/src/state/session.rs | 4 + codex-rs/core/src/truncate.rs | 2 +- codex-rs/core/tests/common/Cargo.toml | 1 + codex-rs/core/tests/common/responses.rs | 7 + codex-rs/core/tests/suite/compact.rs | 134 +++++++++++++++++- 9 files changed, 236 insertions(+), 30 deletions(-) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index ef5d6b272b..0bcfd4f37a 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1777,6 +1777,7 @@ version = "0.0.0" dependencies = [ "anyhow", "assert_cmd", + "base64", "codex-core", "codex-protocol", "notify", diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index a91024348a..96402961bf 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -661,6 +661,11 @@ impl Session { format!("auto-compact-{id}") } + async fn get_total_token_usage(&self) -> i64 { + let state = self.state.lock().await; + state.get_total_token_usage() + } + async fn record_initial_history(&self, conversation_history: InitialHistory) { let turn_context = self.new_turn(SessionSettingsUpdate::default()).await; match conversation_history { @@ -1958,20 +1963,13 @@ pub(crate) async fn run_task( .await { Ok(turn_output) => { - let TurnRunResult { - processed_items, - total_token_usage, - } = turn_output; + let processed_items = turn_output; let limit = turn_context .client .get_auto_compact_token_limit() .unwrap_or(i64::MAX); - let total_usage_tokens = total_token_usage - .as_ref() - .map(TokenUsage::tokens_in_context_window); - let token_limit_reached = total_usage_tokens - .map(|tokens| tokens >= limit) - .unwrap_or(false); + let total_usage_tokens = sess.get_total_token_usage().await; + let token_limit_reached = total_usage_tokens >= limit; let (responses, items_to_record_in_conversation_history) = process_items(processed_items, &sess, &turn_context).await; @@ -2028,7 +2026,7 @@ async fn run_turn( turn_diff_tracker: SharedTurnDiffTracker, input: Vec, cancellation_token: CancellationToken, -) -> CodexResult { +) -> CodexResult> { let mcp_tools = sess .services .mcp_connection_manager @@ -2159,12 +2157,6 @@ pub struct ProcessedResponseItem { pub response: Option, } -#[derive(Debug)] -struct TurnRunResult { - processed_items: Vec, - total_token_usage: Option, -} - #[allow(clippy::too_many_arguments)] async fn try_run_turn( router: Arc, @@ -2173,7 +2165,7 @@ async fn try_run_turn( turn_diff_tracker: SharedTurnDiffTracker, prompt: &Prompt, cancellation_token: CancellationToken, -) -> CodexResult { +) -> CodexResult> { let rollout_item = RolloutItem::TurnContext(TurnContextItem { cwd: turn_context.cwd.clone(), approval_policy: turn_context.approval_policy, @@ -2335,12 +2327,7 @@ async fn try_run_turn( sess.send_event(&turn_context, msg).await; } - let result = TurnRunResult { - processed_items, - total_token_usage: token_usage.clone(), - }; - - return Ok(result); + return Ok(processed_items); } ResponseEvent::OutputTextDelta(delta) => { // In review child threads, suppress assistant text deltas; the diff --git a/codex-rs/core/src/context_manager/history.rs b/codex-rs/core/src/context_manager/history.rs index 8eefcbf855..035d0d2867 100644 --- a/codex-rs/core/src/context_manager/history.rs +++ b/codex-rs/core/src/context_manager/history.rs @@ -2,6 +2,7 @@ use crate::codex::TurnContext; use crate::context_manager::normalize; use crate::truncate::TruncationPolicy; use crate::truncate::approx_token_count; +use crate::truncate::approx_tokens_from_byte_count; use crate::truncate::truncate_function_output_items_with_policy; use crate::truncate::truncate_text; use codex_protocol::models::FunctionCallOutputPayload; @@ -119,6 +120,54 @@ impl ContextManager { ); } + fn get_non_last_reasoning_items_tokens(&self) -> usize { + // get reasoning items excluding all the ones after the last user message + let Some(last_user_index) = self + .items + .iter() + .rposition(|item| matches!(item, ResponseItem::Message { role, .. } if role == "user")) + else { + return 0usize; + }; + + let total_reasoning_bytes = self + .items + .iter() + .take(last_user_index) + .filter_map(|item| { + if let ResponseItem::Reasoning { + encrypted_content: Some(content), + .. + } = item + { + Some(content.len()) + } else { + None + } + }) + .map(Self::estimate_reasoning_length) + .fold(0usize, usize::saturating_add); + + let token_estimate = approx_tokens_from_byte_count(total_reasoning_bytes); + token_estimate as usize + } + + fn estimate_reasoning_length(encoded_len: usize) -> usize { + encoded_len + .saturating_mul(3) + .checked_div(4) + .unwrap_or(0) + .saturating_sub(650) + } + + pub(crate) fn get_total_token_usage(&self) -> i64 { + self.token_info + .as_ref() + .map(|info| info.last_token_usage.total_tokens) + .unwrap_or(0) + .saturating_add(self.get_non_last_reasoning_items_tokens() as i64) + } + /// This function enforces a couple of invariants on the in-memory history: /// 1. every call (function/custom) has a corresponding output entry /// 2. every output has a corresponding call entry diff --git a/codex-rs/core/src/context_manager/history_tests.rs b/codex-rs/core/src/context_manager/history_tests.rs index 1a01604a70..cb59c97231 100644 --- a/codex-rs/core/src/context_manager/history_tests.rs +++ b/codex-rs/core/src/context_manager/history_tests.rs @@ -56,6 +56,17 @@ fn reasoning_msg(text: &str) -> ResponseItem { } } +fn reasoning_with_encrypted_content(len: usize) -> ResponseItem { + ResponseItem::Reasoning { + id: String::new(), + summary: vec![ReasoningItemReasoningSummary::SummaryText { + text: "summary".to_string(), + }], + content: None, + encrypted_content: Some("a".repeat(len)), + } +} + fn truncate_exec_output(content: &str) -> String { truncate::truncate_text(content, TruncationPolicy::Tokens(EXEC_FORMAT_MAX_TOKENS)) } @@ -112,6 +123,28 @@ fn filters_non_api_messages() { ); } +#[test] +fn non_last_reasoning_tokens_return_zero_when_no_user_messages() { + let history = create_history_with_items(vec![reasoning_with_encrypted_content(800)]); + + assert_eq!(history.get_non_last_reasoning_items_tokens(), 0); +} + +#[test] +fn non_last_reasoning_tokens_ignore_entries_after_last_user() { + let history = create_history_with_items(vec![ + reasoning_with_encrypted_content(900), + user_msg("first"), + reasoning_with_encrypted_content(1_000), + user_msg("second"), + reasoning_with_encrypted_content(2_000), + ]); + // first: (900 * 0.75 - 650) / 4 = 6.25 tokens + // second: (1000 * 0.75 - 650) / 4 = 25 tokens + // first + second = 62.5 + assert_eq!(history.get_non_last_reasoning_items_tokens(), 32); +} + #[test] fn get_history_for_prompt_drops_ghost_commits() { let items = vec![ResponseItem::GhostSnapshot { diff --git a/codex-rs/core/src/state/session.rs b/codex-rs/core/src/state/session.rs index 2dfa5199f1..caebac6b86 100644 --- a/codex-rs/core/src/state/session.rs +++ b/codex-rs/core/src/state/session.rs @@ -74,4 +74,8 @@ impl SessionState { pub(crate) fn set_token_usage_full(&mut self, context_window: i64) { self.history.set_token_usage_full(context_window); } + + pub(crate) fn get_total_token_usage(&self) -> i64 { + self.history.get_total_token_usage() + } } diff --git a/codex-rs/core/src/truncate.rs b/codex-rs/core/src/truncate.rs index bf883c0612..6e38ef6986 100644 --- a/codex-rs/core/src/truncate.rs +++ b/codex-rs/core/src/truncate.rs @@ -296,7 +296,7 @@ fn approx_bytes_for_tokens(tokens: usize) -> usize { tokens.saturating_mul(APPROX_BYTES_PER_TOKEN) } -fn approx_tokens_from_byte_count(bytes: usize) -> u64 { +pub(crate) fn approx_tokens_from_byte_count(bytes: usize) -> u64 { let bytes_u64 = bytes as u64; bytes_u64.saturating_add((APPROX_BYTES_PER_TOKEN as u64).saturating_sub(1)) / (APPROX_BYTES_PER_TOKEN as u64) diff --git a/codex-rs/core/tests/common/Cargo.toml b/codex-rs/core/tests/common/Cargo.toml index 094f33a26b..4c47fbb527 100644 --- a/codex-rs/core/tests/common/Cargo.toml +++ b/codex-rs/core/tests/common/Cargo.toml @@ -9,6 +9,7 @@ path = "lib.rs" [dependencies] anyhow = { workspace = true } assert_cmd = { workspace = true } +base64 = { workspace = true } codex-core = { workspace = true } codex-protocol = { workspace = true } notify = { workspace = true } diff --git a/codex-rs/core/tests/common/responses.rs b/codex-rs/core/tests/common/responses.rs index b84e966399..7cc59f26ef 100644 --- a/codex-rs/core/tests/common/responses.rs +++ b/codex-rs/core/tests/common/responses.rs @@ -2,6 +2,7 @@ use std::sync::Arc; use std::sync::Mutex; use anyhow::Result; +use base64::Engine; use serde_json::Value; use wiremock::BodyPrintLimit; use wiremock::Match; @@ -297,12 +298,18 @@ pub fn ev_reasoning_item(id: &str, summary: &[&str], raw_content: &[&str]) -> Va .map(|text| serde_json::json!({"type": "summary_text", "text": text})) .collect(); + let overhead = "b".repeat(550); + let raw_content_joined = raw_content.join(""); + let encrypted_content = + base64::engine::general_purpose::STANDARD.encode(overhead + raw_content_joined.as_str()); + let mut event = serde_json::json!({ "type": "response.output_item.done", "item": { "type": "reasoning", "id": id, "summary": summary_entries, + "encrypted_content": encrypted_content, } }); diff --git a/codex-rs/core/tests/suite/compact.rs b/codex-rs/core/tests/suite/compact.rs index 1324d3edb9..aa74ec8978 100644 --- a/codex-rs/core/tests/suite/compact.rs +++ b/codex-rs/core/tests/suite/compact.rs @@ -7,6 +7,7 @@ use codex_core::built_in_model_providers; use codex_core::compact::SUMMARIZATION_PROMPT; use codex_core::compact::SUMMARY_PREFIX; use codex_core::config::Config; +use codex_core::features::Feature; use codex_core::protocol::EventMsg; use codex_core::protocol::Op; use codex_core::protocol::RolloutItem; @@ -27,6 +28,7 @@ use core_test_support::responses::ev_assistant_message; use core_test_support::responses::ev_completed; use core_test_support::responses::ev_completed_with_tokens; use core_test_support::responses::ev_function_call; +use core_test_support::responses::mount_compact_json_once; use core_test_support::responses::mount_sse_once; use core_test_support::responses::mount_sse_once_match; use core_test_support::responses::mount_sse_sequence; @@ -481,9 +483,14 @@ async fn multiple_auto_compact_per_task_runs_after_token_limit_hit() { // mock responses from the model + let reasoning_response_1 = ev_reasoning_item("m1", &["I will create a react app"], &[]); + let encrypted_content_1 = reasoning_response_1["item"]["encrypted_content"] + .as_str() + .unwrap(); + // first chunk of work let model_reasoning_response_1_sse = sse(vec![ - ev_reasoning_item("m1", &["I will create a react app"], &[]), + reasoning_response_1.clone(), ev_local_shell_call("r1-shell", "completed", vec!["echo", "make-react"]), ev_completed_with_tokens("r1", token_count_used), ]); @@ -494,9 +501,14 @@ async fn multiple_auto_compact_per_task_runs_after_token_limit_hit() { ev_completed_with_tokens("r2", token_count_used_after_compaction), ]); + let reasoning_response_2 = ev_reasoning_item("m3", &["I will create a node app"], &[]); + let encrypted_content_2 = reasoning_response_2["item"]["encrypted_content"] + .as_str() + .unwrap(); + // second chunk of work let model_reasoning_response_2_sse = sse(vec![ - ev_reasoning_item("m3", &["I will create a node app"], &[]), + reasoning_response_2.clone(), ev_local_shell_call("r3-shell", "completed", vec!["echo", "make-node"]), ev_completed_with_tokens("r3", token_count_used), ]); @@ -507,6 +519,11 @@ async fn multiple_auto_compact_per_task_runs_after_token_limit_hit() { ev_completed_with_tokens("r4", token_count_used_after_compaction), ]); + let reasoning_response_3 = ev_reasoning_item("m6", &["I will create a python app"], &[]); + let encrypted_content_3 = reasoning_response_3["item"]["encrypted_content"] + .as_str() + .unwrap(); + // third chunk of work let model_reasoning_response_3_sse = sse(vec![ ev_reasoning_item("m6", &["I will create a python app"], &[]), @@ -635,7 +652,7 @@ async fn multiple_auto_compact_per_task_runs_after_token_limit_hit() { }, { "content": null, - "encrypted_content": null, + "encrypted_content": encrypted_content_1, "summary": [ { "text": "I will create a react app", @@ -745,7 +762,7 @@ async fn multiple_auto_compact_per_task_runs_after_token_limit_hit() { }, { "content": null, - "encrypted_content": null, + "encrypted_content": encrypted_content_2, "summary": [ { "text": "I will create a node app", @@ -855,7 +872,7 @@ async fn multiple_auto_compact_per_task_runs_after_token_limit_hit() { }, { "content": null, - "encrypted_content": null, + "encrypted_content": encrypted_content_3, "summary": [ { "text": "I will create a python app", @@ -1879,3 +1896,110 @@ async fn auto_compact_triggers_after_function_call_over_95_percent_usage() { "auto compact request should include the summarization prompt after exceeding 95% (limit {limit})" ); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn auto_compact_counts_encrypted_reasoning_before_last_user() { + skip_if_no_network!(); + + let server = start_mock_server().await; + + let first_user = "COUNT_PRE_LAST_REASONING"; + let second_user = "TRIGGER_COMPACT_AT_LIMIT"; + + let pre_last_reasoning_content = "a".repeat(2_400); + let post_last_reasoning_content = "b".repeat(4_000); + + let first_turn = sse(vec![ + ev_reasoning_item("pre-reasoning", &["pre"], &[&pre_last_reasoning_content]), + ev_completed_with_tokens("r1", 10), + ]); + let second_turn = sse(vec![ + ev_reasoning_item("post-reasoning", &["post"], &[&post_last_reasoning_content]), + ev_completed_with_tokens("r2", 80), + ]); + let resume_turn = sse(vec![ + ev_assistant_message("m4", FINAL_REPLY), + ev_completed_with_tokens("r4", 1), + ]); + + let request_log = mount_sse_sequence( + &server, + vec![ + // Turn 1: reasoning before last user (should count). + first_turn, + // Turn 2: reasoning after last user (should be ignored for compaction). + second_turn, + // Turn 3: resume after remote compaction. + resume_turn, + ], + ) + .await; + + let compacted_history = vec![codex_protocol::models::ResponseItem::Message { + id: None, + role: "assistant".to_string(), + content: vec![codex_protocol::models::ContentItem::OutputText { + text: "REMOTE_COMPACT_SUMMARY".to_string(), + }], + }]; + let compact_mock = + mount_compact_json_once(&server, serde_json::json!({ "output": compacted_history })).await; + + let codex = test_codex() + .with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing()) + .with_config(|config| { + set_test_compact_prompt(config); + config.model_auto_compact_token_limit = Some(300); + config.features.enable(Feature::RemoteCompaction); + }) + .build(&server) + .await + .expect("build codex") + .codex; + + for (idx, user) in [first_user, second_user].into_iter().enumerate() { + codex + .submit(Op::UserInput { + items: vec![UserInput::Text { text: user.into() }], + }) + .await + .unwrap(); + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TaskComplete(_))).await; + + if idx == 0 { + assert!( + compact_mock.requests().is_empty(), + "remote compaction should not run after the first turn" + ); + } + } + + let compact_requests = compact_mock.requests(); + assert_eq!( + compact_requests.len(), + 1, + "remote compaction should run once after the second turn" + ); + assert_eq!( + compact_requests[0].path(), + "/v1/responses/compact", + "remote compaction should hit the compact endpoint" + ); + + let requests = request_log.requests(); + assert_eq!( + requests.len(), + 3, + "conversation should include two user turns and a post-compaction resume" + ); + let second_request_body = requests[1].body_json().to_string(); + assert!( + !second_request_body.contains("REMOTE_COMPACT_SUMMARY"), + "second turn should not include compacted history" + ); + let resume_body = requests[2].body_json().to_string(); + assert!( + resume_body.contains("REMOTE_COMPACT_SUMMARY") || resume_body.contains(FINAL_REPLY), + "resume request should follow remote compact and use compacted history" + ); +} From 99bcb9035372d29ba2de73860a6e459357e4361c Mon Sep 17 00:00:00 2001 From: jif-oai Date: Mon, 24 Nov 2025 17:51:47 +0100 Subject: [PATCH 02/61] chore: use proxy for encrypted summary (#7252) --- codex-rs/core/src/context_manager/history.rs | 34 +++++++++++++------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/codex-rs/core/src/context_manager/history.rs b/codex-rs/core/src/context_manager/history.rs index 035d0d2867..7277f0b0c6 100644 --- a/codex-rs/core/src/context_manager/history.rs +++ b/codex-rs/core/src/context_manager/history.rs @@ -84,9 +84,19 @@ impl ContextManager { .unwrap_or(i64::MAX); let items_tokens = self.items.iter().fold(0i64, |acc, item| { - let serialized = serde_json::to_string(item).unwrap_or_default(); - let item_tokens = i64::try_from(approx_token_count(&serialized)).unwrap_or(i64::MAX); - acc.saturating_add(item_tokens) + acc + match item { + ResponseItem::Reasoning { + encrypted_content: Some(content), + .. + } + | ResponseItem::CompactionSummary { + encrypted_content: content, + } => estimate_reasoning_length(content.len()) as i64, + item => { + let serialized = serde_json::to_string(item).unwrap_or_default(); + i64::try_from(approx_token_count(&serialized)).unwrap_or(i64::MAX) + } + } }); Some(base_tokens.saturating_add(items_tokens)) @@ -145,21 +155,13 @@ impl ContextManager { None } }) - .map(Self::estimate_reasoning_length) + .map(estimate_reasoning_length) .fold(0usize, usize::saturating_add); let token_estimate = approx_tokens_from_byte_count(total_reasoning_bytes); token_estimate as usize } - fn estimate_reasoning_length(encoded_len: usize) -> usize { - encoded_len - .saturating_mul(3) - .checked_div(4) - .unwrap_or(0) - .saturating_sub(650) - } - pub(crate) fn get_total_token_usage(&self) -> i64 { self.token_info .as_ref() @@ -247,6 +249,14 @@ fn is_api_message(message: &ResponseItem) -> bool { } } +fn estimate_reasoning_length(encoded_len: usize) -> usize { + encoded_len + .saturating_mul(3) + .checked_div(4) + .unwrap_or(0) + .saturating_sub(650) +} + #[cfg(test)] #[path = "history_tests.rs"] mod tests; From 920239f272dbddd1197a0d8f9ff7e93bd2769bb2 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Mon, 24 Nov 2025 17:59:09 +0100 Subject: [PATCH 03/61] fix: codex delegate cancellation (#7092) --- codex-rs/core/src/codex.rs | 5 +- codex-rs/core/src/codex_delegate.rs | 210 ++++++++++++++++++++++------ 2 files changed, 170 insertions(+), 45 deletions(-) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 96402961bf..82e14178eb 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -2436,6 +2436,9 @@ use crate::features::Features; #[cfg(test)] pub(crate) use tests::make_session_and_context; +#[cfg(test)] +pub(crate) use tests::make_session_and_context_with_rx; + #[cfg(test)] mod tests { use super::*; @@ -2712,7 +2715,7 @@ mod tests { // Like make_session_and_context, but returns Arc and the event receiver // so tests can assert on emitted events. - fn make_session_and_context_with_rx() -> ( + pub(crate) fn make_session_and_context_with_rx() -> ( Arc, Arc, async_channel::Receiver, diff --git a/codex-rs/core/src/codex_delegate.rs b/codex-rs/core/src/codex_delegate.rs index 4cb4d4a06a..796331d1e8 100644 --- a/codex-rs/core/src/codex_delegate.rs +++ b/codex-rs/core/src/codex_delegate.rs @@ -13,6 +13,8 @@ use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::SubAgentSource; use codex_protocol::protocol::Submission; use codex_protocol::user_input::UserInput; +use std::time::Duration; +use tokio::time::timeout; use tokio_util::sync::CancellationToken; use crate::AuthManager; @@ -60,14 +62,13 @@ pub(crate) async fn run_codex_conversation_interactive( let parent_ctx_clone = Arc::clone(&parent_ctx); let codex_for_events = Arc::clone(&codex); tokio::spawn(async move { - let _ = forward_events( + forward_events( codex_for_events, tx_sub, parent_session_clone, parent_ctx_clone, - cancel_token_events.clone(), + cancel_token_events, ) - .or_cancel(&cancel_token_events) .await; }); @@ -156,53 +157,92 @@ async fn forward_events( parent_ctx: Arc, cancel_token: CancellationToken, ) { - while let Ok(event) = codex.next_event().await { - match event { - // ignore all legacy delta events - Event { - id: _, - msg: EventMsg::AgentMessageDelta(_) | EventMsg::AgentReasoningDelta(_), - } => continue, - Event { - id: _, - msg: EventMsg::SessionConfigured(_), - } => continue, - Event { - id, - msg: EventMsg::ExecApprovalRequest(event), - } => { - // Initiate approval via parent session; do not surface to consumer. - handle_exec_approval( - &codex, - id, - &parent_session, - &parent_ctx, - event, - &cancel_token, - ) - .await; + let cancelled = cancel_token.cancelled(); + tokio::pin!(cancelled); + + loop { + tokio::select! { + _ = &mut cancelled => { + shutdown_delegate(&codex).await; + break; } - Event { - id, - msg: EventMsg::ApplyPatchApprovalRequest(event), - } => { - handle_patch_approval( - &codex, - id, - &parent_session, - &parent_ctx, - event, - &cancel_token, - ) - .await; - } - other => { - let _ = tx_sub.send(other).await; + event = codex.next_event() => { + let event = match event { + Ok(event) => event, + Err(_) => break, + }; + match event { + // ignore all legacy delta events + Event { + id: _, + msg: EventMsg::AgentMessageDelta(_) | EventMsg::AgentReasoningDelta(_), + } => {} + Event { + id: _, + msg: EventMsg::SessionConfigured(_), + } => {} + Event { + id, + msg: EventMsg::ExecApprovalRequest(event), + } => { + // Initiate approval via parent session; do not surface to consumer. + handle_exec_approval( + &codex, + id, + &parent_session, + &parent_ctx, + event, + &cancel_token, + ) + .await; + } + Event { + id, + msg: EventMsg::ApplyPatchApprovalRequest(event), + } => { + handle_patch_approval( + &codex, + id, + &parent_session, + &parent_ctx, + event, + &cancel_token, + ) + .await; + } + other => { + match tx_sub.send(other).or_cancel(&cancel_token).await { + Ok(Ok(())) => {} + _ => { + shutdown_delegate(&codex).await; + break; + } + } + } + } } } } } +/// Ask the delegate to stop and drain its events so background sends do not hit a closed channel. +async fn shutdown_delegate(codex: &Codex) { + let _ = codex.submit(Op::Interrupt).await; + let _ = codex.submit(Op::Shutdown {}).await; + + let _ = timeout(Duration::from_millis(500), async { + while let Ok(event) = codex.next_event().await { + if matches!( + event.msg, + EventMsg::TurnAborted(_) | EventMsg::TaskComplete(_) + ) { + break; + } + } + }) + .await; +} + /// Forward ops from a caller to a sub-agent, respecting cancellation. async fn forward_ops( codex: Arc, @@ -298,3 +338,85 @@ where } } } + +#[cfg(test)] +mod tests { + use super::*; + use async_channel::bounded; + use codex_protocol::models::ResponseItem; + use codex_protocol::protocol::RawResponseItemEvent; + use codex_protocol::protocol::TurnAbortReason; + use codex_protocol::protocol::TurnAbortedEvent; + use pretty_assertions::assert_eq; + + #[tokio::test] + async fn forward_events_cancelled_while_send_blocked_shuts_down_delegate() { + let (tx_events, rx_events) = bounded(1); + let (tx_sub, rx_sub) = bounded(SUBMISSION_CHANNEL_CAPACITY); + let codex = Arc::new(Codex { + next_id: AtomicU64::new(0), + tx_sub, + rx_event: rx_events, + }); + + let (session, ctx, _rx_evt) = crate::codex::make_session_and_context_with_rx(); + + let (tx_out, rx_out) = bounded(1); + tx_out + .send(Event { + id: "full".to_string(), + msg: EventMsg::TurnAborted(TurnAbortedEvent { + reason: TurnAbortReason::Interrupted, + }), + }) + .await + .unwrap(); + + let cancel = CancellationToken::new(); + let forward = tokio::spawn(forward_events( + Arc::clone(&codex), + tx_out.clone(), + session, + ctx, + cancel.clone(), + )); + + tx_events + .send(Event { + id: "evt".to_string(), + msg: EventMsg::RawResponseItem(RawResponseItemEvent { + item: ResponseItem::CustomToolCall { + id: None, + status: None, + call_id: "call-1".to_string(), + name: "tool".to_string(), + input: "{}".to_string(), + }, + }), + }) + .await + .unwrap(); + + drop(tx_events); + cancel.cancel(); + timeout(std::time::Duration::from_millis(1000), forward) + .await + .expect("forward_events hung") + .expect("forward_events join error"); + + let received = rx_out.recv().await.expect("prefilled event missing"); + assert_eq!("full", received.id); + let mut ops = Vec::new(); + while let Ok(sub) = rx_sub.try_recv() { + ops.push(sub.op); + } + assert!( + ops.iter().any(|op| matches!(op, Op::Interrupt)), + "expected Interrupt op after cancellation" + ); + assert!( + ops.iter().any(|op| matches!(op, Op::Shutdown)), + "expected Shutdown op after cancellation" + ); + } +} From b2cddec3d77887135eb75edada70ae85b804d071 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Mon, 24 Nov 2025 18:22:32 +0100 Subject: [PATCH 04/61] feat: unified exec basic pruning strategy (#7239) LRU + exited sessions first --- codex-rs/core/src/unified_exec/mod.rs | 2 + .../core/src/unified_exec/session_manager.rs | 130 ++++++++++++++- codex-rs/core/tests/suite/unified_exec.rs | 157 ++++++++++++++++++ 3 files changed, 286 insertions(+), 3 deletions(-) diff --git a/codex-rs/core/src/unified_exec/mod.rs b/codex-rs/core/src/unified_exec/mod.rs index 390401d789..1c9194baa8 100644 --- a/codex-rs/core/src/unified_exec/mod.rs +++ b/codex-rs/core/src/unified_exec/mod.rs @@ -46,6 +46,7 @@ pub(crate) const MAX_YIELD_TIME_MS: u64 = 30_000; pub(crate) const DEFAULT_MAX_OUTPUT_TOKENS: usize = 10_000; pub(crate) const UNIFIED_EXEC_OUTPUT_MAX_BYTES: usize = 1024 * 1024; // 1 MiB pub(crate) const UNIFIED_EXEC_OUTPUT_MAX_TOKENS: usize = UNIFIED_EXEC_OUTPUT_MAX_BYTES / 4; +pub(crate) const MAX_UNIFIED_EXEC_SESSIONS: usize = 64; pub(crate) struct UnifiedExecContext { pub session: Arc, @@ -108,6 +109,7 @@ struct SessionEntry { command: Vec, cwd: PathBuf, started_at: tokio::time::Instant, + last_used: tokio::time::Instant, } pub(crate) fn clamp_yield_time(yield_time_ms: u64) -> u64 { diff --git a/codex-rs/core/src/unified_exec/session_manager.rs b/codex-rs/core/src/unified_exec/session_manager.rs index d9f99b9eaf..a5cc174776 100644 --- a/codex-rs/core/src/unified_exec/session_manager.rs +++ b/codex-rs/core/src/unified_exec/session_manager.rs @@ -1,3 +1,6 @@ +use std::cmp::Reverse; +use std::collections::HashMap; +use std::collections::HashSet; use std::path::PathBuf; use std::sync::Arc; @@ -31,6 +34,7 @@ use crate::truncate::approx_token_count; use crate::truncate::formatted_truncate_text; use super::ExecCommandRequest; +use super::MAX_UNIFIED_EXEC_SESSIONS; use super::SessionEntry; use super::UnifiedExecContext; use super::UnifiedExecError; @@ -294,10 +298,11 @@ impl UnifiedExecSessionManager { &self, session_id: i32, ) -> Result { - let sessions = self.sessions.lock().await; + let mut sessions = self.sessions.lock().await; let entry = sessions - .get(&session_id) + .get_mut(&session_id) .ok_or(UnifiedExecError::UnknownSessionId { session_id })?; + entry.last_used = Instant::now(); let OutputHandles { output_buffer, output_notify, @@ -345,8 +350,11 @@ impl UnifiedExecSessionManager { command: command.to_vec(), cwd, started_at, + last_used: started_at, }; - self.sessions.lock().await.insert(session_id, entry); + let mut sessions = self.sessions.lock().await; + Self::prune_sessions_if_needed(&mut sessions); + sessions.insert(session_id, entry); session_id } @@ -548,6 +556,50 @@ impl UnifiedExecSessionManager { collected } + + fn prune_sessions_if_needed(sessions: &mut HashMap) { + if sessions.len() < MAX_UNIFIED_EXEC_SESSIONS { + return; + } + + let meta: Vec<(i32, Instant, bool)> = sessions + .iter() + .map(|(id, entry)| (*id, entry.last_used, entry.session.has_exited())) + .collect(); + + if let Some(session_id) = Self::session_id_to_prune_from_meta(&meta) { + sessions.remove(&session_id); + } + } + + // Centralized pruning policy so we can easily swap strategies later. + fn session_id_to_prune_from_meta(meta: &[(i32, Instant, bool)]) -> Option { + if meta.is_empty() { + return None; + } + + let mut by_recency = meta.to_vec(); + by_recency.sort_by_key(|(_, last_used, _)| Reverse(*last_used)); + let protected: HashSet = by_recency + .iter() + .take(8) + .map(|(session_id, _, _)| *session_id) + .collect(); + + let mut lru = meta.to_vec(); + lru.sort_by_key(|(_, last_used, _)| *last_used); + + if let Some((session_id, _, _)) = lru + .iter() + .find(|(session_id, _, exited)| !protected.contains(session_id) && *exited) + { + return Some(*session_id); + } + + lru.into_iter() + .find(|(session_id, _, _)| !protected.contains(session_id)) + .map(|(session_id, _, _)| session_id) + } } enum SessionStatus { @@ -561,3 +613,75 @@ enum SessionStatus { }, Unknown, } + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + use tokio::time::Duration; + use tokio::time::Instant; + + #[test] + fn pruning_prefers_exited_sessions_outside_recently_used() { + let now = Instant::now(); + let meta = vec![ + (1, now - Duration::from_secs(40), false), + (2, now - Duration::from_secs(30), true), + (3, now - Duration::from_secs(20), false), + (4, now - Duration::from_secs(19), false), + (5, now - Duration::from_secs(18), false), + (6, now - Duration::from_secs(17), false), + (7, now - Duration::from_secs(16), false), + (8, now - Duration::from_secs(15), false), + (9, now - Duration::from_secs(14), false), + (10, now - Duration::from_secs(13), false), + ]; + + let candidate = UnifiedExecSessionManager::session_id_to_prune_from_meta(&meta); + + assert_eq!(candidate, Some(2)); + } + + #[test] + fn pruning_falls_back_to_lru_when_no_exited() { + let now = Instant::now(); + let meta = vec![ + (1, now - Duration::from_secs(40), false), + (2, now - Duration::from_secs(30), false), + (3, now - Duration::from_secs(20), false), + (4, now - Duration::from_secs(19), false), + (5, now - Duration::from_secs(18), false), + (6, now - Duration::from_secs(17), false), + (7, now - Duration::from_secs(16), false), + (8, now - Duration::from_secs(15), false), + (9, now - Duration::from_secs(14), false), + (10, now - Duration::from_secs(13), false), + ]; + + let candidate = UnifiedExecSessionManager::session_id_to_prune_from_meta(&meta); + + assert_eq!(candidate, Some(1)); + } + + #[test] + fn pruning_protects_recent_sessions_even_if_exited() { + let now = Instant::now(); + let meta = vec![ + (1, now - Duration::from_secs(40), false), + (2, now - Duration::from_secs(30), false), + (3, now - Duration::from_secs(20), true), + (4, now - Duration::from_secs(19), false), + (5, now - Duration::from_secs(18), false), + (6, now - Duration::from_secs(17), false), + (7, now - Duration::from_secs(16), false), + (8, now - Duration::from_secs(15), false), + (9, now - Duration::from_secs(14), false), + (10, now - Duration::from_secs(13), true), + ]; + + let candidate = UnifiedExecSessionManager::session_id_to_prune_from_meta(&meta); + + // (10) is exited but among the last 8; we should drop the LRU outside that set. + assert_eq!(candidate, Some(1)); + } +} diff --git a/codex-rs/core/tests/suite/unified_exec.rs b/codex-rs/core/tests/suite/unified_exec.rs index aed4cecef7..3019e6e1e1 100644 --- a/codex-rs/core/tests/suite/unified_exec.rs +++ b/codex-rs/core/tests/suite/unified_exec.rs @@ -1760,3 +1760,160 @@ async fn unified_exec_runs_under_sandbox() -> Result<()> { Ok(()) } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn unified_exec_prunes_exited_sessions_first() -> Result<()> { + skip_if_no_network!(Ok(())); + skip_if_sandbox!(Ok(())); + + let server = start_mock_server().await; + + let mut builder = test_codex().with_config(|config| { + config.use_experimental_unified_exec_tool = true; + config.features.enable(Feature::UnifiedExec); + }); + let TestCodex { + codex, + cwd, + session_configured, + .. + } = builder.build(&server).await?; + + const MAX_SESSIONS_FOR_TEST: i32 = 64; + const FILLER_SESSIONS: i32 = MAX_SESSIONS_FOR_TEST - 1; + + let keep_call_id = "uexec-prune-keep"; + let keep_args = serde_json::json!({ + "cmd": "/bin/cat", + "yield_time_ms": 250, + }); + + let prune_call_id = "uexec-prune-target"; + let prune_args = serde_json::json!({ + "cmd": "sleep 1", + "yield_time_ms": 250, + }); + + let mut events = vec![ev_response_created("resp-prune-1")]; + events.push(ev_function_call( + keep_call_id, + "exec_command", + &serde_json::to_string(&keep_args)?, + )); + events.push(ev_function_call( + prune_call_id, + "exec_command", + &serde_json::to_string(&prune_args)?, + )); + + for idx in 0..FILLER_SESSIONS { + let filler_args = serde_json::json!({ + "cmd": format!("echo filler {idx}"), + "yield_time_ms": 250, + }); + let call_id = format!("uexec-prune-fill-{idx}"); + events.push(ev_function_call( + &call_id, + "exec_command", + &serde_json::to_string(&filler_args)?, + )); + } + + let keep_write_call_id = "uexec-prune-keep-write"; + let keep_write_args = serde_json::json!({ + "chars": "still alive\n", + "session_id": 0, + "yield_time_ms": 500, + }); + events.push(ev_function_call( + keep_write_call_id, + "write_stdin", + &serde_json::to_string(&keep_write_args)?, + )); + + let probe_call_id = "uexec-prune-probe"; + let probe_args = serde_json::json!({ + "chars": "should fail\n", + "session_id": 1, + "yield_time_ms": 500, + }); + events.push(ev_function_call( + probe_call_id, + "write_stdin", + &serde_json::to_string(&probe_args)?, + )); + + events.push(ev_completed("resp-prune-1")); + let first_response = sse(events); + let completion_response = sse(vec![ + ev_response_created("resp-prune-2"), + ev_assistant_message("msg-prune", "done"), + ev_completed("resp-prune-2"), + ]); + let response_mock = + mount_sse_sequence(&server, vec![first_response, completion_response]).await; + + let session_model = session_configured.model.clone(); + + codex + .submit(Op::UserTurn { + items: vec![UserInput::Text { + text: "fill session cache".into(), + }], + final_output_json_schema: None, + cwd: cwd.path().to_path_buf(), + approval_policy: AskForApproval::Never, + sandbox_policy: SandboxPolicy::DangerFullAccess, + model: session_model, + effort: None, + summary: ReasoningSummary::Auto, + }) + .await?; + + wait_for_event(&codex, |event| matches!(event, EventMsg::TaskComplete(_))).await; + + let requests = response_mock.requests(); + assert!( + !requests.is_empty(), + "expected at least one response request" + ); + + let keep_start = requests + .iter() + .find_map(|req| req.function_call_output_text(keep_call_id)) + .expect("missing initial keep session output"); + let keep_start_output = parse_unified_exec_output(&keep_start)?; + pretty_assertions::assert_eq!(keep_start_output.session_id, Some(0)); + assert!(keep_start_output.exit_code.is_none()); + + let prune_start = requests + .iter() + .find_map(|req| req.function_call_output_text(prune_call_id)) + .expect("missing initial prune session output"); + let prune_start_output = parse_unified_exec_output(&prune_start)?; + pretty_assertions::assert_eq!(prune_start_output.session_id, Some(1)); + assert!(prune_start_output.exit_code.is_none()); + + let keep_write = requests + .iter() + .find_map(|req| req.function_call_output_text(keep_write_call_id)) + .expect("missing keep write output"); + let keep_write_output = parse_unified_exec_output(&keep_write)?; + pretty_assertions::assert_eq!(keep_write_output.session_id, Some(0)); + assert!( + keep_write_output.output.contains("still alive"), + "expected cat session to echo input, got {:?}", + keep_write_output.output + ); + + let pruned_probe = requests + .iter() + .find_map(|req| req.function_call_output_text(probe_call_id)) + .expect("missing probe output"); + assert!( + pruned_probe.contains("UnknownSessionId") || pruned_probe.contains("Unknown session id"), + "expected probe to fail after pruning, got {pruned_probe:?}" + ); + + Ok(()) +} From 486b1c4d9d60e7d7690b6db81df9fbc100a2a0e1 Mon Sep 17 00:00:00 2001 From: iceweasel-oai Date: Mon, 24 Nov 2025 09:51:58 -0800 Subject: [PATCH 05/61] consolidate world-writable-directories scanning. (#7234) clean up the code for scanning for world writable directories One path (selecting a sandbox mode from /approvals) was using an incorrect method that did not use the new method of creating deny aces to prevent writing to those directories. Now all paths are the same. --- codex-rs/tui/src/app.rs | 37 ++------------- codex-rs/tui/src/chatwidget.rs | 17 +++++-- codex-rs/windows-sandbox-rs/src/audit.rs | 52 ++++++++++----------- codex-rs/windows-sandbox-rs/src/lib.rs | 59 ++++-------------------- 4 files changed, 53 insertions(+), 112 deletions(-) diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index 9c1bb5a301..42dadd96c9 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -941,47 +941,20 @@ impl App { sandbox_policy: codex_core::protocol::SandboxPolicy, tx: AppEventSender, ) { - #[inline] - fn normalize_windows_path_for_display(p: &std::path::Path) -> String { - let canon = dunce::canonicalize(p).unwrap_or_else(|_| p.to_path_buf()); - canon.display().to_string().replace('/', "\\") - } tokio::task::spawn_blocking(move || { - let result = codex_windows_sandbox::preflight_audit_everyone_writable( + let result = codex_windows_sandbox::apply_world_writable_scan_and_denies( &logs_base_dir, &cwd, &env_map, &sandbox_policy, Some(logs_base_dir.as_path()), ); - if let Ok(ref paths) = result - && !paths.is_empty() - { - let as_strings: Vec = paths - .iter() - .map(|p| normalize_windows_path_for_display(p)) - .collect(); - let sample_paths: Vec = as_strings.iter().take(3).cloned().collect(); - let extra_count = if as_strings.len() > sample_paths.len() { - as_strings.len() - sample_paths.len() - } else { - 0 - }; - + if result.is_err() { + // Scan failed: warn without examples. tx.send(AppEvent::OpenWorldWritableWarningConfirmation { preset: None, - sample_paths, - extra_count, - failed_scan: false, - }); - } else if result.is_err() { - // Scan failed: still warn, but with no examples and mark as failed. - let sample_paths: Vec = Vec::new(); - let extra_count = 0usize; - tx.send(AppEvent::OpenWorldWritableWarningConfirmation { - preset: None, - sample_paths, - extra_count, + sample_paths: Vec::new(), + extra_count: 0usize, failed_scan: true, }); } diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 1e9fbd39fb..ddba41443d 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -2380,11 +2380,18 @@ impl ChatWidget { { return None; } - let cwd = match std::env::current_dir() { - Ok(cwd) => cwd, - Err(_) => return Some((Vec::new(), 0, true)), - }; - codex_windows_sandbox::world_writable_warning_details(self.config.codex_home.as_path(), cwd) + let cwd = self.config.cwd.clone(); + let env_map: std::collections::HashMap = std::env::vars().collect(); + match codex_windows_sandbox::apply_world_writable_scan_and_denies( + self.config.codex_home.as_path(), + cwd.as_path(), + &env_map, + &self.config.sandbox_policy, + Some(self.config.codex_home.as_path()), + ) { + Ok(_) => None, + Err(_) => Some((Vec::new(), 0, true)), + } } #[cfg(not(target_os = "windows"))] diff --git a/codex-rs/windows-sandbox-rs/src/audit.rs b/codex-rs/windows-sandbox-rs/src/audit.rs index 210fd58ee3..6234dbf26a 100644 --- a/codex-rs/windows-sandbox-rs/src/audit.rs +++ b/codex-rs/windows-sandbox-rs/src/audit.rs @@ -8,7 +8,6 @@ use crate::token::world_sid; use anyhow::anyhow; use crate::winutil::to_wide; use anyhow::Result; -use std::collections::HashMap; use std::collections::HashSet; use std::ffi::c_void; use std::path::Path; @@ -296,6 +295,32 @@ pub fn audit_everyone_writable( Ok(Vec::new()) } +pub fn apply_world_writable_scan_and_denies( + codex_home: &Path, + cwd: &Path, + env_map: &std::collections::HashMap, + sandbox_policy: &SandboxPolicy, + logs_base_dir: Option<&Path>, +) -> Result<()> { + let flagged = audit_everyone_writable(cwd, env_map, logs_base_dir)?; + if flagged.is_empty() { + return Ok(()); + } + if let Err(err) = apply_capability_denies_for_world_writable( + codex_home, + &flagged, + sandbox_policy, + cwd, + logs_base_dir, + ) { + log_note( + &format!("AUDIT: failed to apply capability deny ACEs: {}", err), + logs_base_dir, + ); + } + Ok(()) +} + pub fn apply_capability_denies_for_world_writable( codex_home: &Path, flagged: &[PathBuf], @@ -359,31 +384,6 @@ pub fn apply_capability_denies_for_world_writable( } Ok(()) } - -fn normalize_windows_path_for_display(p: impl AsRef) -> String { - let canon = dunce::canonicalize(p.as_ref()).unwrap_or_else(|_| p.as_ref().to_path_buf()); - canon.display().to_string().replace('/', "\\") -} - -pub fn world_writable_warning_details( - codex_home: impl AsRef, - cwd: impl AsRef, -) -> Option<(Vec, usize, bool)> { - let env_map: HashMap = std::env::vars().collect(); - match audit_everyone_writable(cwd.as_ref(), &env_map, Some(codex_home.as_ref())) { - Ok(paths) if paths.is_empty() => None, - Ok(paths) => { - let as_strings: Vec = paths - .iter() - .map(normalize_windows_path_for_display) - .collect(); - let sample_paths: Vec = as_strings.iter().take(3).cloned().collect(); - let extra_count = as_strings.len().saturating_sub(sample_paths.len()); - Some((sample_paths, extra_count, false)) - } - Err(_) => Some((Vec::new(), 0, true)), - } -} // Fast mask-based check: does the DACL contain any ACCESS_ALLOWED ACE for // Everyone that grants write after generic bits are expanded? Skips inherit-only // ACEs (do not apply to the current object). diff --git a/codex-rs/windows-sandbox-rs/src/lib.rs b/codex-rs/windows-sandbox-rs/src/lib.rs index e2277daade..2d591fced0 100644 --- a/codex-rs/windows-sandbox-rs/src/lib.rs +++ b/codex-rs/windows-sandbox-rs/src/lib.rs @@ -7,21 +7,17 @@ macro_rules! windows_modules { windows_modules!(acl, allow, audit, cap, env, logging, policy, token, winutil); #[cfg(target_os = "windows")] -pub use audit::world_writable_warning_details; -#[cfg(target_os = "windows")] -pub use windows_impl::preflight_audit_everyone_writable; +pub use audit::apply_world_writable_scan_and_denies; #[cfg(target_os = "windows")] pub use windows_impl::run_windows_sandbox_capture; #[cfg(target_os = "windows")] pub use windows_impl::CaptureResult; #[cfg(not(target_os = "windows"))] -pub use stub::preflight_audit_everyone_writable; +pub use stub::apply_world_writable_scan_and_denies; #[cfg(not(target_os = "windows"))] pub use stub::run_windows_sandbox_capture; #[cfg(not(target_os = "windows"))] -pub use stub::world_writable_warning_details; -#[cfg(not(target_os = "windows"))] pub use stub::CaptureResult; #[cfg(target_os = "windows")] @@ -30,7 +26,6 @@ mod windows_impl { use super::acl::allow_null_device; use super::acl::revoke_ace; use super::allow::compute_allow_paths; - use super::audit; use super::cap::cap_sid_file; use super::cap::load_or_create_cap_sids; use super::env::apply_no_network_to_env; @@ -38,7 +33,6 @@ mod windows_impl { use super::env::normalize_null_device_env; use super::logging::debug_log; use super::logging::log_failure; - use super::logging::log_note; use super::logging::log_start; use super::logging::log_success; use super::policy::parse_policy; @@ -182,32 +176,6 @@ mod windows_impl { pub timed_out: bool, } - pub fn preflight_audit_everyone_writable( - codex_home: &Path, - cwd: &Path, - env_map: &HashMap, - sandbox_policy: &SandboxPolicy, - logs_base_dir: Option<&Path>, - ) -> Result> { - let flagged = audit::audit_everyone_writable(cwd, env_map, logs_base_dir)?; - if flagged.is_empty() { - return Ok(flagged); - } - if let Err(err) = audit::apply_capability_denies_for_world_writable( - codex_home, - &flagged, - sandbox_policy, - cwd, - logs_base_dir, - ) { - log_note( - &format!("AUDIT: failed to apply capability deny ACEs: {}", err), - logs_base_dir, - ); - } - Ok(Vec::new()) - } - pub fn run_windows_sandbox_capture( policy_json_or_preset: &str, sandbox_policy_cwd: &Path, @@ -502,16 +470,6 @@ mod stub { pub timed_out: bool, } - pub fn preflight_audit_everyone_writable( - _codex_home: &Path, - _cwd: &Path, - _env_map: &HashMap, - _sandbox_policy: &SandboxPolicy, - _logs_base_dir: Option<&Path>, - ) -> Result> { - bail!("Windows sandbox is only available on Windows") - } - pub fn run_windows_sandbox_capture( _policy_json_or_preset: &str, _sandbox_policy_cwd: &Path, @@ -524,10 +482,13 @@ mod stub { bail!("Windows sandbox is only available on Windows") } - pub fn world_writable_warning_details( - _codex_home: impl AsRef, - _cwd: impl AsRef, - ) -> Option<(Vec, usize, bool)> { - None + pub fn apply_world_writable_scan_and_denies( + _codex_home: &Path, + _cwd: &Path, + _env_map: &HashMap, + _sandbox_policy: &SandboxPolicy, + _logs_base_dir: Option<&Path>, + ) -> Result<()> { + bail!("Windows sandbox is only available on Windows") } } From 35d89e820f5358e51a4630003f19fcc58c37941e Mon Sep 17 00:00:00 2001 From: jif-oai Date: Mon, 24 Nov 2025 19:45:41 +0100 Subject: [PATCH 06/61] fix: flaky test (#7257) --- codex-rs/core/tests/suite/unified_exec.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/codex-rs/core/tests/suite/unified_exec.rs b/codex-rs/core/tests/suite/unified_exec.rs index 3019e6e1e1..216d1eea52 100644 --- a/codex-rs/core/tests/suite/unified_exec.rs +++ b/codex-rs/core/tests/suite/unified_exec.rs @@ -1762,6 +1762,7 @@ async fn unified_exec_runs_under_sandbox() -> Result<()> { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore] async fn unified_exec_prunes_exited_sessions_first() -> Result<()> { skip_if_no_network!(Ok(())); skip_if_sandbox!(Ok(())); @@ -1789,9 +1790,10 @@ async fn unified_exec_prunes_exited_sessions_first() -> Result<()> { }); let prune_call_id = "uexec-prune-target"; + // Give the sleeper time to exit before the filler sessions trigger pruning. let prune_args = serde_json::json!({ "cmd": "sleep 1", - "yield_time_ms": 250, + "yield_time_ms": 1_250, }); let mut events = vec![ev_response_created("resp-prune-1")]; From c31663d74579f085c3f38c9c82b99de0fd3fe222 Mon Sep 17 00:00:00 2001 From: Matthew Zeng Date: Mon, 24 Nov 2025 11:05:37 -0800 Subject: [PATCH 07/61] [feedback] Add source info into feedback metadata. (#7140) Verified the source info is correctly attached based on whether it's cli or vscode. --- .../app-server/src/codex_message_processor.rs | 2 ++ codex-rs/core/src/conversation_manager.rs | 4 ++++ codex-rs/feedback/src/lib.rs | 5 ++++ codex-rs/protocol/src/protocol.rs | 23 +++++++++++++++++++ codex-rs/tui/src/bottom_pane/feedback_view.rs | 2 ++ 5 files changed, 36 insertions(+) diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index 5815e57850..a9f56de115 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -2794,6 +2794,7 @@ impl CodexMessageProcessor { } else { None }; + let session_source = self.conversation_manager.session_source(); let upload_result = tokio::task::spawn_blocking(move || { let rollout_path_ref = validated_rollout_path.as_deref(); @@ -2802,6 +2803,7 @@ impl CodexMessageProcessor { reason.as_deref(), include_logs, rollout_path_ref, + Some(session_source), ) }) .await; diff --git a/codex-rs/core/src/conversation_manager.rs b/codex-rs/core/src/conversation_manager.rs index 8ffefd5679..0f4577bf1e 100644 --- a/codex-rs/core/src/conversation_manager.rs +++ b/codex-rs/core/src/conversation_manager.rs @@ -56,6 +56,10 @@ impl ConversationManager { ) } + pub fn session_source(&self) -> SessionSource { + self.session_source.clone() + } + pub async fn new_conversation(&self, config: Config) -> CodexResult { self.spawn_conversation(config, self.auth_manager.clone()) .await diff --git a/codex-rs/feedback/src/lib.rs b/codex-rs/feedback/src/lib.rs index e1ccc3aac9..eaa949717a 100644 --- a/codex-rs/feedback/src/lib.rs +++ b/codex-rs/feedback/src/lib.rs @@ -10,6 +10,7 @@ use std::time::Duration; use anyhow::Result; use anyhow::anyhow; use codex_protocol::ConversationId; +use codex_protocol::protocol::SessionSource; use tracing_subscriber::fmt::writer::MakeWriter; const DEFAULT_MAX_BYTES: usize = 4 * 1024 * 1024; // 4 MiB @@ -174,6 +175,7 @@ impl CodexLogSnapshot { reason: Option<&str>, include_logs: bool, rollout_path: Option<&std::path::Path>, + session_source: Option, ) -> Result<()> { use std::collections::BTreeMap; use std::fs; @@ -203,6 +205,9 @@ impl CodexLogSnapshot { (String::from("classification"), classification.to_string()), (String::from("cli_version"), cli_version.to_string()), ]); + if let Some(source) = session_source.as_ref() { + tags.insert(String::from("session_source"), source.to_string()); + } if let Some(r) = reason { tags.insert(String::from("reason"), r.to_string()); } diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index 27267de439..e5dbb26b90 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -1131,6 +1131,29 @@ pub enum SubAgentSource { Other(String), } +impl fmt::Display for SessionSource { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + SessionSource::Cli => f.write_str("cli"), + SessionSource::VSCode => f.write_str("vscode"), + SessionSource::Exec => f.write_str("exec"), + SessionSource::Mcp => f.write_str("mcp"), + SessionSource::SubAgent(sub_source) => write!(f, "subagent_{sub_source}"), + SessionSource::Unknown => f.write_str("unknown"), + } + } +} + +impl fmt::Display for SubAgentSource { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + SubAgentSource::Review => f.write_str("review"), + SubAgentSource::Compact => f.write_str("compact"), + SubAgentSource::Other(other) => f.write_str(other), + } + } +} + #[derive(Serialize, Deserialize, Clone, Debug, JsonSchema, TS)] pub struct SessionMeta { pub id: ConversationId, diff --git a/codex-rs/tui/src/bottom_pane/feedback_view.rs b/codex-rs/tui/src/bottom_pane/feedback_view.rs index 8a42e563cd..c563ab8e90 100644 --- a/codex-rs/tui/src/bottom_pane/feedback_view.rs +++ b/codex-rs/tui/src/bottom_pane/feedback_view.rs @@ -19,6 +19,7 @@ use crate::app_event::FeedbackCategory; use crate::app_event_sender::AppEventSender; use crate::history_cell; use crate::render::renderable::Renderable; +use codex_core::protocol::SessionSource; use super::CancellationEvent; use super::bottom_pane_view::BottomPaneView; @@ -85,6 +86,7 @@ impl FeedbackNoteView { } else { None }, + Some(SessionSource::Cli), ); match result { From 1e832b1438631cda9b3d6fa8f0ae955d7af7813e Mon Sep 17 00:00:00 2001 From: Dylan Hurd Date: Mon, 24 Nov 2025 11:32:47 -0800 Subject: [PATCH 08/61] fix(windows) support apply_patch parsing in powershell (#7221) ## Summary Support powershell parsing of apply_patch ## Testing - [x] Enable apply_patch unit tests --------- Co-authored-by: jif-oai --- .../app-server/tests/suite/v2/turn_start.rs | 8 - codex-rs/apply-patch/src/lib.rs | 143 ++++++++++++++---- codex-rs/core/tests/suite/exec_policy.rs | 2 +- 3 files changed, 115 insertions(+), 38 deletions(-) diff --git a/codex-rs/app-server/tests/suite/v2/turn_start.rs b/codex-rs/app-server/tests/suite/v2/turn_start.rs index de4d1cd2cd..a3e75268c9 100644 --- a/codex-rs/app-server/tests/suite/v2/turn_start.rs +++ b/codex-rs/app-server/tests/suite/v2/turn_start.rs @@ -614,10 +614,6 @@ async fn turn_start_updates_sandbox_and_cwd_between_turns_v2() -> Result<()> { #[tokio::test] async fn turn_start_file_change_approval_v2() -> Result<()> { skip_if_no_network!(Ok(())); - if cfg!(windows) { - // TODO apply_patch approvals are not parsed from powershell commands yet - return Ok(()); - } let tmp = TempDir::new()?; let codex_home = tmp.path().join("codex_home"); @@ -764,10 +760,6 @@ async fn turn_start_file_change_approval_v2() -> Result<()> { #[tokio::test] async fn turn_start_file_change_approval_decline_v2() -> Result<()> { skip_if_no_network!(Ok(())); - if cfg!(windows) { - // TODO apply_patch approvals are not parsed from powershell commands yet - return Ok(()); - } let tmp = TempDir::new()?; let codex_home = tmp.path().join("codex_home"); diff --git a/codex-rs/apply-patch/src/lib.rs b/codex-rs/apply-patch/src/lib.rs index a2bea0514a..867d19a2e8 100644 --- a/codex-rs/apply-patch/src/lib.rs +++ b/codex-rs/apply-patch/src/lib.rs @@ -30,7 +30,13 @@ pub use standalone_executable::main; pub const APPLY_PATCH_TOOL_INSTRUCTIONS: &str = include_str!("../apply_patch_tool_instructions.md"); const APPLY_PATCH_COMMANDS: [&str; 2] = ["apply_patch", "applypatch"]; -const APPLY_PATCH_SHELLS: [&str; 3] = ["bash", "zsh", "sh"]; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ApplyPatchShell { + Unix, + PowerShell, + Cmd, +} #[derive(Debug, Error, PartialEq)] pub enum ApplyPatchError { @@ -97,11 +103,55 @@ pub struct ApplyPatchArgs { pub workdir: Option, } -fn shell_supports_apply_patch(shell: &str) -> bool { +fn classify_shell_name(shell: &str) -> Option { std::path::Path::new(shell) - .file_name() + .file_stem() .and_then(|name| name.to_str()) - .is_some_and(|name| APPLY_PATCH_SHELLS.contains(&name)) + .map(str::to_ascii_lowercase) +} + +fn classify_shell(shell: &str, flag: &str) -> Option { + classify_shell_name(shell).and_then(|name| match name.as_str() { + "bash" | "zsh" | "sh" if flag == "-lc" => Some(ApplyPatchShell::Unix), + "pwsh" | "powershell" if flag.eq_ignore_ascii_case("-command") => { + Some(ApplyPatchShell::PowerShell) + } + "cmd" if flag.eq_ignore_ascii_case("/c") => Some(ApplyPatchShell::Cmd), + _ => None, + }) +} + +fn can_skip_flag(shell: &str, flag: &str) -> bool { + classify_shell_name(shell).is_some_and(|name| { + matches!(name.as_str(), "pwsh" | "powershell") && flag.eq_ignore_ascii_case("-noprofile") + }) +} + +fn parse_shell_script(argv: &[String]) -> Option<(ApplyPatchShell, &str)> { + match argv { + [shell, flag, script] => classify_shell(shell, flag).map(|shell_type| { + let script = script.as_str(); + (shell_type, script) + }), + [shell, skip_flag, flag, script] if can_skip_flag(shell, skip_flag) => { + classify_shell(shell, flag).map(|shell_type| { + let script = script.as_str(); + (shell_type, script) + }) + } + _ => None, + } +} + +fn extract_apply_patch_from_shell( + shell: ApplyPatchShell, + script: &str, +) -> std::result::Result<(String, Option), ExtractHeredocError> { + match shell { + ApplyPatchShell::Unix | ApplyPatchShell::PowerShell | ApplyPatchShell::Cmd => { + extract_apply_patch_from_bash(script) + } + } } pub fn maybe_parse_apply_patch(argv: &[String]) -> MaybeApplyPatch { @@ -111,9 +161,9 @@ pub fn maybe_parse_apply_patch(argv: &[String]) -> MaybeApplyPatch { Ok(source) => MaybeApplyPatch::Body(source), Err(e) => MaybeApplyPatch::PatchParseError(e), }, - // Bash heredoc form: (optional `cd &&`) apply_patch <<'EOF' ... - [shell, flag, script] if shell_supports_apply_patch(shell) && flag == "-lc" => { - match extract_apply_patch_from_bash(script) { + // Shell heredoc form: (optional `cd &&`) apply_patch <<'EOF' ... + _ => match parse_shell_script(argv) { + Some((shell, script)) => match extract_apply_patch_from_shell(shell, script) { Ok((body, workdir)) => match parse_patch(&body) { Ok(mut source) => { source.workdir = workdir; @@ -125,9 +175,9 @@ pub fn maybe_parse_apply_patch(argv: &[String]) -> MaybeApplyPatch { MaybeApplyPatch::NotApplyPatch } Err(e) => MaybeApplyPatch::ShellParseError(e), - } - } - _ => MaybeApplyPatch::NotApplyPatch, + }, + None => MaybeApplyPatch::NotApplyPatch, + }, } } @@ -222,24 +272,17 @@ impl ApplyPatchAction { /// cwd must be an absolute path so that we can resolve relative paths in the /// patch. pub fn maybe_parse_apply_patch_verified(argv: &[String], cwd: &Path) -> MaybeApplyPatchVerified { - // Detect a raw patch body passed directly as the command or as the body of a bash -lc + // Detect a raw patch body passed directly as the command or as the body of a shell // script. In these cases, report an explicit error rather than applying the patch. - match argv { - [body] => { - if parse_patch(body).is_ok() { - return MaybeApplyPatchVerified::CorrectnessError( - ApplyPatchError::ImplicitInvocation, - ); - } - } - [shell, flag, script] - if shell_supports_apply_patch(shell) - && flag == "-lc" - && parse_patch(script).is_ok() => - { - return MaybeApplyPatchVerified::CorrectnessError(ApplyPatchError::ImplicitInvocation); - } - _ => {} + if let [body] = argv + && parse_patch(body).is_ok() + { + return MaybeApplyPatchVerified::CorrectnessError(ApplyPatchError::ImplicitInvocation); + } + if let Some((_, script)) = parse_shell_script(argv) + && parse_patch(script).is_ok() + { + return MaybeApplyPatchVerified::CorrectnessError(ApplyPatchError::ImplicitInvocation); } match maybe_parse_apply_patch(argv) { @@ -871,6 +914,22 @@ mod tests { strs_to_strings(&["bash", "-lc", script]) } + fn args_powershell(script: &str) -> Vec { + strs_to_strings(&["powershell.exe", "-Command", script]) + } + + fn args_powershell_no_profile(script: &str) -> Vec { + strs_to_strings(&["powershell.exe", "-NoProfile", "-Command", script]) + } + + fn args_pwsh(script: &str) -> Vec { + strs_to_strings(&["pwsh", "-NoProfile", "-Command", script]) + } + + fn args_cmd(script: &str) -> Vec { + strs_to_strings(&["cmd.exe", "/c", script]) + } + fn heredoc_script(prefix: &str) -> String { format!( "{prefix}apply_patch <<'PATCH'\n*** Begin Patch\n*** Add File: foo\n+hi\n*** End Patch\nPATCH" @@ -890,8 +949,7 @@ mod tests { }] } - fn assert_match(script: &str, expected_workdir: Option<&str>) { - let args = args_bash(script); + fn assert_match_args(args: Vec, expected_workdir: Option<&str>) { match maybe_parse_apply_patch(&args) { MaybeApplyPatch::Body(ApplyPatchArgs { hunks, workdir, .. }) => { assert_eq!(workdir.as_deref(), expected_workdir); @@ -901,6 +959,11 @@ mod tests { } } + fn assert_match(script: &str, expected_workdir: Option<&str>) { + let args = args_bash(script); + assert_match_args(args, expected_workdir); + } + fn assert_not_match(script: &str) { let args = args_bash(script); assert_matches!( @@ -1014,6 +1077,28 @@ PATCH"#, } } + #[test] + fn test_powershell_heredoc() { + let script = heredoc_script(""); + assert_match_args(args_powershell(&script), None); + } + #[test] + fn test_powershell_heredoc_no_profile() { + let script = heredoc_script(""); + assert_match_args(args_powershell_no_profile(&script), None); + } + #[test] + fn test_pwsh_heredoc() { + let script = heredoc_script(""); + assert_match_args(args_pwsh(&script), None); + } + + #[test] + fn test_cmd_heredoc_with_cd() { + let script = heredoc_script("cd foo && "); + assert_match_args(args_cmd(&script), Some("foo")); + } + #[test] fn test_heredoc_with_leading_cd() { assert_match(&heredoc_script("cd foo && "), Some("foo")); diff --git a/codex-rs/core/tests/suite/exec_policy.rs b/codex-rs/core/tests/suite/exec_policy.rs index 443e37cdb4..b65b2994ee 100644 --- a/codex-rs/core/tests/suite/exec_policy.rs +++ b/codex-rs/core/tests/suite/exec_policy.rs @@ -26,7 +26,7 @@ async fn execpolicy_blocks_shell_invocation() -> Result<()> { return Ok(()); } - let mut builder = test_codex().with_model("gpt-5.1").with_config(|config| { + let mut builder = test_codex().with_config(|config| { let policy_path = config.codex_home.join("policy").join("policy.codexpolicy"); fs::create_dir_all( policy_path From be022be381acaa764b207b9ae0ca96b720e72338 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Nov 2025 11:43:57 -0800 Subject: [PATCH 09/61] chore(deps): bump regex from 1.11.1 to 1.12.2 in /codex-rs (#7222) Bumps [regex](https://github.com/rust-lang/regex) from 1.11.1 to 1.12.2.
Changelog

Sourced from regex's changelog.

1.12.2 (2025-10-13)

This release fixes a cargo doc breakage on nightly when --cfg docsrs is enabled. This caused documentation to fail to build on docs.rs.

Bug fixes:

1.12.1 (2025-10-10)

This release makes a bug fix in the new regex::Captures::get_match API introduced in 1.12.0. There was an oversight with the lifetime parameter for the Match returned. This is technically a breaking change, but given that it was caught almost immediately and I've yanked the 1.12.0 release, I think this is fine.

1.12.0 (2025-10-10)

This release contains a smattering of bug fixes, a fix for excessive memory consumption in some cases and a new regex::Captures::get_match API.

Improvements:

Bug fixes:

1.11.3 (2025-09-25)

This is a small patch release with an improvement in memory usage in some cases.

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=regex&package-manager=cargo&previous-version=1.11.1&new-version=1.12.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- codex-rs/Cargo.lock | 8 ++++---- codex-rs/Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 0bcfd4f37a..98ad4e0ab2 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -4994,9 +4994,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.11.1" +version = "1.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" dependencies = [ "aho-corasick", "memchr", @@ -5006,9 +5006,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.9" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" dependencies = [ "aho-corasick", "memchr", diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index e1f64cb5d9..6eac9391ab 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -165,7 +165,7 @@ rand = "0.9" ratatui = "0.29.0" ratatui-macros = "0.6.0" regex-lite = "0.1.7" -regex = "1.11.1" +regex = "1.12.2" reqwest = "0.12" rmcp = { version = "0.9.0", default-features = false } schemars = "0.8.22" From e8b6fb7937f2617b5880c4647bc49bb4567bedf4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Nov 2025 11:44:14 -0800 Subject: [PATCH 10/61] chore(deps): bump toml_edit from 0.23.4 to 0.23.5 in /codex-rs (#7223) Bumps [toml_edit](https://github.com/toml-rs/toml) from 0.23.4 to 0.23.5.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=toml_edit&package-manager=cargo&previous-version=0.23.4&new-version=0.23.5)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- codex-rs/Cargo.lock | 22 +++++++++++----------- codex-rs/Cargo.toml | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 98ad4e0ab2..7d57683de7 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -6553,18 +6553,18 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.7.0" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bade1c3e902f58d73d3f294cd7f20391c1cb2fbcb643b73566bc773971df91e3" +checksum = "f2cdb639ebbc97961c51720f858597f7f24c4fc295327923af55b74c3c724533" dependencies = [ - "serde", + "serde_core", ] [[package]] name = "toml_edit" -version = "0.23.4" +version = "0.23.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7211ff1b8f0d3adae1663b7da9ffe396eabe1ca25f0b0bee42b0da29a9ddce93" +checksum = "6485ef6d0d9b5d0ec17244ff7eb05310113c3f316f2d14200d4de56b3cb98f8d" dependencies = [ "indexmap 2.12.0", "toml_datetime", @@ -6575,18 +6575,18 @@ dependencies = [ [[package]] name = "toml_parser" -version = "1.0.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b551886f449aa90d4fe2bdaa9f4a2577ad2dde302c61ecf262d80b116db95c10" +checksum = "c0cbe268d35bdb4bb5a56a2de88d0ad0eb70af5384a99d648cd4b3d04039800e" dependencies = [ "winnow", ] [[package]] name = "toml_writer" -version = "1.0.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcc842091f2def52017664b53082ecbbeb5c7731092bad69d2c63050401dfd64" +checksum = "df8b2b54733674ad286d16267dcfc7a71ed5c776e4ac7aa3c3e2561f7c637bf2" [[package]] name = "tonic" @@ -7751,9 +7751,9 @@ checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" [[package]] name = "winnow" -version = "0.7.12" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3edebf492c8125044983378ecb5766203ad3b4c2f7a922bd7dd207f6d443e95" +checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf" dependencies = [ "memchr", ] diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index 6eac9391ab..a14c1b7332 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -195,7 +195,7 @@ tokio-stream = "0.1.17" tokio-test = "0.4" tokio-util = "0.7.16" toml = "0.9.5" -toml_edit = "0.23.4" +toml_edit = "0.23.5" tonic = "0.13.1" tracing = "0.1.41" tracing-appender = "0.2.3" From 35850caff6d8f2de1807d917cd41bc3926bfb1d0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Nov 2025 11:45:29 -0800 Subject: [PATCH 11/61] chore(deps): bump actions/upload-artifact from 4 to 5 (#7229) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 4 to 5.
Release notes

Sourced from actions/upload-artifact's releases.

v5.0.0

What's Changed

BREAKING CHANGE: this update supports Node v24.x. This is not a breaking change per-se but we're treating it as such.

New Contributors

Full Changelog: https://github.com/actions/upload-artifact/compare/v4...v5.0.0

v4.6.2

What's Changed

New Contributors

Full Changelog: https://github.com/actions/upload-artifact/compare/v4...v4.6.2

v4.6.1

What's Changed

Full Changelog: https://github.com/actions/upload-artifact/compare/v4...v4.6.1

v4.6.0

What's Changed

Full Changelog: https://github.com/actions/upload-artifact/compare/v4...v4.6.0

v4.5.0

What's Changed

New Contributors

... (truncated)

Commits
  • 330a01c Merge pull request #734 from actions/danwkennedy/prepare-5.0.0
  • 03f2824 Update github.dep.yml
  • 905a1ec Prepare v5.0.0
  • 2d9f9cd Merge pull request #725 from patrikpolyak/patch-1
  • 9687587 Merge branch 'main' into patch-1
  • 2848b2c Merge pull request #727 from danwkennedy/patch-1
  • 9b51177 Spell out the first use of GHES
  • cd231ca Update GHES guidance to include reference to Node 20 version
  • de65e23 Merge pull request #712 from actions/nebuk89-patch-1
  • 8747d8c Update README.md
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/upload-artifact&package-manager=github_actions&previous-version=4&new-version=5)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/shell-tool-mcp.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/shell-tool-mcp.yml b/.github/workflows/shell-tool-mcp.yml index d12f57476d..bc4992f277 100644 --- a/.github/workflows/shell-tool-mcp.yml +++ b/.github/workflows/shell-tool-mcp.yml @@ -113,7 +113,7 @@ jobs: cp "target/${{ matrix.target }}/release/codex-exec-mcp-server" "$dest/" cp "target/${{ matrix.target }}/release/codex-execve-wrapper" "$dest/" - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v5 with: name: shell-tool-mcp-rust-${{ matrix.target }} path: artifacts/** @@ -215,7 +215,7 @@ jobs: mkdir -p "$dest" cp bash "$dest/bash" - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v5 with: name: shell-tool-mcp-bash-${{ matrix.target }}-${{ matrix.variant }} path: artifacts/** @@ -260,7 +260,7 @@ jobs: mkdir -p "$dest" cp bash "$dest/bash" - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v5 with: name: shell-tool-mcp-bash-${{ matrix.target }}-${{ matrix.variant }} path: artifacts/** @@ -359,7 +359,7 @@ jobs: filename=$(PACK_INFO="$pack_info" node -e 'const data = JSON.parse(process.env.PACK_INFO); console.log(data[0].filename);') mv "dist/npm/${filename}" "dist/npm/codex-shell-tool-mcp-npm-${PACKAGE_VERSION}.tgz" - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v5 with: name: codex-shell-tool-mcp-npm path: dist/npm/codex-shell-tool-mcp-npm-${{ env.PACKAGE_VERSION }}.tgz From 6dd36067732e50775ca73f0cc671d1c248bd7a94 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Nov 2025 11:45:57 -0800 Subject: [PATCH 12/61] chore(deps): bump actions/checkout from 5 to 6 (#7230) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/checkout](https://github.com/actions/checkout) from 5 to 6.
Release notes

Sourced from actions/checkout's releases.

v6.0.0

What's Changed

Full Changelog: https://github.com/actions/checkout/compare/v5.0.0...v6.0.0

v6-beta

What's Changed

Updated persist-credentials to store the credentials under $RUNNER_TEMP instead of directly in the local git config.

This requires a minimum Actions Runner version of v2.329.0 to access the persisted credentials for Docker container action scenarios.

v5.0.1

What's Changed

Full Changelog: https://github.com/actions/checkout/compare/v5...v5.0.1

Changelog

Sourced from actions/checkout's changelog.

Changelog

V6.0.0

V5.0.1

V5.0.0

V4.3.1

V4.3.0

v4.2.2

v4.2.1

v4.2.0

v4.1.7

v4.1.6

v4.1.5

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/checkout&package-manager=github_actions&previous-version=5&new-version=6)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- .github/workflows/codespell.yml | 2 +- .github/workflows/issue-deduplicator.yml | 2 +- .github/workflows/issue-labeler.yml | 2 +- .github/workflows/rust-ci.yml | 10 +++++----- .github/workflows/rust-release.yml | 6 +++--- .github/workflows/sdk.yml | 2 +- .github/workflows/shell-tool-mcp-ci.yml | 2 +- .github/workflows/shell-tool-mcp.yml | 8 ++++---- 9 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 38773bb9f1..724995e9c9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: NODE_OPTIONS: --max-old-space-size=4096 steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Setup pnpm uses: pnpm/action-setup@v4 diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index c036581325..bbbb06d06f 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -18,7 +18,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Annotate locations with typos uses: codespell-project/codespell-problem-matcher@b80729f885d32f78a716c2f107b4db1025001c42 # v1 - name: Codespell diff --git a/.github/workflows/issue-deduplicator.yml b/.github/workflows/issue-deduplicator.yml index 5c5463e2c7..c78b1f3161 100644 --- a/.github/workflows/issue-deduplicator.yml +++ b/.github/workflows/issue-deduplicator.yml @@ -16,7 +16,7 @@ jobs: outputs: codex_output: ${{ steps.codex.outputs.final-message }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - name: Prepare Codex inputs env: diff --git a/.github/workflows/issue-labeler.yml b/.github/workflows/issue-labeler.yml index 39f9d47f14..424c7e7263 100644 --- a/.github/workflows/issue-labeler.yml +++ b/.github/workflows/issue-labeler.yml @@ -16,7 +16,7 @@ jobs: outputs: codex_output: ${{ steps.codex.outputs.final-message }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - id: codex uses: openai/codex-action@main diff --git a/.github/workflows/rust-ci.yml b/.github/workflows/rust-ci.yml index 0bd91ca53b..e72a69f7e3 100644 --- a/.github/workflows/rust-ci.yml +++ b/.github/workflows/rust-ci.yml @@ -17,7 +17,7 @@ jobs: codex: ${{ steps.detect.outputs.codex }} workflows: ${{ steps.detect.outputs.workflows }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: fetch-depth: 0 - name: Detect changed paths (no external action) @@ -56,7 +56,7 @@ jobs: run: working-directory: codex-rs steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: dtolnay/rust-toolchain@1.90 with: components: rustfmt @@ -74,7 +74,7 @@ jobs: run: working-directory: codex-rs steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: dtolnay/rust-toolchain@1.90 - uses: taiki-e/install-action@44c6d64aa62cd779e873306675c7a58e86d6d532 # v2 with: @@ -147,7 +147,7 @@ jobs: profile: release steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: dtolnay/rust-toolchain@1.90 with: targets: ${{ matrix.target }} @@ -359,7 +359,7 @@ jobs: profile: dev steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: dtolnay/rust-toolchain@1.90 with: targets: ${{ matrix.target }} diff --git a/.github/workflows/rust-release.yml b/.github/workflows/rust-release.yml index 9a9903579c..14f8aa0327 100644 --- a/.github/workflows/rust-release.yml +++ b/.github/workflows/rust-release.yml @@ -19,7 +19,7 @@ jobs: tag-check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - name: Validate tag matches Cargo.toml version shell: bash @@ -76,7 +76,7 @@ jobs: target: aarch64-pc-windows-msvc steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: dtolnay/rust-toolchain@1.90 with: targets: ${{ matrix.target }} @@ -397,7 +397,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 - uses: actions/download-artifact@v4 with: diff --git a/.github/workflows/sdk.yml b/.github/workflows/sdk.yml index 0f3a7a194b..d0c9f6eb95 100644 --- a/.github/workflows/sdk.yml +++ b/.github/workflows/sdk.yml @@ -11,7 +11,7 @@ jobs: timeout-minutes: 10 steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Setup pnpm uses: pnpm/action-setup@v4 diff --git a/.github/workflows/shell-tool-mcp-ci.yml b/.github/workflows/shell-tool-mcp-ci.yml index 8354b127f9..5e09459640 100644 --- a/.github/workflows/shell-tool-mcp-ci.yml +++ b/.github/workflows/shell-tool-mcp-ci.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Setup pnpm uses: pnpm/action-setup@v4 diff --git a/.github/workflows/shell-tool-mcp.yml b/.github/workflows/shell-tool-mcp.yml index bc4992f277..14a6a8bcd5 100644 --- a/.github/workflows/shell-tool-mcp.yml +++ b/.github/workflows/shell-tool-mcp.yml @@ -91,7 +91,7 @@ jobs: install_musl: true steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 - uses: dtolnay/rust-toolchain@1.90 with: @@ -196,7 +196,7 @@ jobs: fi - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Build patched Bash shell: bash @@ -241,7 +241,7 @@ jobs: variant: macos-13 steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Build patched Bash shell: bash @@ -278,7 +278,7 @@ jobs: PACKAGE_VERSION: ${{ needs.metadata.outputs.version }} steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Setup pnpm uses: pnpm/action-setup@v4 From 481c84e1187f46336abc314d9a5b4bd52199e3b1 Mon Sep 17 00:00:00 2001 From: Thomas Klausner Date: Mon, 24 Nov 2025 20:46:43 +0100 Subject: [PATCH 13/61] fix: Fix build process-hardening build on NetBSD (#7238) This fixes the build of codex on NetBSD. --- codex-rs/process-hardening/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/codex-rs/process-hardening/src/lib.rs b/codex-rs/process-hardening/src/lib.rs index 0a624fb387..772647671f 100644 --- a/codex-rs/process-hardening/src/lib.rs +++ b/codex-rs/process-hardening/src/lib.rs @@ -29,6 +29,7 @@ const PTRACE_DENY_ATTACH_FAILED_EXIT_CODE: i32 = 6; target_os = "android", target_os = "macos", target_os = "freebsd", + target_os = "netbsd", target_os = "openbsd" ))] const SET_RLIMIT_CORE_FAILED_EXIT_CODE: i32 = 7; From 207d94b0e7da2e0a766a8e90ed8f3642170753bf Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Mon, 24 Nov 2025 13:47:57 -0600 Subject: [PATCH 14/61] Removed `streamable_shell` from docs (#7235) This config option no longer exists Addresses #7207 --- docs/config.md | 2 -- docs/example-config.md | 1 - 2 files changed, 3 deletions(-) diff --git a/docs/config.md b/docs/config.md index d14f7482f2..5051f27069 100644 --- a/docs/config.md +++ b/docs/config.md @@ -33,7 +33,6 @@ Optional and experimental capabilities are toggled via the `[features]` table in ```toml [features] -streamable_shell = true # enable the streamable exec tool web_search_request = true # allow the model to request web searches # view_image_tool defaults to true; omit to keep defaults ``` @@ -43,7 +42,6 @@ Supported features: | Key | Default | Stage | Description | | ----------------------------------------- | :-----: | ------------ | ---------------------------------------------------- | | `unified_exec` | false | Experimental | Use the unified PTY-backed exec tool | -| `streamable_shell` | false | Experimental | Use the streamable exec-command/write-stdin pair | | `rmcp_client` | false | Experimental | Enable oauth support for streamable HTTP MCP servers | | `apply_patch_freeform` | false | Beta | Include the freeform `apply_patch` tool | | `view_image_tool` | true | Stable | Include the `view_image` tool | diff --git a/docs/example-config.md b/docs/example-config.md index 117d0cb2a2..97ed22a2b7 100644 --- a/docs/example-config.md +++ b/docs/example-config.md @@ -211,7 +211,6 @@ view_image = true [features] # Leave this table empty to accept defaults. Set explicit booleans to opt in/out. unified_exec = false -streamable_shell = false rmcp_client = false apply_patch_freeform = false view_image_tool = true From 1875700220dac4d0a0b910fc0fb0ad387cc27a57 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Nov 2025 12:02:48 -0800 Subject: [PATCH 15/61] chore(deps): bump libc from 0.2.175 to 0.2.177 in /codex-rs (#7224) Bumps [libc](https://github.com/rust-lang/libc) from 0.2.175 to 0.2.177.
Release notes

Sourced from libc's releases.

0.2.177

Added

  • Apple: Add TIOCGETA, TIOCSETA, TIOCSETAW, TIOCSETAF constants (#4736)
  • Apple: Add pthread_cond_timedwait_relative_np (#4719)
  • BSDs: Add _CS_PATH constant (#4738)
  • Linux-like: Add SIGEMT for mips* and sparc* architectures (#4730)
  • OpenBSD: Add elf_aux_info (#4729)
  • Redox: Add more sysconf constants (#4728)
  • Windows: Add wcsnlen (#4721)

Changed

  • WASIP2: Invert conditional to include p2 APIs (#4733)

0.2.176

Support

  • The default FreeBSD version has been raised from 11 to 12. This matches rustc since 1.78. (#2406)
  • Debug is now always implemented, rather than being gated behind the extra_traits feature. (#4624)

Added

  • AIX: Restore some non-POSIX functions guarded by the _KERNEL macro. (#4607)
  • FreeBSD 14: Add st_fileref to struct stat (#4642)
  • Haiku: Add the accept4 POSIX call (#4586)
  • Introduce a wrapper for representing padding (#4632)
  • Linux: Add EM_RISCV (#4659)
  • Linux: Add MS_NOSYMFOLLOW (#4389)
  • Linux: Add backtrace_symbols(_fd) (#4668)
  • Linux: Add missing SOL_PACKET optnames (#4669)
  • Musl s390x: Add SYS_mseal (#4549)
  • NuttX: Add __errno (#4687)
  • Redox: Add dirfd, VDISABLE, and resource consts (#4660)
  • Redox: Add more resource.h, fcntl.h constants (#4666)
  • Redox: Enable strftime and mkostemp[s] (#4629)
  • Unix, Windows: Add qsort_r (Unix), and qsort(_s) (Windows) (#4677)
  • Unix: Add dlvsym for Linux-gnu, FreeBSD, and NetBSD (#4671)
  • Unix: Add sigqueue (#4620)

Changed

  • FreeBSD 15: Mark kinfo_proc as non-exhaustive (#4553)
  • FreeBSD: Set the ELF symbol version for readdir_r (#4694)
  • Linux: Correct the config for whether or not epoll_event is packed (#4639)
  • Tests: Replace the old ctest with the much more reliable new implementation (#4655 and many related PRs)

Fixed

... (truncated)

Changelog

Sourced from libc's changelog.

0.2.177 - 2025-10-09

Added

  • Apple: Add TIOCGETA, TIOCSETA, TIOCSETAW, TIOCSETAF constants (#4736)
  • Apple: Add pthread_cond_timedwait_relative_np (#4719)
  • BSDs: Add _CS_PATH constant (#4738)
  • Linux-like: Add SIGEMT for mips* and sparc* architectures (#4730)
  • OpenBSD: Add elf_aux_info (#4729)
  • Redox: Add more sysconf constants (#4728)
  • Windows: Add wcsnlen (#4721)

Changed

  • WASIP2: Invert conditional to include p2 APIs (#4733)

0.2.176 - 2025-09-23

Support

  • The default FreeBSD version has been raised from 11 to 12. This matches rustc since 1.78. (#2406)
  • Debug is now always implemented, rather than being gated behind the extra_traits feature. (#4624)

Added

  • AIX: Restore some non-POSIX functions guarded by the _KERNEL macro. (#4607)
  • FreeBSD 14: Add st_fileref to struct stat (#4642)
  • Haiku: Add the accept4 POSIX call (#4586)
  • Introduce a wrapper for representing padding (#4632)
  • Linux: Add EM_RISCV (#4659)
  • Linux: Add MS_NOSYMFOLLOW (#4389)
  • Linux: Add backtrace_symbols(_fd) (#4668)
  • Linux: Add missing SOL_PACKET optnames (#4669)
  • Musl s390x: Add SYS_mseal (#4549)
  • NuttX: Add __errno (#4687)
  • Redox: Add dirfd, VDISABLE, and resource consts (#4660)
  • Redox: Add more resource.h, fcntl.h constants (#4666)
  • Redox: Enable strftime and mkostemp[s] (#4629)
  • Unix, Windows: Add qsort_r (Unix), and qsort(_s) (Windows) (#4677)
  • Unix: Add dlvsym for Linux-gnu, FreeBSD, and NetBSD (#4671)
  • Unix: Add sigqueue (#4620)

Changed

  • FreeBSD 15: Mark kinfo_proc as non-exhaustive (#4553)
  • FreeBSD: Set the ELF symbol version for readdir_r (#4694)
  • Linux: Correct the config for whether or not epoll_event is packed (#4639)
  • Tests: Replace the old ctest with the much more reliable new implementation (#4655 and many related PRs)

Fixed

... (truncated)

Commits
  • 9f598d2 chore: release libc 0.2.177
  • 329a5e7 Add missing TIOCGETA/TIOCSETA constants for macOS
  • 72a40e2 add pthread_cond_timedwait_relative_np
  • 2914d6f linux_like: add SIGEMT for mips* and sparc*
  • ff2ff25 openbsd add elf_aux_info
  • 4ae44a4 Update semver tests
  • d5737a0 Define _CS_PATH on the BSDs
  • fe277da redox: more sysconf constants
  • bdad426 wasip2: Invert conditional to include p2 APIs
  • 0af069d Windows: add wcsnlen
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=libc&package-manager=cargo&previous-version=0.2.175&new-version=0.2.177)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Eric Traut --- codex-rs/Cargo.lock | 4 ++-- codex-rs/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 7d57683de7..883df163cd 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -3588,9 +3588,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.175" +version = "0.2.177" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" +checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" [[package]] name = "libdbus-sys" diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index a14c1b7332..6ba445edc4 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -138,7 +138,7 @@ itertools = "0.14.0" keyring = { version = "3.6", default-features = false } landlock = "0.4.1" lazy_static = "1" -libc = "0.2.175" +libc = "0.2.177" log = "0.4" lru = "0.12.5" maplit = "1.0.2" From f6f4c8f5ee6cfc58a4df05bea0170058174442ad Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Nov 2025 12:03:10 -0800 Subject: [PATCH 16/61] chore(deps): bump webbrowser from 1.0.5 to 1.0.6 in /codex-rs (#7225) Bumps [webbrowser](https://github.com/amodm/webbrowser-rs) from 1.0.5 to 1.0.6.
Release notes

Sourced from webbrowser's releases.

v1.0.6

Fixed

  • Windows: fix browser opening when unicode characters exist in path. See PR #108
Changelog

Sourced from webbrowser's changelog.

[1.0.6] - 2025-10-15

Fixed

  • Windows: fix browser opening when unicode characters exist in path. See PR #108
Commits
  • 6596229 Release v1.0.6 [skip ci]
  • 44908ca ios: fix lint for objc2 invocation #build-ios
  • b76a217 Merge branch 'Nodeigi-fix/107'
  • ee2b1cd fix opening a browser that is located in a path that contains unicode characters
  • 061e65e ios: fix lints
  • 85dd4a3 macos: fix lints
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=webbrowser&package-manager=cargo&previous-version=1.0.5&new-version=1.0.6)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Eric Traut --- codex-rs/Cargo.lock | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 883df163cd..d966bc6cee 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -212,7 +212,7 @@ dependencies = [ "objc2-foundation", "parking_lot", "percent-encoding", - "windows-sys 0.59.0", + "windows-sys 0.52.0", "wl-clipboard-rs", "x11rb", ] @@ -2492,7 +2492,7 @@ checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" dependencies = [ "cfg-if", "rustix 1.0.8", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -3396,7 +3396,7 @@ checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -5173,7 +5173,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -7256,9 +7256,9 @@ dependencies = [ [[package]] name = "webbrowser" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaf4f3c0ba838e82b4e5ccc4157003fb8c324ee24c058470ffb82820becbde98" +checksum = "00f1243ef785213e3a32fa0396093424a3a6ea566f9948497e5a2309261a4c97" dependencies = [ "core-foundation 0.10.1", "jni", @@ -7325,7 +7325,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] From 2c885edefa21bc12340812ad69b84939ce6c5e73 Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Mon, 24 Nov 2025 14:04:27 -0600 Subject: [PATCH 17/61] Added alternate form of `dependabot` to CLA allow list (#7260) This fixes the CI check for CLA, which recently started to flag the GitHub dependabot as "not having signed the CLA". --- .github/workflows/cla.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cla.yml b/.github/workflows/cla.yml index 17d54f214a..12ee971c4b 100644 --- a/.github/workflows/cla.yml +++ b/.github/workflows/cla.yml @@ -48,4 +48,5 @@ jobs: branch: cla-signatures allowlist: | codex + dependabot dependabot[bot] From 3741f387e94c2c1e79c200b3ee9700b271f95676 Mon Sep 17 00:00:00 2001 From: Gabriel Peal Date: Mon, 24 Nov 2025 12:04:49 -0800 Subject: [PATCH 18/61] Allow enterprises to skip upgrade checks and messages (#7213) This is a feature primarily for enterprises who centrally manage Codex updates. --- codex-rs/core/src/config/mod.rs | 17 +++++++++++++++++ codex-rs/tui/src/updates.rs | 8 ++++++++ docs/config.md | 1 + 3 files changed, 26 insertions(+) diff --git a/codex-rs/core/src/config/mod.rs b/codex-rs/core/src/config/mod.rs index 3d5f25763c..b15465653d 100644 --- a/codex-rs/core/src/config/mod.rs +++ b/codex-rs/core/src/config/mod.rs @@ -267,6 +267,11 @@ pub struct Config { /// Collection of various notices we show the user pub notices: Notice, + /// When `true`, checks for Codex updates on startup and surfaces update prompts. + /// Set to `false` only if your Codex updates are centrally managed. + /// Defaults to `true`. + pub check_for_update_on_startup: bool, + /// When true, disables burst-paste detection for typed input entirely. /// All characters are inserted as they are received, and no buffering /// or placeholder replacement will occur for fast keypress bursts. @@ -685,6 +690,11 @@ pub struct ConfigToml { #[serde(default)] pub features: Option, + /// When `true`, checks for Codex updates on startup and surfaces update prompts. + /// Set to `false` only if your Codex updates are centrally managed. + /// Defaults to `true`. + pub check_for_update_on_startup: Option, + /// When true, disables burst-paste detection for typed input entirely. /// All characters are inserted as they are received, and no buffering /// or placeholder replacement will occur for fast keypress bursts. @@ -1162,6 +1172,8 @@ impl Config { .or(cfg.review_model) .unwrap_or_else(default_review_model); + let check_for_update_on_startup = cfg.check_for_update_on_startup.unwrap_or(true); + let config = Self { model, review_model, @@ -1238,6 +1250,7 @@ impl Config { active_project, windows_wsl_setup_acknowledged: cfg.windows_wsl_setup_acknowledged.unwrap_or(false), notices: cfg.notice.unwrap_or_default(), + check_for_update_on_startup, disable_paste_burst: cfg.disable_paste_burst.unwrap_or(false), tui_notifications: cfg .tui @@ -2992,6 +3005,7 @@ model_verbosity = "high" active_project: ProjectConfig { trust_level: None }, windows_wsl_setup_acknowledged: false, notices: Default::default(), + check_for_update_on_startup: true, disable_paste_burst: false, tui_notifications: Default::default(), animations: true, @@ -3064,6 +3078,7 @@ model_verbosity = "high" active_project: ProjectConfig { trust_level: None }, windows_wsl_setup_acknowledged: false, notices: Default::default(), + check_for_update_on_startup: true, disable_paste_burst: false, tui_notifications: Default::default(), animations: true, @@ -3151,6 +3166,7 @@ model_verbosity = "high" active_project: ProjectConfig { trust_level: None }, windows_wsl_setup_acknowledged: false, notices: Default::default(), + check_for_update_on_startup: true, disable_paste_burst: false, tui_notifications: Default::default(), animations: true, @@ -3224,6 +3240,7 @@ model_verbosity = "high" active_project: ProjectConfig { trust_level: None }, windows_wsl_setup_acknowledged: false, notices: Default::default(), + check_for_update_on_startup: true, disable_paste_burst: false, tui_notifications: Default::default(), animations: true, diff --git a/codex-rs/tui/src/updates.rs b/codex-rs/tui/src/updates.rs index 0c3f2044ef..89fd6f32f6 100644 --- a/codex-rs/tui/src/updates.rs +++ b/codex-rs/tui/src/updates.rs @@ -15,6 +15,10 @@ use std::path::PathBuf; use crate::version::CODEX_CLI_VERSION; pub fn get_upgrade_version(config: &Config) -> Option { + if !config.check_for_update_on_startup { + return None; + } + let version_file = version_filepath(config); let info = read_version_info(&version_file).ok(); @@ -141,6 +145,10 @@ fn extract_version_from_latest_tag(latest_tag_name: &str) -> anyhow::Result Option { + if !config.check_for_update_on_startup { + return None; + } + let version_file = version_filepath(config); let latest = get_upgrade_version(config)?; // If the user dismissed this exact version previously, do not show the popup. diff --git a/docs/config.md b/docs/config.md index 5051f27069..df4da3b005 100644 --- a/docs/config.md +++ b/docs/config.md @@ -977,6 +977,7 @@ Valid values: | `tui` | table | TUI‑specific options. | | `tui.notifications` | boolean \| array | Enable desktop notifications in the tui (default: true). | | `hide_agent_reasoning` | boolean | Hide model reasoning events. | +| `check_for_update_on_startup` | boolean | Check for Codex updates on startup (default: true). Set to `false` only if updates are centrally managed. | | `show_raw_agent_reasoning` | boolean | Show raw reasoning (when available). | | `model_reasoning_effort` | `minimal` \| `low` \| `medium` \| `high` | Responses API reasoning effort. | | `model_reasoning_summary` | `auto` \| `concise` \| `detailed` \| `none` | Reasoning summaries. | From 523dabc1eee4797a3d59fabe7208e5a83c662bc7 Mon Sep 17 00:00:00 2001 From: Priyadarshini Tamilselvan Date: Mon, 24 Nov 2025 15:18:32 -0500 Subject: [PATCH 19/61] fix: custom prompt expansion with large pastes (#7154) ### **Summary of Changes** **What?** Fix for slash commands (e.g., /prompts:code-review) not being recognized when large content (>3000 chars) is pasted. [Bug Report](https://github.com/openai/codex/issues/7047) **Why?** With large pastes, slash commands were ignored, so custom prompts weren't expanded and were submitted as literal text. **How?** Refactored the early return block in handle_key_event_without_popup (lines 957-968). Instead of returning early after replacing placeholders, the code now replaces placeholders in the textarea and continues to the normal submission flow. This reuses the existing slash command detection and custom prompt expansion logic (lines 981-1047), avoiding duplication. **Changes:** Modified codex-rs/tui/src/bottom_pane/chat_composer.rs: refactored early return block to continue to normal flow instead of returning immediately Added test: custom_prompt_with_large_paste_expands_correctly to verify the fix **Code Quality:** No lint warnings Code follows existing patterns and reuses existing logic Atomic change focused on the bug fix --- codex-rs/tui/src/bottom_pane/chat_composer.rs | 74 +++++++++++++++++-- 1 file changed, 67 insertions(+), 7 deletions(-) diff --git a/codex-rs/tui/src/bottom_pane/chat_composer.rs b/codex-rs/tui/src/bottom_pane/chat_composer.rs index 30727c298b..6b42e5134c 100644 --- a/codex-rs/tui/src/bottom_pane/chat_composer.rs +++ b/codex-rs/tui/src/bottom_pane/chat_composer.rs @@ -954,21 +954,17 @@ impl ChatComposer { return (InputResult::None, true); } } - // If we have pending placeholder pastes, submit immediately to expand them. + // If we have pending placeholder pastes, replace them in the textarea text + // and continue to the normal submission flow to handle slash commands. if !self.pending_pastes.is_empty() { let mut text = self.textarea.text().to_string(); - self.textarea.set_text(""); for (placeholder, actual) in &self.pending_pastes { if text.contains(placeholder) { text = text.replace(placeholder, actual); } } + self.textarea.set_text(&text); self.pending_pastes.clear(); - if text.is_empty() { - return (InputResult::None, true); - } - self.history.record_local_submission(&text); - return (InputResult::Submitted(text), true); } // During a paste-like burst, treat Enter as a newline instead of submit. @@ -3030,6 +3026,70 @@ mod tests { assert!(composer.textarea.is_empty()); } + #[test] + fn custom_prompt_with_large_paste_expands_correctly() { + use crossterm::event::KeyCode; + use crossterm::event::KeyEvent; + use crossterm::event::KeyModifiers; + + let (tx, _rx) = unbounded_channel::(); + let sender = AppEventSender::new(tx); + let mut composer = ChatComposer::new( + true, + sender, + false, + "Ask Codex to do anything".to_string(), + false, + ); + + // Create a custom prompt with positional args (no named args like $USER) + composer.set_custom_prompts(vec![CustomPrompt { + name: "code-review".to_string(), + path: "/tmp/code-review.md".to_string().into(), + content: "Please review the following code:\n\n$1".to_string(), + description: None, + argument_hint: None, + }]); + + // Type the slash command + let command_text = "/prompts:code-review "; + composer.textarea.set_text(command_text); + composer.textarea.set_cursor(command_text.len()); + + // Paste large content (>3000 chars) to trigger placeholder + let large_content = "x".repeat(LARGE_PASTE_CHAR_THRESHOLD + 3000); + composer.handle_paste(large_content.clone()); + + // Verify placeholder was created + let placeholder = format!("[Pasted Content {} chars]", large_content.chars().count()); + assert_eq!( + composer.textarea.text(), + format!("/prompts:code-review {}", placeholder) + ); + assert_eq!(composer.pending_pastes.len(), 1); + assert_eq!(composer.pending_pastes[0].0, placeholder); + assert_eq!(composer.pending_pastes[0].1, large_content); + + // Submit by pressing Enter + let (result, _needs_redraw) = + composer.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); + + // Verify the custom prompt was expanded with the large content as positional arg + match result { + InputResult::Submitted(text) => { + // The prompt should be expanded, with the large content replacing $1 + assert_eq!( + text, + format!("Please review the following code:\n\n{}", large_content), + "Expected prompt expansion with large content as $1" + ); + } + _ => panic!("expected Submitted, got: {result:?}"), + } + assert!(composer.textarea.is_empty()); + assert!(composer.pending_pastes.is_empty()); + } + #[test] fn slash_path_input_submits_without_command_error() { use crossterm::event::KeyCode; From 4ed4c73d6b071232cd6ea67170b6d6a2e382d1fa Mon Sep 17 00:00:00 2001 From: Josh McKinney Date: Mon, 24 Nov 2025 12:20:55 -0800 Subject: [PATCH 20/61] chore(ci): add cargo audit workflow and policy (#7108) - add to ignore current unmaintained advisories (derivative, fxhash, paste) so audits gate new issues only - introduce GitHub Actions workflow to run on push/PR using to install cargo-audit Existing advisories (all "unmaintained"): - https://rustsec.org/advisories/RUSTSEC-2024-0388 - https://rustsec.org/advisories/RUSTSEC-2025-0057 - https://rustsec.org/advisories/RUSTSEC-2024-0436 --- codex-rs/.cargo/audit.toml | 6 +++++ codex-rs/.github/workflows/cargo-audit.yml | 26 ++++++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 codex-rs/.cargo/audit.toml create mode 100644 codex-rs/.github/workflows/cargo-audit.yml diff --git a/codex-rs/.cargo/audit.toml b/codex-rs/.cargo/audit.toml new file mode 100644 index 0000000000..143e64163a --- /dev/null +++ b/codex-rs/.cargo/audit.toml @@ -0,0 +1,6 @@ +[advisories] +ignore = [ + "RUSTSEC-2024-0388", # derivative 2.2.0 via starlark; upstream crate is unmaintained + "RUSTSEC-2025-0057", # fxhash 0.2.1 via starlark_map; upstream crate is unmaintained + "RUSTSEC-2024-0436", # paste 1.0.15 via starlark/ratatui; upstream crate is unmaintained +] diff --git a/codex-rs/.github/workflows/cargo-audit.yml b/codex-rs/.github/workflows/cargo-audit.yml new file mode 100644 index 0000000000..e75c841ab4 --- /dev/null +++ b/codex-rs/.github/workflows/cargo-audit.yml @@ -0,0 +1,26 @@ +name: Cargo audit + +on: + pull_request: + push: + branches: + - main + +permissions: + contents: read + +jobs: + audit: + runs-on: ubuntu-latest + defaults: + run: + working-directory: codex-rs + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - name: Install cargo-audit + uses: taiki-e/install-action@v2 + with: + tool: cargo-audit + - name: Run cargo audit + run: cargo audit --deny warnings From ec49b5687476bd4a285e85b6aa380738db025a7f Mon Sep 17 00:00:00 2001 From: Josh McKinney Date: Mon, 24 Nov 2025 12:22:18 -0800 Subject: [PATCH 21/61] chore: add cargo-deny configuration (#7119) - add GitHub workflow running cargo-deny on push/PR - document cargo-deny allowlist with workspace-dep notes and advisory ignores - align workspace crates to inherit version/edition/license for consistent checks --- .github/workflows/cargo-deny.yml | 26 ++ codex-rs/Cargo.lock | 2 +- codex-rs/Cargo.toml | 1 + codex-rs/ansi-escape/Cargo.toml | 5 +- codex-rs/app-server-protocol/Cargo.toml | 5 +- codex-rs/app-server-test-client/Cargo.toml | 5 +- codex-rs/app-server/Cargo.toml | 5 +- codex-rs/app-server/tests/common/Cargo.toml | 5 +- codex-rs/apply-patch/Cargo.toml | 5 +- codex-rs/arg0/Cargo.toml | 5 +- codex-rs/async-utils/Cargo.toml | 3 +- codex-rs/backend-client/Cargo.toml | 5 +- codex-rs/chatgpt/Cargo.toml | 5 +- codex-rs/cli/Cargo.toml | 5 +- codex-rs/cloud-tasks-client/Cargo.toml | 5 +- codex-rs/cloud-tasks/Cargo.toml | 5 +- .../codex-backend-openapi-models/Cargo.toml | 5 +- codex-rs/common/Cargo.toml | 5 +- codex-rs/core/Cargo.toml | 5 +- codex-rs/core/tests/common/Cargo.toml | 5 +- codex-rs/deny.toml | 272 ++++++++++++++++++ codex-rs/exec-server/Cargo.toml | 5 +- codex-rs/exec/Cargo.toml | 5 +- codex-rs/execpolicy-legacy/Cargo.toml | 5 +- codex-rs/execpolicy/Cargo.toml | 5 +- codex-rs/feedback/Cargo.toml | 3 +- codex-rs/file-search/Cargo.toml | 5 +- codex-rs/keyring-store/Cargo.toml | 5 +- codex-rs/linux-sandbox/Cargo.toml | 5 +- codex-rs/lmstudio/Cargo.toml | 1 + codex-rs/login/Cargo.toml | 5 +- codex-rs/mcp-server/Cargo.toml | 5 +- codex-rs/mcp-server/tests/common/Cargo.toml | 5 +- codex-rs/mcp-types/Cargo.toml | 5 +- codex-rs/ollama/Cargo.toml | 5 +- codex-rs/otel/Cargo.toml | 5 +- codex-rs/process-hardening/Cargo.toml | 5 +- codex-rs/protocol/Cargo.toml | 5 +- codex-rs/responses-api-proxy/Cargo.toml | 5 +- codex-rs/rmcp-client/Cargo.toml | 5 +- codex-rs/stdio-to-uds/Cargo.toml | 5 +- codex-rs/tui/Cargo.toml | 5 +- codex-rs/utils/cache/Cargo.toml | 1 + codex-rs/utils/git/Cargo.toml | 5 +- codex-rs/utils/image/Cargo.toml | 1 + codex-rs/utils/json-to-toml/Cargo.toml | 3 +- codex-rs/utils/pty/Cargo.toml | 9 +- codex-rs/utils/readiness/Cargo.toml | 1 + codex-rs/utils/string/Cargo.toml | 3 +- codex-rs/windows-sandbox-rs/Cargo.toml | 11 +- 50 files changed, 429 insertions(+), 88 deletions(-) create mode 100644 .github/workflows/cargo-deny.yml create mode 100644 codex-rs/deny.toml diff --git a/.github/workflows/cargo-deny.yml b/.github/workflows/cargo-deny.yml new file mode 100644 index 0000000000..e365420cca --- /dev/null +++ b/.github/workflows/cargo-deny.yml @@ -0,0 +1,26 @@ +name: cargo-deny + +on: + pull_request: + push: + branches: + - main + +jobs: + cargo-deny: + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./codex-rs + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Run cargo-deny + uses: EmbarkStudios/cargo-deny-action@v1 + with: + rust-version: stable + manifest-path: ./codex-rs/Cargo.toml diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index d966bc6cee..09db252005 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1637,7 +1637,7 @@ version = "0.0.0" [[package]] name = "codex-windows-sandbox" -version = "0.1.0" +version = "0.0.0" dependencies = [ "anyhow", "codex-protocol", diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index 6ba445edc4..7d57b62ec1 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -51,6 +51,7 @@ version = "0.0.0" # crates created with `cargo new -w ...` automatically inherit the 2024 # edition. edition = "2024" +license = "Apache-2.0" [workspace.dependencies] # Internal diff --git a/codex-rs/ansi-escape/Cargo.toml b/codex-rs/ansi-escape/Cargo.toml index 4107a72754..a10dbf9134 100644 --- a/codex-rs/ansi-escape/Cargo.toml +++ b/codex-rs/ansi-escape/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-ansi-escape" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [lib] name = "codex_ansi_escape" diff --git a/codex-rs/app-server-protocol/Cargo.toml b/codex-rs/app-server-protocol/Cargo.toml index 47753b344a..c1a251c656 100644 --- a/codex-rs/app-server-protocol/Cargo.toml +++ b/codex-rs/app-server-protocol/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-app-server-protocol" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [lib] name = "codex_app_server_protocol" diff --git a/codex-rs/app-server-test-client/Cargo.toml b/codex-rs/app-server-test-client/Cargo.toml index 2fd14fb158..25a8813645 100644 --- a/codex-rs/app-server-test-client/Cargo.toml +++ b/codex-rs/app-server-test-client/Cargo.toml @@ -1,7 +1,8 @@ [package] name = "codex-app-server-test-client" -version = { workspace = true } -edition = "2024" +version.workspace = true +edition.workspace = true +license.workspace = true [lints] workspace = true diff --git a/codex-rs/app-server/Cargo.toml b/codex-rs/app-server/Cargo.toml index 0e7849b168..3189199a9f 100644 --- a/codex-rs/app-server/Cargo.toml +++ b/codex-rs/app-server/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-app-server" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [[bin]] name = "codex-app-server" diff --git a/codex-rs/app-server/tests/common/Cargo.toml b/codex-rs/app-server/tests/common/Cargo.toml index f8e6529d46..380f70d9b1 100644 --- a/codex-rs/app-server/tests/common/Cargo.toml +++ b/codex-rs/app-server/tests/common/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "app_test_support" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [lib] path = "lib.rs" diff --git a/codex-rs/apply-patch/Cargo.toml b/codex-rs/apply-patch/Cargo.toml index a239cd6310..1a918ce93b 100644 --- a/codex-rs/apply-patch/Cargo.toml +++ b/codex-rs/apply-patch/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-apply-patch" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [lib] name = "codex_apply_patch" diff --git a/codex-rs/arg0/Cargo.toml b/codex-rs/arg0/Cargo.toml index 10d09e4a4b..c82bdd58da 100644 --- a/codex-rs/arg0/Cargo.toml +++ b/codex-rs/arg0/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-arg0" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [lib] name = "codex_arg0" diff --git a/codex-rs/async-utils/Cargo.toml b/codex-rs/async-utils/Cargo.toml index 5203db0f5d..891af17a5f 100644 --- a/codex-rs/async-utils/Cargo.toml +++ b/codex-rs/async-utils/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition.workspace = true name = "codex-async-utils" version.workspace = true +edition.workspace = true +license.workspace = true [lints] workspace = true diff --git a/codex-rs/backend-client/Cargo.toml b/codex-rs/backend-client/Cargo.toml index 0cf802399c..ec5546a670 100644 --- a/codex-rs/backend-client/Cargo.toml +++ b/codex-rs/backend-client/Cargo.toml @@ -1,7 +1,8 @@ [package] name = "codex-backend-client" -version = "0.0.0" -edition = "2024" +version.workspace = true +edition.workspace = true +license.workspace = true publish = false [lib] diff --git a/codex-rs/chatgpt/Cargo.toml b/codex-rs/chatgpt/Cargo.toml index c46046b1f6..b58cd62340 100644 --- a/codex-rs/chatgpt/Cargo.toml +++ b/codex-rs/chatgpt/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-chatgpt" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [lints] workspace = true diff --git a/codex-rs/cli/Cargo.toml b/codex-rs/cli/Cargo.toml index e7999b5ce7..6c80a12595 100644 --- a/codex-rs/cli/Cargo.toml +++ b/codex-rs/cli/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-cli" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [[bin]] name = "codex" diff --git a/codex-rs/cloud-tasks-client/Cargo.toml b/codex-rs/cloud-tasks-client/Cargo.toml index 1a4eaa7aa8..15a206079a 100644 --- a/codex-rs/cloud-tasks-client/Cargo.toml +++ b/codex-rs/cloud-tasks-client/Cargo.toml @@ -1,7 +1,8 @@ [package] name = "codex-cloud-tasks-client" -version = { workspace = true } -edition = "2024" +version.workspace = true +edition.workspace = true +license.workspace = true [lib] name = "codex_cloud_tasks_client" diff --git a/codex-rs/cloud-tasks/Cargo.toml b/codex-rs/cloud-tasks/Cargo.toml index 46044fbb8d..c9edf5b4ad 100644 --- a/codex-rs/cloud-tasks/Cargo.toml +++ b/codex-rs/cloud-tasks/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-cloud-tasks" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [lib] name = "codex_cloud_tasks" diff --git a/codex-rs/codex-backend-openapi-models/Cargo.toml b/codex-rs/codex-backend-openapi-models/Cargo.toml index 1a600495b9..f9bad4a494 100644 --- a/codex-rs/codex-backend-openapi-models/Cargo.toml +++ b/codex-rs/codex-backend-openapi-models/Cargo.toml @@ -1,7 +1,8 @@ [package] name = "codex-backend-openapi-models" -version = { workspace = true } -edition = "2024" +version.workspace = true +edition.workspace = true +license.workspace = true [lib] name = "codex_backend_openapi_models" diff --git a/codex-rs/common/Cargo.toml b/codex-rs/common/Cargo.toml index cff9c4b307..377d054483 100644 --- a/codex-rs/common/Cargo.toml +++ b/codex-rs/common/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-common" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [lints] workspace = true diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index f0f1a430f3..d6db71973a 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-core" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [lib] doctest = false diff --git a/codex-rs/core/tests/common/Cargo.toml b/codex-rs/core/tests/common/Cargo.toml index 4c47fbb527..75af1b4dd6 100644 --- a/codex-rs/core/tests/common/Cargo.toml +++ b/codex-rs/core/tests/common/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "core_test_support" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [lib] path = "lib.rs" diff --git a/codex-rs/deny.toml b/codex-rs/deny.toml new file mode 100644 index 0000000000..f9d411f841 --- /dev/null +++ b/codex-rs/deny.toml @@ -0,0 +1,272 @@ +# This template contains all of the possible sections and their default values + +# Note that all fields that take a lint level have these possible values: +# * deny - An error will be produced and the check will fail +# * warn - A warning will be produced, but the check will not fail +# * allow - No warning or error will be produced, though in some cases a note +# will be + +# The values provided in this template are the default values that will be used +# when any section or field is not specified in your own configuration + +# Root options + +# The graph table configures how the dependency graph is constructed and thus +# which crates the checks are performed against +[graph] +# If 1 or more target triples (and optionally, target_features) are specified, +# only the specified targets will be checked when running `cargo deny check`. +# This means, if a particular package is only ever used as a target specific +# dependency, such as, for example, the `nix` crate only being used via the +# `target_family = "unix"` configuration, that only having windows targets in +# this list would mean the nix crate, as well as any of its exclusive +# dependencies not shared by any other crates, would be ignored, as the target +# list here is effectively saying which targets you are building for. +targets = [ + # The triple can be any string, but only the target triples built in to + # rustc (as of 1.40) can be checked against actual config expressions + #"x86_64-unknown-linux-musl", + # You can also specify which target_features you promise are enabled for a + # particular target. target_features are currently not validated against + # the actual valid features supported by the target architecture. + #{ triple = "wasm32-unknown-unknown", features = ["atomics"] }, +] +# When creating the dependency graph used as the source of truth when checks are +# executed, this field can be used to prune crates from the graph, removing them +# from the view of cargo-deny. This is an extremely heavy hammer, as if a crate +# is pruned from the graph, all of its dependencies will also be pruned unless +# they are connected to another crate in the graph that hasn't been pruned, +# so it should be used with care. The identifiers are [Package ID Specifications] +# (https://doc.rust-lang.org/cargo/reference/pkgid-spec.html) +#exclude = [] +# If true, metadata will be collected with `--all-features`. Note that this can't +# be toggled off if true, if you want to conditionally enable `--all-features` it +# is recommended to pass `--all-features` on the cmd line instead +all-features = false +# If true, metadata will be collected with `--no-default-features`. The same +# caveat with `all-features` applies +no-default-features = false +# If set, these feature will be enabled when collecting metadata. If `--features` +# is specified on the cmd line they will take precedence over this option. +#features = [] + +# The output table provides options for how/if diagnostics are outputted +[output] +# When outputting inclusion graphs in diagnostics that include features, this +# option can be used to specify the depth at which feature edges will be added. +# This option is included since the graphs can be quite large and the addition +# of features from the crate(s) to all of the graph roots can be far too verbose. +# This option can be overridden via `--feature-depth` on the cmd line +feature-depth = 1 + +# This section is considered when running `cargo deny check advisories` +# More documentation for the advisories section can be found here: +# https://embarkstudios.github.io/cargo-deny/checks/advisories/cfg.html +[advisories] +# The path where the advisory databases are cloned/fetched into +#db-path = "$CARGO_HOME/advisory-dbs" +# The url(s) of the advisory databases to use +#db-urls = ["https://github.com/rustsec/advisory-db"] +# A list of advisory IDs to ignore. Note that ignored advisories will still +# output a note when they are encountered. +ignore = [ + { id = "RUSTSEC-2024-0388", reason = "derivative is unmaintained; pulled in via starlark v0.13.0 used by execpolicy/cli/core; no fixed release yet" }, + { id = "RUSTSEC-2025-0057", reason = "fxhash is unmaintained; pulled in via starlark_map/starlark v0.13.0 used by execpolicy/cli/core; no fixed release yet" }, + { id = "RUSTSEC-2024-0436", reason = "paste is unmaintained; pulled in via ratatui/rmcp/starlark used by tui/execpolicy; no fixed release yet" }, +] +# If this is true, then cargo deny will use the git executable to fetch advisory database. +# If this is false, then it uses a built-in git library. +# Setting this to true can be helpful if you have special authentication requirements that cargo-deny does not support. +# See Git Authentication for more information about setting up git authentication. +#git-fetch-with-cli = true + +# This section is considered when running `cargo deny check licenses` +# More documentation for the licenses section can be found here: +# https://embarkstudios.github.io/cargo-deny/checks/licenses/cfg.html +[licenses] +# List of explicitly allowed licenses +# See https://spdx.org/licenses/ for list of possible licenses +# [possible values: any SPDX 3.11 short identifier (+ optional exception)]. +allow = [ + # Apache-2.0 - https://www.apache.org/licenses/LICENSE-2.0 + # Used by: allocative, anyhow, arboard, askama, assert_cmd, assert_matches, async-channel, async-trait, base64, chardetng, chrono, clap, clap_complete, color-eyre, ctor, diffy, dirs, dunce, encoding_rs, env_logger, escargot, eventsource-stream, futures, http, image, indexmap, insta, itertools, keyring, landlock, lazy_static, libc, log, maplit, multimap, once_cell, opentelemetry, opentelemetry-appender-tracing, opentelemetry-otlp, opentelemetry-semantic-conventions, opentelemetry_sdk, pathdiff, predicates, pretty_assertions, rand, regex-lite, reqwest, seccompiler, serde, serde_json, serde_with, sha1, sha2, shlex, similar, socket2, starlark, supports-color, sys-locale, tempfile, test-log, thiserror, time, tiny_http, toml, toml_edit, unicode-segmentation, unicode-width, url, uuid, webbrowser, wiremock, zeroize + "Apache-2.0", + # BSD-2-Clause - https://opensource.org/license/bsd-2-clause + # Used by: transitive only + "BSD-2-Clause", + # BSD-3-Clause - https://opensource.org/license/bsd-3-clause + # Used by: encoding_rs, seccompiler + "BSD-3-Clause", + # BSL-1.0 - https://www.boost.org/users/license.html + # Used by: transitive only + "BSL-1.0", + # CC0-1.0 - https://creativecommons.org/publicdomain/zero/1.0/ + # Used by: dunce, notify + "CC0-1.0", + # CDLA-Permissive-2.0 - https://cdla.dev/permissive-2-0/ + # Used by: transitive only + "CDLA-Permissive-2.0", + # ISC - https://opensource.org/license/isc-license-txt + # Used by: transitive only + "ISC", + # MIT - https://opensource.org/license/mit + # Used by: allocative, ansi-to-tui, anyhow, arboard, askama, assert_cmd, assert_matches, async-channel, async-stream, async-trait, axum, base64, bytes, chardetng, chrono, clap, clap_complete, color-eyre, crossterm, ctor, derive_more, diffy, dirs, dotenvy, encoding_rs, env-flags, env_logger, escargot, eventsource-stream, futures, http, ignore, image, indexmap, itertools, keyring, landlock, lazy_static, libc, log, lru, maplit, mime_guess, multimap, once_cell, openssl-sys, os_info, owo-colors, path-absolutize, pathdiff, portable-pty, predicates, pretty_assertions, pulldown-cmark, rand, ratatui, ratatui-macros, regex-lite, reqwest, rmcp, schemars, serde, serde_json, serde_with, serial_test, sha1, sha2, shlex, socket2, strum, strum_macros, sys-locale, tempfile, test-log, textwrap, thiserror, time, tiny_http, tokio, tokio-stream, tokio-test, tokio-util, toml, toml_edit, tonic, tracing, tracing-appender, tracing-subscriber, tracing-test, tree-sitter, tree-sitter-bash, tree-sitter-highlight, ts-rs, uds_windows, unicode-segmentation, unicode-width, url, urlencoding, uuid, vt100, walkdir, webbrowser, which, wildmatch, wiremock, zeroize + "MIT", + # MIT-0 - https://opensource.org/license/mit-0 + # Used by: dunce + "MIT-0", + # MPL-2.0 - https://www.mozilla.org/MPL/2.0/ + # Used by: nucleo-matcher + "MPL-2.0", + # Unicode-3.0 - https://opensource.org/license/unicode + # Used by: icu_decimal, icu_locale_core, icu_provider + "Unicode-3.0", + # Unlicense - https://opensource.org/license/unlicense/ + # Used by: ignore, walkdir + "Unlicense", + # Zlib - https://opensource.org/license/zlib + # Used by: transitive only + "Zlib", +] +# The confidence threshold for detecting a license from license text. +# The higher the value, the more closely the license text must be to the +# canonical license text of a valid SPDX license file. +# [possible values: any between 0.0 and 1.0]. +confidence-threshold = 0.8 +# Allow 1 or more licenses on a per-crate basis, so that particular licenses +# aren't accepted for every possible crate as with the normal allow list +exceptions = [ + # Each entry is the crate and version constraint, and its specific allow + # list + #{ allow = ["Zlib"], crate = "adler32" }, +] + +# Some crates don't have (easily) machine readable licensing information, +# adding a clarification entry for it allows you to manually specify the +# licensing information +#[[licenses.clarify]] +# The package spec the clarification applies to +#crate = "ring" +# The SPDX expression for the license requirements of the crate +#expression = "MIT AND ISC AND OpenSSL" +# One or more files in the crate's source used as the "source of truth" for +# the license expression. If the contents match, the clarification will be used +# when running the license check, otherwise the clarification will be ignored +# and the crate will be checked normally, which may produce warnings or errors +# depending on the rest of your configuration +#license-files = [ +# Each entry is a crate relative path, and the (opaque) hash of its contents +#{ path = "LICENSE", hash = 0xbd0eed23 } +#] + +[licenses.private] +# If true, ignores workspace crates that aren't published, or are only +# published to private registries. +# To see how to mark a crate as unpublished (to the official registry), +# visit https://doc.rust-lang.org/cargo/reference/manifest.html#the-publish-field. +ignore = false +# One or more private registries that you might publish crates to, if a crate +# is only published to private registries, and ignore is true, the crate will +# not have its license(s) checked +registries = [ + #"https://sekretz.com/registry +] + +# This section is considered when running `cargo deny check bans`. +# More documentation about the 'bans' section can be found here: +# https://embarkstudios.github.io/cargo-deny/checks/bans/cfg.html +[bans] +# Lint level for when multiple versions of the same crate are detected +multiple-versions = "warn" +# Lint level for when a crate version requirement is `*` +wildcards = "allow" +# The graph highlighting used when creating dotgraphs for crates +# with multiple versions +# * lowest-version - The path to the lowest versioned duplicate is highlighted +# * simplest-path - The path to the version with the fewest edges is highlighted +# * all - Both lowest-version and simplest-path are used +highlight = "all" +# The default lint level for `default` features for crates that are members of +# the workspace that is being checked. This can be overridden by allowing/denying +# `default` on a crate-by-crate basis if desired. +workspace-default-features = "allow" +# The default lint level for `default` features for external crates that are not +# members of the workspace. This can be overridden by allowing/denying `default` +# on a crate-by-crate basis if desired. +external-default-features = "allow" +# List of crates that are allowed. Use with care! +allow = [ + #"ansi_term@0.11.0", + #{ crate = "ansi_term@0.11.0", reason = "you can specify a reason it is allowed" }, +] +# List of crates to deny +deny = [ + #"ansi_term@0.11.0", + #{ crate = "ansi_term@0.11.0", reason = "you can specify a reason it is banned" }, + # Wrapper crates can optionally be specified to allow the crate when it + # is a direct dependency of the otherwise banned crate + #{ crate = "ansi_term@0.11.0", wrappers = ["this-crate-directly-depends-on-ansi_term"] }, +] + +# List of features to allow/deny +# Each entry the name of a crate and a version range. If version is +# not specified, all versions will be matched. +#[[bans.features]] +#crate = "reqwest" +# Features to not allow +#deny = ["json"] +# Features to allow +#allow = [ +# "rustls", +# "__rustls", +# "__tls", +# "hyper-rustls", +# "rustls", +# "rustls-pemfile", +# "rustls-tls-webpki-roots", +# "tokio-rustls", +# "webpki-roots", +#] +# If true, the allowed features must exactly match the enabled feature set. If +# this is set there is no point setting `deny` +#exact = true + +# Certain crates/versions that will be skipped when doing duplicate detection. +skip = [ + #"ansi_term@0.11.0", + #{ crate = "ansi_term@0.11.0", reason = "you can specify a reason why it can't be updated/removed" }, +] +# Similarly to `skip` allows you to skip certain crates during duplicate +# detection. Unlike skip, it also includes the entire tree of transitive +# dependencies starting at the specified crate, up to a certain depth, which is +# by default infinite. +skip-tree = [ + #"ansi_term@0.11.0", # will be skipped along with _all_ of its direct and transitive dependencies + #{ crate = "ansi_term@0.11.0", depth = 20 }, +] + +# This section is considered when running `cargo deny check sources`. +# More documentation about the 'sources' section can be found here: +# https://embarkstudios.github.io/cargo-deny/checks/sources/cfg.html +[sources] +# Lint level for what to happen when a crate from a crate registry that is not +# in the allow list is encountered +unknown-registry = "warn" +# Lint level for what to happen when a crate from a git repository that is not +# in the allow list is encountered +unknown-git = "warn" +# List of URLs for allowed crate registries. Defaults to the crates.io index +# if not specified. If it is specified but empty, no registries are allowed. +allow-registry = ["https://github.com/rust-lang/crates.io-index"] +# List of URLs for allowed Git repositories +allow-git = [] + +[sources.allow-org] +# github.com organizations to allow git sources for +github = [ + "nornagon", # ratatui and crossterm forks +] +# gitlab.com organizations to allow git sources for +gitlab = [] +# bitbucket.org organizations to allow git sources for +bitbucket = [] diff --git a/codex-rs/exec-server/Cargo.toml b/codex-rs/exec-server/Cargo.toml index 24c13e0e25..5f8032595e 100644 --- a/codex-rs/exec-server/Cargo.toml +++ b/codex-rs/exec-server/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-exec-server" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [[bin]] name = "codex-execve-wrapper" diff --git a/codex-rs/exec/Cargo.toml b/codex-rs/exec/Cargo.toml index 4a0ddf7dde..f26bec0e67 100644 --- a/codex-rs/exec/Cargo.toml +++ b/codex-rs/exec/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-exec" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [[bin]] name = "codex-exec" diff --git a/codex-rs/execpolicy-legacy/Cargo.toml b/codex-rs/execpolicy-legacy/Cargo.toml index 89e6e43b41..17628a0478 100644 --- a/codex-rs/execpolicy-legacy/Cargo.toml +++ b/codex-rs/execpolicy-legacy/Cargo.toml @@ -1,8 +1,9 @@ [package] -edition = "2024" name = "codex-execpolicy-legacy" +version.workspace = true +edition.workspace = true +license.workspace = true description = "Legacy exec policy engine for validating proposed exec calls." -version = { workspace = true } [[bin]] name = "codex-execpolicy-legacy" diff --git a/codex-rs/execpolicy/Cargo.toml b/codex-rs/execpolicy/Cargo.toml index bececed4b2..77278bb118 100644 --- a/codex-rs/execpolicy/Cargo.toml +++ b/codex-rs/execpolicy/Cargo.toml @@ -1,7 +1,8 @@ [package] name = "codex-execpolicy" -version = { workspace = true } -edition = "2024" +version.workspace = true +edition.workspace = true +license.workspace = true description = "Codex exec policy: prefix-based Starlark rules for command decisions." [lib] diff --git a/codex-rs/feedback/Cargo.toml b/codex-rs/feedback/Cargo.toml index b104f51225..0697704072 100644 --- a/codex-rs/feedback/Cargo.toml +++ b/codex-rs/feedback/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition.workspace = true name = "codex-feedback" version.workspace = true +edition.workspace = true +license.workspace = true [dependencies] anyhow = { workspace = true } diff --git a/codex-rs/file-search/Cargo.toml b/codex-rs/file-search/Cargo.toml index 40671389e1..e0dea1c139 100644 --- a/codex-rs/file-search/Cargo.toml +++ b/codex-rs/file-search/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-file-search" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [[bin]] name = "codex-file-search" diff --git a/codex-rs/keyring-store/Cargo.toml b/codex-rs/keyring-store/Cargo.toml index 932693de50..dd3686ee59 100644 --- a/codex-rs/keyring-store/Cargo.toml +++ b/codex-rs/keyring-store/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-keyring-store" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [lints] workspace = true diff --git a/codex-rs/linux-sandbox/Cargo.toml b/codex-rs/linux-sandbox/Cargo.toml index 264f15e751..cac8604f5d 100644 --- a/codex-rs/linux-sandbox/Cargo.toml +++ b/codex-rs/linux-sandbox/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-linux-sandbox" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [[bin]] name = "codex-linux-sandbox" diff --git a/codex-rs/lmstudio/Cargo.toml b/codex-rs/lmstudio/Cargo.toml index 4035d5529b..597ad3033f 100644 --- a/codex-rs/lmstudio/Cargo.toml +++ b/codex-rs/lmstudio/Cargo.toml @@ -2,6 +2,7 @@ name = "codex-lmstudio" version.workspace = true edition.workspace = true +license.workspace = true [lib] name = "codex_lmstudio" diff --git a/codex-rs/login/Cargo.toml b/codex-rs/login/Cargo.toml index 563515c637..94af9e19af 100644 --- a/codex-rs/login/Cargo.toml +++ b/codex-rs/login/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-login" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [lints] workspace = true diff --git a/codex-rs/mcp-server/Cargo.toml b/codex-rs/mcp-server/Cargo.toml index 484af6d8e8..00a2204578 100644 --- a/codex-rs/mcp-server/Cargo.toml +++ b/codex-rs/mcp-server/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-mcp-server" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [[bin]] name = "codex-mcp-server" diff --git a/codex-rs/mcp-server/tests/common/Cargo.toml b/codex-rs/mcp-server/tests/common/Cargo.toml index 286cd63c7f..c549a03fc9 100644 --- a/codex-rs/mcp-server/tests/common/Cargo.toml +++ b/codex-rs/mcp-server/tests/common/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "mcp_test_support" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [lib] path = "lib.rs" diff --git a/codex-rs/mcp-types/Cargo.toml b/codex-rs/mcp-types/Cargo.toml index 8ea4fdfee5..92cf539611 100644 --- a/codex-rs/mcp-types/Cargo.toml +++ b/codex-rs/mcp-types/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "mcp-types" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [lints] workspace = true diff --git a/codex-rs/ollama/Cargo.toml b/codex-rs/ollama/Cargo.toml index 14dd6d2fcd..ee16bd5e05 100644 --- a/codex-rs/ollama/Cargo.toml +++ b/codex-rs/ollama/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-ollama" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [lib] name = "codex_ollama" diff --git a/codex-rs/otel/Cargo.toml b/codex-rs/otel/Cargo.toml index 2ab170c917..059e8e38e6 100644 --- a/codex-rs/otel/Cargo.toml +++ b/codex-rs/otel/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-otel" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [lib] doctest = false diff --git a/codex-rs/process-hardening/Cargo.toml b/codex-rs/process-hardening/Cargo.toml index 2ba4b0d5ca..2a867572df 100644 --- a/codex-rs/process-hardening/Cargo.toml +++ b/codex-rs/process-hardening/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-process-hardening" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [lib] name = "codex_process_hardening" diff --git a/codex-rs/protocol/Cargo.toml b/codex-rs/protocol/Cargo.toml index 00ed100e08..08f8375357 100644 --- a/codex-rs/protocol/Cargo.toml +++ b/codex-rs/protocol/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-protocol" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [lib] name = "codex_protocol" diff --git a/codex-rs/responses-api-proxy/Cargo.toml b/codex-rs/responses-api-proxy/Cargo.toml index 651019a816..e0ea600033 100644 --- a/codex-rs/responses-api-proxy/Cargo.toml +++ b/codex-rs/responses-api-proxy/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-responses-api-proxy" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [lib] name = "codex_responses_api_proxy" diff --git a/codex-rs/rmcp-client/Cargo.toml b/codex-rs/rmcp-client/Cargo.toml index 5c3f1dc0c8..f2525c18d3 100644 --- a/codex-rs/rmcp-client/Cargo.toml +++ b/codex-rs/rmcp-client/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-rmcp-client" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [lints] workspace = true diff --git a/codex-rs/stdio-to-uds/Cargo.toml b/codex-rs/stdio-to-uds/Cargo.toml index 4f7132883d..71ed3722c3 100644 --- a/codex-rs/stdio-to-uds/Cargo.toml +++ b/codex-rs/stdio-to-uds/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-stdio-to-uds" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [[bin]] name = "codex-stdio-to-uds" diff --git a/codex-rs/tui/Cargo.toml b/codex-rs/tui/Cargo.toml index 01fef9cc74..d9906b2f01 100644 --- a/codex-rs/tui/Cargo.toml +++ b/codex-rs/tui/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition = "2024" name = "codex-tui" -version = { workspace = true } +version.workspace = true +edition.workspace = true +license.workspace = true [[bin]] name = "codex-tui" diff --git a/codex-rs/utils/cache/Cargo.toml b/codex-rs/utils/cache/Cargo.toml index e3397bfe1b..c034ad1346 100644 --- a/codex-rs/utils/cache/Cargo.toml +++ b/codex-rs/utils/cache/Cargo.toml @@ -2,6 +2,7 @@ name = "codex-utils-cache" version.workspace = true edition.workspace = true +license.workspace = true [lints] workspace = true diff --git a/codex-rs/utils/git/Cargo.toml b/codex-rs/utils/git/Cargo.toml index 5fed0aecf2..ef673644a7 100644 --- a/codex-rs/utils/git/Cargo.toml +++ b/codex-rs/utils/git/Cargo.toml @@ -1,8 +1,9 @@ [package] -edition.workspace = true name = "codex-git" -readme = "README.md" version.workspace = true +edition.workspace = true +license.workspace = true +readme = "README.md" [lints] workspace = true diff --git a/codex-rs/utils/image/Cargo.toml b/codex-rs/utils/image/Cargo.toml index 6e83d4d0df..b523de5102 100644 --- a/codex-rs/utils/image/Cargo.toml +++ b/codex-rs/utils/image/Cargo.toml @@ -2,6 +2,7 @@ name = "codex-utils-image" version.workspace = true edition.workspace = true +license.workspace = true [lints] workspace = true diff --git a/codex-rs/utils/json-to-toml/Cargo.toml b/codex-rs/utils/json-to-toml/Cargo.toml index a665724d78..36e848d7de 100644 --- a/codex-rs/utils/json-to-toml/Cargo.toml +++ b/codex-rs/utils/json-to-toml/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition.workspace = true name = "codex-utils-json-to-toml" version.workspace = true +edition.workspace = true +license.workspace = true [dependencies] serde_json = { workspace = true } diff --git a/codex-rs/utils/pty/Cargo.toml b/codex-rs/utils/pty/Cargo.toml index 117d626318..d640c71aa7 100644 --- a/codex-rs/utils/pty/Cargo.toml +++ b/codex-rs/utils/pty/Cargo.toml @@ -1,7 +1,8 @@ [package] edition = "2021" +license.workspace = true name = "codex-utils-pty" -version = { workspace = true } +version.workspace = true [lints] workspace = true @@ -9,8 +10,4 @@ workspace = true [dependencies] anyhow = { workspace = true } portable-pty = { workspace = true } -tokio = { workspace = true, features = [ - "macros", - "rt-multi-thread", - "sync", -] } +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "sync"] } diff --git a/codex-rs/utils/readiness/Cargo.toml b/codex-rs/utils/readiness/Cargo.toml index cb76b52b79..12519d4adb 100644 --- a/codex-rs/utils/readiness/Cargo.toml +++ b/codex-rs/utils/readiness/Cargo.toml @@ -2,6 +2,7 @@ name = "codex-utils-readiness" version.workspace = true edition.workspace = true +license.workspace = true [dependencies] async-trait = { workspace = true } diff --git a/codex-rs/utils/string/Cargo.toml b/codex-rs/utils/string/Cargo.toml index 698c4b2f6f..29437eaefc 100644 --- a/codex-rs/utils/string/Cargo.toml +++ b/codex-rs/utils/string/Cargo.toml @@ -1,7 +1,8 @@ [package] -edition.workspace = true name = "codex-utils-string" version.workspace = true +edition.workspace = true +license.workspace = true [lints] workspace = true diff --git a/codex-rs/windows-sandbox-rs/Cargo.toml b/codex-rs/windows-sandbox-rs/Cargo.toml index 7664ca719a..481074b925 100644 --- a/codex-rs/windows-sandbox-rs/Cargo.toml +++ b/codex-rs/windows-sandbox-rs/Cargo.toml @@ -1,7 +1,8 @@ [package] -name = "codex-windows-sandbox" -version = "0.1.0" edition = "2021" +license.workspace = true +name = "codex-windows-sandbox" +version.workspace = true [lib] name = "codex_windows_sandbox" @@ -9,20 +10,19 @@ path = "src/lib.rs" [dependencies] anyhow = "1.0" +dunce = "1.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -dunce = "1.0" [dependencies.codex-protocol] package = "codex-protocol" path = "../protocol" [dependencies.rand] -version = "0.8" default-features = false features = ["std", "small_rng"] +version = "0.8" [dependencies.dirs-next] version = "2.0" [dependencies.windows-sys] -version = "0.52" features = [ "Win32_Foundation", "Win32_System_Diagnostics_Debug", @@ -45,3 +45,4 @@ features = [ "Win32_System_Com", "Win32_Security_Authentication_Identity", ] +version = "0.52" From 99e5340c54d1b66f3c080863c24e890541f8d05b Mon Sep 17 00:00:00 2001 From: iceweasel-oai Date: Mon, 24 Nov 2025 12:41:21 -0800 Subject: [PATCH 22/61] Windows Sandbox: treat /.git as read-only in workspace-write mode (#7142) this functionality is [supported](https://github.com/openai/codex/blob/main/codex-rs/protocol/src/protocol.rs#L421-L422) in the MacOs sandbox as well. Adding it to Windows for parity This PR also changes `rust-ci.yaml` to work around a github `hashFiles` issue. Others have done something [similar](https://github.com/openai/superassistant/pull/32156) today --- .github/workflows/rust-ci.yml | 38 ++++-- codex-rs/Cargo.lock | 1 + codex-rs/windows-sandbox-rs/Cargo.toml | 2 + codex-rs/windows-sandbox-rs/src/allow.rs | 154 +++++++++++++++++------ codex-rs/windows-sandbox-rs/src/lib.rs | 14 ++- 5 files changed, 160 insertions(+), 49 deletions(-) diff --git a/.github/workflows/rust-ci.yml b/.github/workflows/rust-ci.yml index e72a69f7e3..08c39db69d 100644 --- a/.github/workflows/rust-ci.yml +++ b/.github/workflows/rust-ci.yml @@ -153,6 +153,15 @@ jobs: targets: ${{ matrix.target }} components: clippy + - name: Compute lockfile hash + id: lockhash + working-directory: codex-rs + shell: bash + run: | + set -euo pipefail + echo "hash=$(sha256sum Cargo.lock | cut -d' ' -f1)" >> "$GITHUB_OUTPUT" + echo "toolchain_hash=$(sha256sum rust-toolchain.toml | cut -d' ' -f1)" >> "$GITHUB_OUTPUT" + # Explicit cache restore: split cargo home vs target, so we can # avoid caching the large target dir on the gnu-dev job. - name: Restore cargo home cache @@ -164,7 +173,7 @@ jobs: ~/.cargo/registry/index/ ~/.cargo/registry/cache/ ~/.cargo/git/db/ - key: cargo-home-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ hashFiles('**/Cargo.lock') }}-${{ hashFiles('codex-rs/rust-toolchain.toml') }} + key: cargo-home-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ steps.lockhash.outputs.hash }}-${{ steps.lockhash.outputs.toolchain_hash }} restore-keys: | cargo-home-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}- @@ -201,9 +210,9 @@ jobs: uses: actions/cache/restore@v4 with: path: ${{ github.workspace }}/.sccache/ - key: sccache-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ hashFiles('**/Cargo.lock') }}-${{ github.run_id }} + key: sccache-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ steps.lockhash.outputs.hash }}-${{ github.run_id }} restore-keys: | - sccache-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ hashFiles('**/Cargo.lock') }}- + sccache-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ steps.lockhash.outputs.hash }}- sccache-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}- - if: ${{ matrix.target == 'x86_64-unknown-linux-musl' || matrix.target == 'aarch64-unknown-linux-musl'}} @@ -278,7 +287,7 @@ jobs: ~/.cargo/registry/index/ ~/.cargo/registry/cache/ ~/.cargo/git/db/ - key: cargo-home-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ hashFiles('**/Cargo.lock') }}-${{ hashFiles('codex-rs/rust-toolchain.toml') }} + key: cargo-home-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ steps.lockhash.outputs.hash }}-${{ steps.lockhash.outputs.toolchain_hash }} - name: Save sccache cache (fallback) if: always() && !cancelled() && env.USE_SCCACHE == 'true' && env.SCCACHE_GHA_ENABLED != 'true' @@ -286,7 +295,7 @@ jobs: uses: actions/cache/save@v4 with: path: ${{ github.workspace }}/.sccache/ - key: sccache-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ hashFiles('**/Cargo.lock') }}-${{ github.run_id }} + key: sccache-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ steps.lockhash.outputs.hash }}-${{ github.run_id }} - name: sccache stats if: always() && env.USE_SCCACHE == 'true' @@ -364,6 +373,15 @@ jobs: with: targets: ${{ matrix.target }} + - name: Compute lockfile hash + id: lockhash + working-directory: codex-rs + shell: bash + run: | + set -euo pipefail + echo "hash=$(sha256sum Cargo.lock | cut -d' ' -f1)" >> "$GITHUB_OUTPUT" + echo "toolchain_hash=$(sha256sum rust-toolchain.toml | cut -d' ' -f1)" >> "$GITHUB_OUTPUT" + - name: Restore cargo home cache id: cache_cargo_home_restore uses: actions/cache/restore@v4 @@ -373,7 +391,7 @@ jobs: ~/.cargo/registry/index/ ~/.cargo/registry/cache/ ~/.cargo/git/db/ - key: cargo-home-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ hashFiles('**/Cargo.lock') }}-${{ hashFiles('codex-rs/rust-toolchain.toml') }} + key: cargo-home-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ steps.lockhash.outputs.hash }}-${{ steps.lockhash.outputs.toolchain_hash }} restore-keys: | cargo-home-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}- @@ -409,9 +427,9 @@ jobs: uses: actions/cache/restore@v4 with: path: ${{ github.workspace }}/.sccache/ - key: sccache-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ hashFiles('**/Cargo.lock') }}-${{ github.run_id }} + key: sccache-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ steps.lockhash.outputs.hash }}-${{ github.run_id }} restore-keys: | - sccache-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ hashFiles('**/Cargo.lock') }}- + sccache-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ steps.lockhash.outputs.hash }}- sccache-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}- - uses: taiki-e/install-action@44c6d64aa62cd779e873306675c7a58e86d6d532 # v2 @@ -436,7 +454,7 @@ jobs: ~/.cargo/registry/index/ ~/.cargo/registry/cache/ ~/.cargo/git/db/ - key: cargo-home-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ hashFiles('**/Cargo.lock') }}-${{ hashFiles('codex-rs/rust-toolchain.toml') }} + key: cargo-home-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ steps.lockhash.outputs.hash }}-${{ steps.lockhash.outputs.toolchain_hash }} - name: Save sccache cache (fallback) if: always() && !cancelled() && env.USE_SCCACHE == 'true' && env.SCCACHE_GHA_ENABLED != 'true' @@ -444,7 +462,7 @@ jobs: uses: actions/cache/save@v4 with: path: ${{ github.workspace }}/.sccache/ - key: sccache-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ hashFiles('**/Cargo.lock') }}-${{ github.run_id }} + key: sccache-${{ matrix.runner }}-${{ matrix.target }}-${{ matrix.profile }}-${{ steps.lockhash.outputs.hash }}-${{ github.run_id }} - name: sccache stats if: always() && env.USE_SCCACHE == 'true' diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 09db252005..350561ba5d 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1646,6 +1646,7 @@ dependencies = [ "rand 0.8.5", "serde", "serde_json", + "tempfile", "windows-sys 0.52.0", ] diff --git a/codex-rs/windows-sandbox-rs/Cargo.toml b/codex-rs/windows-sandbox-rs/Cargo.toml index 481074b925..29306371b2 100644 --- a/codex-rs/windows-sandbox-rs/Cargo.toml +++ b/codex-rs/windows-sandbox-rs/Cargo.toml @@ -46,3 +46,5 @@ features = [ "Win32_Security_Authentication_Identity", ] version = "0.52" +[dev-dependencies] +tempfile = "3" diff --git a/codex-rs/windows-sandbox-rs/src/allow.rs b/codex-rs/windows-sandbox-rs/src/allow.rs index 7a879df4ac..1957ffcfe9 100644 --- a/codex-rs/windows-sandbox-rs/src/allow.rs +++ b/codex-rs/windows-sandbox-rs/src/allow.rs @@ -1,21 +1,33 @@ use crate::policy::SandboxPolicy; use dunce::canonicalize; use std::collections::HashMap; +use std::collections::HashSet; use std::path::Path; use std::path::PathBuf; +#[derive(Debug, Default, PartialEq, Eq)] +pub struct AllowDenyPaths { + pub allow: HashSet, + pub deny: HashSet, +} + pub fn compute_allow_paths( policy: &SandboxPolicy, policy_cwd: &Path, command_cwd: &Path, env_map: &HashMap, -) -> Vec { - let mut allow: Vec = Vec::new(); - let mut seen = std::collections::HashSet::new(); +) -> AllowDenyPaths { + let mut allow: HashSet = HashSet::new(); + let mut deny: HashSet = HashSet::new(); - let mut add_path = |p: PathBuf| { - if seen.insert(p.to_string_lossy().to_string()) && p.exists() { - allow.push(p); + let mut add_allow_path = |p: PathBuf| { + if p.exists() { + allow.insert(p); + } + }; + let mut add_deny_path = |p: PathBuf| { + if p.exists() { + deny.insert(p); } }; let include_tmp_env_vars = matches!( @@ -27,16 +39,40 @@ pub fn compute_allow_paths( ); if matches!(policy, SandboxPolicy::WorkspaceWrite { .. }) { - add_path(command_cwd.to_path_buf()); - if let SandboxPolicy::WorkspaceWrite { writable_roots, .. } = policy { - for root in writable_roots { + let add_writable_root = + |root: PathBuf, + policy_cwd: &Path, + add_allow: &mut dyn FnMut(PathBuf), + add_deny: &mut dyn FnMut(PathBuf)| { let candidate = if root.is_absolute() { - root.clone() + root } else { policy_cwd.join(root) }; let canonical = canonicalize(&candidate).unwrap_or(candidate); - add_path(canonical); + add_allow(canonical.clone()); + + let git_dir = canonical.join(".git"); + if git_dir.is_dir() { + add_deny(git_dir); + } + }; + + add_writable_root( + command_cwd.to_path_buf(), + policy_cwd, + &mut add_allow_path, + &mut add_deny_path, + ); + + if let SandboxPolicy::WorkspaceWrite { writable_roots, .. } = policy { + for root in writable_roots { + add_writable_root( + root.clone(), + policy_cwd, + &mut add_allow_path, + &mut add_deny_path, + ); } } } @@ -44,14 +80,14 @@ pub fn compute_allow_paths( for key in ["TEMP", "TMP"] { if let Some(v) = env_map.get(key) { let abs = PathBuf::from(v); - add_path(abs); + add_allow_path(abs); } else if let Ok(v) = std::env::var(key) { let abs = PathBuf::from(v); - add_path(abs); + add_allow_path(abs); } } } - allow + AllowDenyPaths { allow, deny } } #[cfg(test)] @@ -59,13 +95,16 @@ mod tests { use super::compute_allow_paths; use codex_protocol::protocol::SandboxPolicy; use std::collections::HashMap; + use std::collections::HashSet; use std::fs; use std::path::PathBuf; + use tempfile::TempDir; #[test] fn includes_additional_writable_roots() { - let command_cwd = PathBuf::from(r"C:\Workspace"); - let extra_root = PathBuf::from(r"C:\AdditionalWritableRoot"); + let tmp = TempDir::new().expect("tempdir"); + let command_cwd = tmp.path().join("workspace"); + let extra_root = tmp.path().join("extra"); let _ = fs::create_dir_all(&command_cwd); let _ = fs::create_dir_all(&extra_root); @@ -76,22 +115,22 @@ mod tests { exclude_slash_tmp: false, }; - let allow = compute_allow_paths(&policy, &command_cwd, &command_cwd, &HashMap::new()); + let paths = compute_allow_paths(&policy, &command_cwd, &command_cwd, &HashMap::new()); - assert!( - allow.iter().any(|p| p == &command_cwd), - "command cwd should be allowed" - ); - assert!( - allow.iter().any(|p| p == &extra_root), - "additional writable root should be allowed" - ); + assert!(paths + .allow + .contains(&dunce::canonicalize(&command_cwd).unwrap())); + assert!(paths + .allow + .contains(&dunce::canonicalize(&extra_root).unwrap())); + assert!(paths.deny.is_empty(), "no deny paths expected"); } #[test] fn excludes_tmp_env_vars_when_requested() { - let command_cwd = PathBuf::from(r"C:\Workspace"); - let temp_dir = PathBuf::from(r"C:\TempDir"); + let tmp = TempDir::new().expect("tempdir"); + let command_cwd = tmp.path().join("workspace"); + let temp_dir = tmp.path().join("temp"); let _ = fs::create_dir_all(&command_cwd); let _ = fs::create_dir_all(&temp_dir); @@ -104,15 +143,58 @@ mod tests { let mut env_map = HashMap::new(); env_map.insert("TEMP".into(), temp_dir.to_string_lossy().to_string()); - let allow = compute_allow_paths(&policy, &command_cwd, &command_cwd, &env_map); + let paths = compute_allow_paths(&policy, &command_cwd, &command_cwd, &env_map); - assert!( - allow.iter().any(|p| p == &command_cwd), - "command cwd should be allowed" - ); - assert!( - !allow.iter().any(|p| p == &temp_dir), - "TEMP should be excluded when exclude_tmpdir_env_var is true" - ); + assert!(paths + .allow + .contains(&dunce::canonicalize(&command_cwd).unwrap())); + assert!(!paths + .allow + .contains(&dunce::canonicalize(&temp_dir).unwrap())); + assert!(paths.deny.is_empty(), "no deny paths expected"); + } + + #[test] + fn denies_git_dir_inside_writable_root() { + let tmp = TempDir::new().expect("tempdir"); + let command_cwd = tmp.path().join("workspace"); + let git_dir = command_cwd.join(".git"); + let _ = fs::create_dir_all(&git_dir); + + let policy = SandboxPolicy::WorkspaceWrite { + writable_roots: vec![], + network_access: false, + exclude_tmpdir_env_var: true, + exclude_slash_tmp: false, + }; + + let paths = compute_allow_paths(&policy, &command_cwd, &command_cwd, &HashMap::new()); + let expected_allow: HashSet = [dunce::canonicalize(&command_cwd).unwrap()] + .into_iter() + .collect(); + let expected_deny: HashSet = [dunce::canonicalize(&git_dir).unwrap()] + .into_iter() + .collect(); + + assert_eq!(expected_allow, paths.allow); + assert_eq!(expected_deny, paths.deny); + } + + #[test] + fn skips_git_dir_when_missing() { + let tmp = TempDir::new().expect("tempdir"); + let command_cwd = tmp.path().join("workspace"); + let _ = fs::create_dir_all(&command_cwd); + + let policy = SandboxPolicy::WorkspaceWrite { + writable_roots: vec![], + network_access: false, + exclude_tmpdir_env_var: true, + exclude_slash_tmp: false, + }; + + let paths = compute_allow_paths(&policy, &command_cwd, &command_cwd, &HashMap::new()); + assert_eq!(paths.allow.len(), 1); + assert!(paths.deny.is_empty(), "no deny when .git is absent"); } } diff --git a/codex-rs/windows-sandbox-rs/src/lib.rs b/codex-rs/windows-sandbox-rs/src/lib.rs index 2d591fced0..45595052ab 100644 --- a/codex-rs/windows-sandbox-rs/src/lib.rs +++ b/codex-rs/windows-sandbox-rs/src/lib.rs @@ -23,9 +23,11 @@ pub use stub::CaptureResult; #[cfg(target_os = "windows")] mod windows_impl { use super::acl::add_allow_ace; + use super::acl::add_deny_write_ace; use super::acl::allow_null_device; use super::acl::revoke_ace; use super::allow::compute_allow_paths; + use super::allow::AllowDenyPaths; use super::cap::cap_sid_file; use super::cap::load_or_create_cap_sids; use super::env::apply_no_network_to_env; @@ -195,8 +197,6 @@ mod windows_impl { ensure_codex_home_exists(codex_home)?; let current_dir = cwd.to_path_buf(); - // for now, don't fail if we detect world-writable directories - // audit::audit_everyone_writable(¤t_dir, &env_map)?; let logs_base_dir = Some(codex_home); log_start(&command, logs_base_dir); let cap_sid_path = cap_sid_file(codex_home); @@ -238,7 +238,8 @@ mod windows_impl { } let persist_aces = is_workspace_write; - let allow = compute_allow_paths(&policy, sandbox_policy_cwd, ¤t_dir, &env_map); + let AllowDenyPaths { allow, deny } = + compute_allow_paths(&policy, sandbox_policy_cwd, ¤t_dir, &env_map); let mut guards: Vec<(PathBuf, *mut c_void)> = Vec::new(); unsafe { for p in &allow { @@ -254,6 +255,13 @@ mod windows_impl { } } } + for p in &deny { + if let Ok(added) = add_deny_write_ace(p, psid_to_use) { + if added && !persist_aces { + guards.push((p.clone(), psid_to_use)); + } + } + } allow_null_device(psid_to_use); } From b8978803782045a1d30ec29c0cbb1fe8b741bcb6 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Mon, 24 Nov 2025 21:45:40 +0100 Subject: [PATCH 23/61] chore: dedup unified exec "waited" rendering (#7256) From Screenshot 2025-11-24 at 18 02 25 To Screenshot 2025-11-24 at 18 02 40 --- codex-rs/tui/src/chatwidget.rs | 49 ++++++++++++++++++++++++++++ codex-rs/tui/src/chatwidget/tests.rs | 3 ++ 2 files changed, 52 insertions(+) diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index ddba41443d..980bdf7f4a 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -1,4 +1,5 @@ use std::collections::HashMap; +use std::collections::HashSet; use std::collections::VecDeque; use std::path::PathBuf; use std::sync::Arc; @@ -140,6 +141,20 @@ struct RunningCommand { source: ExecCommandSource, } +struct UnifiedExecWaitState { + command_display: String, +} + +impl UnifiedExecWaitState { + fn new(command_display: String) -> Self { + Self { command_display } + } + + fn is_duplicate(&self, command_display: &str) -> bool { + self.command_display == command_display + } +} + const RATE_LIMIT_WARNING_THRESHOLDS: [f64; 3] = [75.0, 90.0, 95.0]; const NUDGE_MODEL_SLUG: &str = "gpt-5.1-codex-mini"; const RATE_LIMIT_SWITCH_PROMPT_THRESHOLD: f64 = 90.0; @@ -266,6 +281,8 @@ pub(crate) struct ChatWidget { // Stream lifecycle controller stream_controller: Option, running_commands: HashMap, + suppressed_exec_calls: HashSet, + last_unified_wait: Option, task_complete_pending: bool, mcp_startup_status: Option>, // Queue of interruptive UI events deferred during an active write cycle @@ -479,6 +496,8 @@ impl ChatWidget { // Mark task stopped and request redraw now that all content is in history. self.bottom_pane.set_task_running(false); self.running_commands.clear(); + self.suppressed_exec_calls.clear(); + self.last_unified_wait = None; self.request_redraw(); // If there is a queued user message, send exactly one now to begin the next turn. @@ -588,6 +607,8 @@ impl ChatWidget { // Reset running state and clear streaming buffers. self.bottom_pane.set_task_running(false); self.running_commands.clear(); + self.suppressed_exec_calls.clear(); + self.last_unified_wait = None; self.stream_controller = None; self.maybe_show_pending_rate_limit_prompt(); } @@ -947,6 +968,9 @@ impl ChatWidget { pub(crate) fn handle_exec_end_now(&mut self, ev: ExecCommandEndEvent) { let running = self.running_commands.remove(&ev.call_id); + if self.suppressed_exec_calls.remove(&ev.call_id) { + return; + } let (command, parsed, source) = match running { Some(rc) => (rc.command, rc.parsed_cmd, rc.source), None => ( @@ -1074,6 +1098,27 @@ impl ChatWidget { source: ev.source, }, ); + let is_wait_interaction = matches!(ev.source, ExecCommandSource::UnifiedExecInteraction) + && ev + .interaction_input + .as_deref() + .map(str::is_empty) + .unwrap_or(true); + let command_display = ev.command.join(" "); + let should_suppress_unified_wait = is_wait_interaction + && self + .last_unified_wait + .as_ref() + .is_some_and(|wait| wait.is_duplicate(&command_display)); + if is_wait_interaction { + self.last_unified_wait = Some(UnifiedExecWaitState::new(command_display)); + } else { + self.last_unified_wait = None; + } + if should_suppress_unified_wait { + self.suppressed_exec_calls.insert(ev.call_id); + return; + } let interaction_input = ev.interaction_input.clone(); if let Some(cell) = self .active_cell @@ -1195,6 +1240,8 @@ impl ChatWidget { rate_limit_poller: None, stream_controller: None, running_commands: HashMap::new(), + suppressed_exec_calls: HashSet::new(), + last_unified_wait: None, task_complete_pending: false, mcp_startup_status: None, interrupts: InterruptManager::new(), @@ -1270,6 +1317,8 @@ impl ChatWidget { rate_limit_poller: None, stream_controller: None, running_commands: HashMap::new(), + suppressed_exec_calls: HashSet::new(), + last_unified_wait: None, task_complete_pending: false, mcp_startup_status: None, interrupts: InterruptManager::new(), diff --git a/codex-rs/tui/src/chatwidget/tests.rs b/codex-rs/tui/src/chatwidget/tests.rs index 61e23fb3fc..7b044e8dfb 100644 --- a/codex-rs/tui/src/chatwidget/tests.rs +++ b/codex-rs/tui/src/chatwidget/tests.rs @@ -56,6 +56,7 @@ use crossterm::event::KeyEvent; use crossterm::event::KeyModifiers; use insta::assert_snapshot; use pretty_assertions::assert_eq; +use std::collections::HashSet; use std::path::PathBuf; use tempfile::NamedTempFile; use tempfile::tempdir; @@ -358,6 +359,8 @@ fn make_chatwidget_manual() -> ( rate_limit_poller: None, stream_controller: None, running_commands: HashMap::new(), + suppressed_exec_calls: HashSet::new(), + last_unified_wait: None, task_complete_pending: false, mcp_startup_status: None, interrupts: InterruptManager::new(), From fc2ff624ac5ce24e26a01ca8e46d7e711c20f2b0 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Mon, 24 Nov 2025 21:14:24 +0000 Subject: [PATCH 24/61] fix: don't store early exit sessions (#7263) --- .../core/src/unified_exec/session_manager.rs | 26 +++++++++---------- codex-rs/core/tests/suite/unified_exec.rs | 10 +++---- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/codex-rs/core/src/unified_exec/session_manager.rs b/codex-rs/core/src/unified_exec/session_manager.rs index a5cc174776..ed489e18cd 100644 --- a/codex-rs/core/src/unified_exec/session_manager.rs +++ b/codex-rs/core/src/unified_exec/session_manager.rs @@ -101,20 +101,18 @@ impl UnifiedExecSessionManager { let text = String::from_utf8_lossy(&collected).to_string(); let output = formatted_truncate_text(&text, TruncationPolicy::Tokens(max_tokens)); - let chunk_id = generate_chunk_id(); let has_exited = session.has_exited(); - let stored_id = self - .store_session(session, context, &request.command, cwd.clone(), start) - .await; - let exit_code = self - .sessions - .lock() - .await - .get(&stored_id) - .map(|entry| entry.session.exit_code()); - // Only include a session_id in the response if the process is still alive. - let session_id = if has_exited { None } else { Some(stored_id) }; - + let exit_code = session.exit_code(); + let chunk_id = generate_chunk_id(); + let session_id = if has_exited { + None + } else { + // Only store session if not exited. + let stored_id = self + .store_session(session, context, &request.command, cwd.clone(), start) + .await; + Some(stored_id) + }; let original_token_count = approx_token_count(&text); let response = UnifiedExecResponse { @@ -123,7 +121,7 @@ impl UnifiedExecSessionManager { wall_time, output, session_id, - exit_code: exit_code.flatten(), + exit_code, original_token_count: Some(original_token_count), session_command: Some(request.command.clone()), }; diff --git a/codex-rs/core/tests/suite/unified_exec.rs b/codex-rs/core/tests/suite/unified_exec.rs index 216d1eea52..404801658c 100644 --- a/codex-rs/core/tests/suite/unified_exec.rs +++ b/codex-rs/core/tests/suite/unified_exec.rs @@ -585,7 +585,7 @@ async fn unified_exec_emits_begin_for_write_stdin() -> Result<()> { let open_call_id = "uexec-open-for-begin"; let open_args = json!({ - "cmd": "/bin/sh -c echo ready".to_string(), + "cmd": "bash -i".to_string(), "yield_time_ms": 200, }); @@ -651,7 +651,7 @@ async fn unified_exec_emits_begin_for_write_stdin() -> Result<()> { vec![ "/bin/bash".to_string(), "-lc".to_string(), - "/bin/sh -c echo ready".to_string() + "bash -i".to_string() ] ); assert_eq!( @@ -687,7 +687,7 @@ async fn unified_exec_emits_begin_event_for_write_stdin_requests() -> Result<()> let open_call_id = "uexec-open-session"; let open_args = json!({ - "cmd": "/bin/sh -c echo ready".to_string(), + "cmd": "bash -i".to_string(), "yield_time_ms": 250, }); @@ -767,7 +767,7 @@ async fn unified_exec_emits_begin_event_for_write_stdin_requests() -> Result<()> vec![ "/bin/bash".to_string(), "-lc".to_string(), - "/bin/sh -c echo ready".to_string() + "bash -i".to_string() ] ); assert!( @@ -785,7 +785,7 @@ async fn unified_exec_emits_begin_event_for_write_stdin_requests() -> Result<()> vec![ "/bin/bash".to_string(), "-lc".to_string(), - "/bin/sh -c echo ready".to_string() + "bash -i".to_string() ] ); assert!( From 3308dc5e4831e57de11eeb8e7f196ea5ff1a4281 Mon Sep 17 00:00:00 2001 From: Clifford Ressel Date: Mon, 24 Nov 2025 23:16:29 -0500 Subject: [PATCH 25/61] fix: Correct the stream error message (#7266) Fixes a copy paste bug with the error handling in `try_run_turn` I have read the CLA Document and I hereby sign the CLA --- codex-rs/core/src/codex.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 82e14178eb..799374d6d6 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -2342,7 +2342,7 @@ async fn try_run_turn( sess.send_event(&turn_context, EventMsg::AgentMessageContentDelta(event)) .await; } else { - error_or_panic("ReasoningSummaryDelta without active item".to_string()); + error_or_panic("OutputTextDelta without active item".to_string()); } } ResponseEvent::ReasoningSummaryDelta { From 0dd822264a9265749d4f698674553a8d8f747ede Mon Sep 17 00:00:00 2001 From: Celia Chen Date: Tue, 25 Nov 2025 00:04:27 -0800 Subject: [PATCH 26/61] [app-server-test-client] add send-followup-v2 (#7271) Add a new endpoint that allows us to test multi-turn behavior. Tested with running: ``` RUST_LOG=codex_app_server=debug CODEX_BIN=target/debug/codex \ cargo run -p codex-app-server-test-client -- \ send-follow-up-v2 "hello" "and now a follow-up question" ``` --- codex-rs/app-server-test-client/src/main.rs | 51 +++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/codex-rs/app-server-test-client/src/main.rs b/codex-rs/app-server-test-client/src/main.rs index 7f2b75bd21..f75f3f98a8 100644 --- a/codex-rs/app-server-test-client/src/main.rs +++ b/codex-rs/app-server-test-client/src/main.rs @@ -101,6 +101,15 @@ enum CliCommand { /// Start a V2 turn that should not elicit an ExecCommand approval. #[command(name = "no-trigger-cmd-approval")] NoTriggerCmdApproval, + /// Send two sequential V2 turns in the same thread to test follow-up behavior. + SendFollowUpV2 { + /// Initial user message for the first turn. + #[arg()] + first_message: String, + /// Follow-up user message for the second turn. + #[arg()] + follow_up_message: String, + }, /// Trigger the ChatGPT login flow and wait for completion. TestLogin, /// Fetch the current account rate limits from the Codex app-server. @@ -120,6 +129,10 @@ fn main() -> Result<()> { trigger_patch_approval(codex_bin, user_message) } CliCommand::NoTriggerCmdApproval => no_trigger_cmd_approval(codex_bin), + CliCommand::SendFollowUpV2 { + first_message, + follow_up_message, + } => send_follow_up_v2(codex_bin, first_message, follow_up_message), CliCommand::TestLogin => test_login(codex_bin), CliCommand::GetAccountRateLimits => get_account_rate_limits(codex_bin), } @@ -209,6 +222,44 @@ fn send_message_v2_with_policies( Ok(()) } +fn send_follow_up_v2( + codex_bin: String, + first_message: String, + follow_up_message: String, +) -> Result<()> { + let mut client = CodexClient::spawn(codex_bin)?; + + let initialize = client.initialize()?; + println!("< initialize response: {initialize:?}"); + + let thread_response = client.thread_start(ThreadStartParams::default())?; + println!("< thread/start response: {thread_response:?}"); + + let first_turn_params = TurnStartParams { + thread_id: thread_response.thread.id.clone(), + input: vec![V2UserInput::Text { + text: first_message, + }], + ..Default::default() + }; + let first_turn_response = client.turn_start(first_turn_params)?; + println!("< turn/start response (initial): {first_turn_response:?}"); + client.stream_turn(&thread_response.thread.id, &first_turn_response.turn.id)?; + + let follow_up_params = TurnStartParams { + thread_id: thread_response.thread.id.clone(), + input: vec![V2UserInput::Text { + text: follow_up_message, + }], + ..Default::default() + }; + let follow_up_response = client.turn_start(follow_up_params)?; + println!("< turn/start response (follow-up): {follow_up_response:?}"); + client.stream_turn(&thread_response.thread.id, &follow_up_response.turn.id)?; + + Ok(()) +} + fn test_login(codex_bin: String) -> Result<()> { let mut client = CodexClient::spawn(codex_bin)?; From 523b40a129fdb8bac46b25b09276ccb1d7cb63fd Mon Sep 17 00:00:00 2001 From: jif-oai Date: Tue, 25 Nov 2025 09:29:38 +0000 Subject: [PATCH 27/61] feat[app-serve]: config management (#7241) --- codex-rs/Cargo.lock | 1 + .../src/protocol/common.rs | 13 + .../app-server-protocol/src/protocol/v2.rs | 121 +++ codex-rs/app-server/Cargo.toml | 5 +- .../app-server/src/codex_message_processor.rs | 5 + codex-rs/app-server/src/config_api.rs | 974 ++++++++++++++++++ codex-rs/app-server/src/lib.rs | 15 +- codex-rs/app-server/src/message_processor.rs | 60 +- .../app-server/tests/common/mcp_process.rs | 27 + .../app-server/tests/suite/v2/config_rpc.rs | 347 +++++++ codex-rs/app-server/tests/suite/v2/mod.rs | 1 + codex-rs/core/src/config/mod.rs | 2 +- codex-rs/core/src/config_loader/mod.rs | 20 +- 13 files changed, 1572 insertions(+), 19 deletions(-) create mode 100644 codex-rs/app-server/src/config_api.rs create mode 100644 codex-rs/app-server/tests/suite/v2/config_rpc.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 350561ba5d..800ef1ce73 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -870,6 +870,7 @@ dependencies = [ "serde", "serde_json", "serial_test", + "sha2", "shlex", "tempfile", "tokio", diff --git a/codex-rs/app-server-protocol/src/protocol/common.rs b/codex-rs/app-server-protocol/src/protocol/common.rs index fecdc5b71b..721ffe2555 100644 --- a/codex-rs/app-server-protocol/src/protocol/common.rs +++ b/codex-rs/app-server-protocol/src/protocol/common.rs @@ -164,6 +164,19 @@ client_request_definitions! { response: v2::FeedbackUploadResponse, }, + ConfigRead => "config/read" { + params: v2::ConfigReadParams, + response: v2::ConfigReadResponse, + }, + ConfigValueWrite => "config/value/write" { + params: v2::ConfigValueWriteParams, + response: v2::ConfigWriteResponse, + }, + ConfigBatchWrite => "config/batchWrite" { + params: v2::ConfigBatchWriteParams, + response: v2::ConfigWriteResponse, + }, + GetAccount => "account/read" { params: v2::GetAccountParams, response: v2::GetAccountResponse, diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index 13b7b8888c..71249b667f 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -128,6 +128,127 @@ v2_enum_from_core!( } ); +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub enum ConfigLayerName { + Mdm, + System, + SessionFlags, + User, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ConfigLayerMetadata { + pub name: ConfigLayerName, + pub source: String, + pub version: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ConfigLayer { + pub name: ConfigLayerName, + pub source: String, + pub version: String, + pub config: JsonValue, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub enum MergeStrategy { + Replace, + Upsert, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub enum WriteStatus { + Ok, + OkOverridden, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct OverriddenMetadata { + pub message: String, + pub overriding_layer: ConfigLayerMetadata, + pub effective_value: JsonValue, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ConfigWriteResponse { + pub status: WriteStatus, + pub version: String, + pub overridden_metadata: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub enum ConfigWriteErrorCode { + ConfigLayerReadonly, + ConfigVersionConflict, + ConfigValidationError, + ConfigPathNotFound, + ConfigSchemaUnknownKey, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ConfigReadParams { + #[serde(default)] + pub include_layers: bool, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ConfigReadResponse { + pub config: JsonValue, + pub origins: HashMap, + #[serde(skip_serializing_if = "Option::is_none")] + pub layers: Option>, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ConfigValueWriteParams { + pub file_path: String, + pub key_path: String, + pub value: JsonValue, + pub merge_strategy: MergeStrategy, + pub expected_version: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ConfigBatchWriteParams { + pub file_path: String, + pub edits: Vec, + pub expected_version: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ConfigEdit { + pub key_path: String, + pub value: JsonValue, + pub merge_strategy: MergeStrategy, +} + v2_enum_from_core!( pub enum CommandRiskLevel from codex_protocol::approvals::SandboxRiskLevel { Low, diff --git a/codex-rs/app-server/Cargo.toml b/codex-rs/app-server/Cargo.toml index 3189199a9f..4ffe2d8913 100644 --- a/codex-rs/app-server/Cargo.toml +++ b/codex-rs/app-server/Cargo.toml @@ -30,6 +30,9 @@ codex-utils-json-to-toml = { workspace = true } chrono = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } +sha2 = { workspace = true } +tempfile = { workspace = true } +toml = { workspace = true } tokio = { workspace = true, features = [ "io-std", "macros", @@ -51,7 +54,5 @@ mcp-types = { workspace = true } os_info = { workspace = true } pretty_assertions = { workspace = true } serial_test = { workspace = true } -tempfile = { workspace = true } -toml = { workspace = true } wiremock = { workspace = true } shlex = { workspace = true } diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index a9f56de115..9dc2f2e016 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -472,6 +472,11 @@ impl CodexMessageProcessor { ClientRequest::ExecOneOffCommand { request_id, params } => { self.exec_one_off_command(request_id, params).await; } + ClientRequest::ConfigRead { .. } + | ClientRequest::ConfigValueWrite { .. } + | ClientRequest::ConfigBatchWrite { .. } => { + warn!("Config request reached CodexMessageProcessor unexpectedly"); + } ClientRequest::GetAccountRateLimits { request_id, params: _, diff --git a/codex-rs/app-server/src/config_api.rs b/codex-rs/app-server/src/config_api.rs new file mode 100644 index 0000000000..68bbdd8c66 --- /dev/null +++ b/codex-rs/app-server/src/config_api.rs @@ -0,0 +1,974 @@ +use crate::error_code::INTERNAL_ERROR_CODE; +use crate::error_code::INVALID_REQUEST_ERROR_CODE; +use anyhow::anyhow; +use codex_app_server_protocol::ConfigBatchWriteParams; +use codex_app_server_protocol::ConfigLayer; +use codex_app_server_protocol::ConfigLayerMetadata; +use codex_app_server_protocol::ConfigLayerName; +use codex_app_server_protocol::ConfigReadParams; +use codex_app_server_protocol::ConfigReadResponse; +use codex_app_server_protocol::ConfigValueWriteParams; +use codex_app_server_protocol::ConfigWriteErrorCode; +use codex_app_server_protocol::ConfigWriteResponse; +use codex_app_server_protocol::JSONRPCErrorError; +use codex_app_server_protocol::MergeStrategy; +use codex_app_server_protocol::OverriddenMetadata; +use codex_app_server_protocol::WriteStatus; +use codex_core::config::ConfigToml; +use codex_core::config_loader::LoadedConfigLayers; +use codex_core::config_loader::LoaderOverrides; +use codex_core::config_loader::load_config_layers_with_overrides; +use codex_core::config_loader::merge_toml_values; +use serde_json::Value as JsonValue; +use serde_json::json; +use sha2::Digest; +use sha2::Sha256; +use std::collections::HashMap; +use std::path::Path; +use std::path::PathBuf; +use tempfile::NamedTempFile; +use tokio::task; +use toml::Value as TomlValue; + +const SESSION_FLAGS_SOURCE: &str = "--config"; +const MDM_SOURCE: &str = "com.openai.codex/config_toml_base64"; +const CONFIG_FILE_NAME: &str = "config.toml"; + +#[derive(Clone)] +pub(crate) struct ConfigApi { + codex_home: PathBuf, + cli_overrides: Vec<(String, TomlValue)>, + loader_overrides: LoaderOverrides, +} + +impl ConfigApi { + pub(crate) fn new(codex_home: PathBuf, cli_overrides: Vec<(String, TomlValue)>) -> Self { + Self { + codex_home, + cli_overrides, + loader_overrides: LoaderOverrides::default(), + } + } + + #[cfg(test)] + fn with_overrides( + codex_home: PathBuf, + cli_overrides: Vec<(String, TomlValue)>, + loader_overrides: LoaderOverrides, + ) -> Self { + Self { + codex_home, + cli_overrides, + loader_overrides, + } + } + + pub(crate) async fn read( + &self, + params: ConfigReadParams, + ) -> Result { + let layers = self + .load_layers_state() + .await + .map_err(|err| internal_error("failed to read configuration layers", err))?; + + let effective = layers.effective_config(); + validate_config(&effective).map_err(|err| internal_error("invalid configuration", err))?; + + let response = ConfigReadResponse { + config: to_json_value(&effective), + origins: layers.origins(), + layers: params.include_layers.then(|| layers.layers_high_to_low()), + }; + + Ok(response) + } + + pub(crate) async fn write_value( + &self, + params: ConfigValueWriteParams, + ) -> Result { + let edits = vec![(params.key_path, params.value, params.merge_strategy)]; + self.apply_edits(params.file_path, params.expected_version, edits) + .await + } + + pub(crate) async fn batch_write( + &self, + params: ConfigBatchWriteParams, + ) -> Result { + let edits = params + .edits + .into_iter() + .map(|edit| (edit.key_path, edit.value, edit.merge_strategy)) + .collect(); + + self.apply_edits(params.file_path, params.expected_version, edits) + .await + } + + async fn apply_edits( + &self, + file_path: String, + expected_version: Option, + edits: Vec<(String, JsonValue, MergeStrategy)>, + ) -> Result { + let allowed_path = self.codex_home.join(CONFIG_FILE_NAME); + if !paths_match(&allowed_path, &file_path) { + return Err(config_write_error( + ConfigWriteErrorCode::ConfigLayerReadonly, + "Only writes to the user config are allowed", + )); + } + + let layers = self + .load_layers_state() + .await + .map_err(|err| internal_error("failed to load configuration", err))?; + + if let Some(expected) = expected_version.as_deref() + && expected != layers.user.version + { + return Err(config_write_error( + ConfigWriteErrorCode::ConfigVersionConflict, + "Configuration was modified since last read. Fetch latest version and retry.", + )); + } + + let mut user_config = layers.user.config.clone(); + let mut mutated = false; + let mut parsed_segments = Vec::new(); + + for (key_path, value, strategy) in edits.into_iter() { + let segments = parse_key_path(&key_path).map_err(|message| { + config_write_error(ConfigWriteErrorCode::ConfigValidationError, message) + })?; + let parsed_value = parse_value(value).map_err(|message| { + config_write_error(ConfigWriteErrorCode::ConfigValidationError, message) + })?; + + let changed = apply_merge(&mut user_config, &segments, parsed_value.as_ref(), strategy) + .map_err(|err| match err { + MergeError::PathNotFound => config_write_error( + ConfigWriteErrorCode::ConfigPathNotFound, + "Path not found", + ), + MergeError::Validation(message) => { + config_write_error(ConfigWriteErrorCode::ConfigValidationError, message) + } + })?; + + mutated |= changed; + parsed_segments.push(segments); + } + + validate_config(&user_config).map_err(|err| { + config_write_error( + ConfigWriteErrorCode::ConfigValidationError, + format!("Invalid configuration: {err}"), + ) + })?; + + let updated_layers = layers.with_user_config(user_config.clone()); + let effective = updated_layers.effective_config(); + validate_config(&effective).map_err(|err| { + config_write_error( + ConfigWriteErrorCode::ConfigValidationError, + format!("Invalid configuration: {err}"), + ) + })?; + + if mutated { + self.persist_user_config(&user_config) + .await + .map_err(|err| internal_error("failed to persist config.toml", err))?; + } + + let overridden = first_overridden_edit(&updated_layers, &effective, &parsed_segments); + let status = overridden + .as_ref() + .map(|_| WriteStatus::OkOverridden) + .unwrap_or(WriteStatus::Ok); + + Ok(ConfigWriteResponse { + status, + version: updated_layers.user.version.clone(), + overridden_metadata: overridden, + }) + } + + async fn load_layers_state(&self) -> std::io::Result { + let LoadedConfigLayers { + base, + managed_config, + managed_preferences, + } = load_config_layers_with_overrides(&self.codex_home, self.loader_overrides.clone()) + .await?; + + let user = LayerState::new( + ConfigLayerName::User, + self.codex_home.join(CONFIG_FILE_NAME), + base, + ); + + let session_flags = LayerState::new( + ConfigLayerName::SessionFlags, + PathBuf::from(SESSION_FLAGS_SOURCE), + { + let mut root = TomlValue::Table(toml::map::Map::new()); + for (path, value) in self.cli_overrides.iter() { + apply_override(&mut root, path, value.clone()); + } + root + }, + ); + + let system = managed_config.map(|cfg| { + LayerState::new( + ConfigLayerName::System, + system_config_path(&self.codex_home), + cfg, + ) + }); + + let mdm = managed_preferences + .map(|cfg| LayerState::new(ConfigLayerName::Mdm, PathBuf::from(MDM_SOURCE), cfg)); + + Ok(LayersState { + user, + session_flags, + system, + mdm, + }) + } + + async fn persist_user_config(&self, user_config: &TomlValue) -> anyhow::Result<()> { + let codex_home = self.codex_home.clone(); + let serialized = toml::to_string_pretty(user_config)?; + + task::spawn_blocking(move || -> anyhow::Result<()> { + std::fs::create_dir_all(&codex_home)?; + + let target = codex_home.join(CONFIG_FILE_NAME); + let tmp = NamedTempFile::new_in(&codex_home)?; + std::fs::write(tmp.path(), serialized.as_bytes())?; + tmp.persist(&target)?; + Ok(()) + }) + .await + .map_err(|err| anyhow!("config persistence task panicked: {err}"))??; + + Ok(()) + } +} + +fn parse_value(value: JsonValue) -> Result, String> { + if value.is_null() { + return Ok(None); + } + + serde_json::from_value::(value) + .map(Some) + .map_err(|err| format!("invalid value: {err}")) +} + +fn parse_key_path(path: &str) -> Result, String> { + if path.trim().is_empty() { + return Err("keyPath must not be empty".to_string()); + } + Ok(path + .split('.') + .map(std::string::ToString::to_string) + .collect()) +} + +fn apply_override(target: &mut TomlValue, path: &str, value: TomlValue) { + use toml::value::Table; + + let segments: Vec<&str> = path.split('.').collect(); + let mut current = target; + + for (idx, segment) in segments.iter().enumerate() { + let is_last = idx == segments.len() - 1; + + if is_last { + match current { + TomlValue::Table(table) => { + table.insert(segment.to_string(), value); + } + _ => { + let mut table = Table::new(); + table.insert(segment.to_string(), value); + *current = TomlValue::Table(table); + } + } + return; + } + + match current { + TomlValue::Table(table) => { + current = table + .entry((*segment).to_string()) + .or_insert_with(|| TomlValue::Table(Table::new())); + } + _ => { + *current = TomlValue::Table(Table::new()); + if let TomlValue::Table(tbl) = current { + current = tbl + .entry((*segment).to_string()) + .or_insert_with(|| TomlValue::Table(Table::new())); + } + } + } + } +} + +#[derive(Debug)] +enum MergeError { + PathNotFound, + Validation(String), +} + +fn apply_merge( + root: &mut TomlValue, + segments: &[String], + value: Option<&TomlValue>, + strategy: MergeStrategy, +) -> Result { + let Some(value) = value else { + return clear_path(root, segments); + }; + + let Some((last, parents)) = segments.split_last() else { + return Err(MergeError::Validation( + "keyPath must not be empty".to_string(), + )); + }; + + let mut current = root; + + for segment in parents { + match current { + TomlValue::Table(table) => { + current = table + .entry(segment.clone()) + .or_insert_with(|| TomlValue::Table(toml::map::Map::new())); + } + _ => { + *current = TomlValue::Table(toml::map::Map::new()); + if let TomlValue::Table(table) = current { + current = table + .entry(segment.clone()) + .or_insert_with(|| TomlValue::Table(toml::map::Map::new())); + } + } + } + } + + let table = current.as_table_mut().ok_or_else(|| { + MergeError::Validation("cannot set value on non-table parent".to_string()) + })?; + + if matches!(strategy, MergeStrategy::Upsert) + && let Some(existing) = table.get_mut(last) + && matches!(existing, TomlValue::Table(_)) + && matches!(value, TomlValue::Table(_)) + { + merge_toml_values(existing, value); + return Ok(true); + } + + let changed = table + .get(last) + .map(|existing| Some(existing) != Some(value)) + .unwrap_or(true); + table.insert(last.clone(), value.clone()); + Ok(changed) +} + +fn clear_path(root: &mut TomlValue, segments: &[String]) -> Result { + let Some((last, parents)) = segments.split_last() else { + return Err(MergeError::Validation( + "keyPath must not be empty".to_string(), + )); + }; + + let mut current = root; + for segment in parents { + match current { + TomlValue::Table(table) => { + current = table.get_mut(segment).ok_or(MergeError::PathNotFound)?; + } + _ => return Err(MergeError::PathNotFound), + } + } + + let Some(parent) = current.as_table_mut() else { + return Err(MergeError::PathNotFound); + }; + + Ok(parent.remove(last).is_some()) +} + +#[derive(Clone)] +struct LayerState { + name: ConfigLayerName, + source: PathBuf, + config: TomlValue, + version: String, +} + +impl LayerState { + fn new(name: ConfigLayerName, source: PathBuf, config: TomlValue) -> Self { + let version = version_for_toml(&config); + Self { + name, + source, + config, + version, + } + } + + fn metadata(&self) -> ConfigLayerMetadata { + ConfigLayerMetadata { + name: self.name.clone(), + source: self.source.display().to_string(), + version: self.version.clone(), + } + } + + fn as_layer(&self) -> ConfigLayer { + ConfigLayer { + name: self.name.clone(), + source: self.source.display().to_string(), + version: self.version.clone(), + config: to_json_value(&self.config), + } + } +} + +#[derive(Clone)] +struct LayersState { + user: LayerState, + session_flags: LayerState, + system: Option, + mdm: Option, +} + +impl LayersState { + fn with_user_config(self, user_config: TomlValue) -> Self { + Self { + user: LayerState::new(self.user.name, self.user.source, user_config), + session_flags: self.session_flags, + system: self.system, + mdm: self.mdm, + } + } + + fn effective_config(&self) -> TomlValue { + let mut merged = self.user.config.clone(); + merge_toml_values(&mut merged, &self.session_flags.config); + if let Some(system) = &self.system { + merge_toml_values(&mut merged, &system.config); + } + if let Some(mdm) = &self.mdm { + merge_toml_values(&mut merged, &mdm.config); + } + merged + } + + fn origins(&self) -> HashMap { + let mut origins = HashMap::new(); + let mut path = Vec::new(); + + record_origins( + &self.user.config, + &self.user.metadata(), + &mut path, + &mut origins, + ); + record_origins( + &self.session_flags.config, + &self.session_flags.metadata(), + &mut path, + &mut origins, + ); + if let Some(system) = &self.system { + record_origins(&system.config, &system.metadata(), &mut path, &mut origins); + } + if let Some(mdm) = &self.mdm { + record_origins(&mdm.config, &mdm.metadata(), &mut path, &mut origins); + } + + origins + } + + fn layers_high_to_low(&self) -> Vec { + let mut layers = Vec::new(); + if let Some(mdm) = &self.mdm { + layers.push(mdm.as_layer()); + } + if let Some(system) = &self.system { + layers.push(system.as_layer()); + } + layers.push(self.session_flags.as_layer()); + layers.push(self.user.as_layer()); + layers + } +} + +fn record_origins( + value: &TomlValue, + meta: &ConfigLayerMetadata, + path: &mut Vec, + origins: &mut HashMap, +) { + match value { + TomlValue::Table(table) => { + for (key, val) in table { + path.push(key.clone()); + record_origins(val, meta, path, origins); + path.pop(); + } + } + TomlValue::Array(items) => { + for (idx, item) in items.iter().enumerate() { + path.push(idx.to_string()); + record_origins(item, meta, path, origins); + path.pop(); + } + } + _ => { + if !path.is_empty() { + origins.insert(path.join("."), meta.clone()); + } + } + } +} + +fn to_json_value(value: &TomlValue) -> JsonValue { + serde_json::to_value(value).unwrap_or(JsonValue::Null) +} + +fn validate_config(value: &TomlValue) -> Result<(), toml::de::Error> { + let _: ConfigToml = value.clone().try_into()?; + Ok(()) +} + +fn version_for_toml(value: &TomlValue) -> String { + let json = to_json_value(value); + let canonical = canonical_json(&json); + let serialized = serde_json::to_vec(&canonical).unwrap_or_default(); + let mut hasher = Sha256::new(); + hasher.update(serialized); + let hash = hasher.finalize(); + let hex = hash + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::(); + format!("sha256:{hex}") +} + +fn canonical_json(value: &JsonValue) -> JsonValue { + match value { + JsonValue::Object(map) => { + let mut sorted = serde_json::Map::new(); + let mut keys = map.keys().cloned().collect::>(); + keys.sort(); + for key in keys { + if let Some(val) = map.get(&key) { + sorted.insert(key, canonical_json(val)); + } + } + JsonValue::Object(sorted) + } + JsonValue::Array(items) => JsonValue::Array(items.iter().map(canonical_json).collect()), + other => other.clone(), + } +} + +fn paths_match(expected: &Path, provided: &str) -> bool { + let provided_path = PathBuf::from(provided); + if let (Ok(expanded_expected), Ok(expanded_provided)) = + (expected.canonicalize(), provided_path.canonicalize()) + { + return expanded_expected == expanded_provided; + } + + expected == provided_path +} + +fn value_at_path<'a>(root: &'a TomlValue, segments: &[String]) -> Option<&'a TomlValue> { + let mut current = root; + for segment in segments { + match current { + TomlValue::Table(table) => { + current = table.get(segment)?; + } + TomlValue::Array(items) => { + let idx: usize = segment.parse().ok()?; + current = items.get(idx)?; + } + _ => return None, + } + } + Some(current) +} + +fn override_message(layer: &ConfigLayerName) -> String { + match layer { + ConfigLayerName::Mdm => "Overridden by managed policy (mdm)".to_string(), + ConfigLayerName::System => "Overridden by managed config (system)".to_string(), + ConfigLayerName::SessionFlags => "Overridden by session flags".to_string(), + ConfigLayerName::User => "Overridden by user config".to_string(), + } +} + +fn compute_override_metadata( + layers: &LayersState, + effective: &TomlValue, + segments: &[String], +) -> Option { + let user_value = value_at_path(&layers.user.config, segments); + let effective_value = value_at_path(effective, segments); + + if user_value.is_some() && user_value == effective_value { + return None; + } + + if user_value.is_none() && effective_value.is_none() { + return None; + } + + let effective_layer = find_effective_layer(layers, segments); + let overriding_layer = effective_layer.unwrap_or_else(|| layers.user.metadata()); + let message = override_message(&overriding_layer.name); + + Some(OverriddenMetadata { + message, + overriding_layer, + effective_value: effective_value + .map(to_json_value) + .unwrap_or(JsonValue::Null), + }) +} + +fn first_overridden_edit( + layers: &LayersState, + effective: &TomlValue, + edits: &[Vec], +) -> Option { + for segments in edits { + if let Some(meta) = compute_override_metadata(layers, effective, segments) { + return Some(meta); + } + } + None +} + +fn find_effective_layer(layers: &LayersState, segments: &[String]) -> Option { + let check = + |state: &LayerState| value_at_path(&state.config, segments).map(|_| state.metadata()); + + if let Some(mdm) = &layers.mdm + && let Some(meta) = check(mdm) + { + return Some(meta); + } + if let Some(system) = &layers.system + && let Some(meta) = check(system) + { + return Some(meta); + } + if let Some(meta) = check(&layers.session_flags) { + return Some(meta); + } + check(&layers.user) +} + +fn system_config_path(codex_home: &Path) -> PathBuf { + if let Ok(path) = std::env::var("CODEX_MANAGED_CONFIG_PATH") { + return PathBuf::from(path); + } + + #[cfg(unix)] + { + let _ = codex_home; + PathBuf::from("/etc/codex/managed_config.toml") + } + + #[cfg(not(unix))] + { + codex_home.join("managed_config.toml") + } +} + +fn internal_error(context: &str, err: E) -> JSONRPCErrorError { + JSONRPCErrorError { + code: INTERNAL_ERROR_CODE, + message: format!("{context}: {err}"), + data: None, + } +} + +fn config_write_error(code: ConfigWriteErrorCode, message: impl Into) -> JSONRPCErrorError { + JSONRPCErrorError { + code: INVALID_REQUEST_ERROR_CODE, + message: message.into(), + data: Some(json!({ + "config_write_error_code": code, + })), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + use tempfile::tempdir; + + #[tokio::test] + async fn read_includes_origins_and_layers() { + let tmp = tempdir().expect("tempdir"); + std::fs::write(tmp.path().join(CONFIG_FILE_NAME), "model = \"user\"").unwrap(); + + let managed_path = tmp.path().join("managed_config.toml"); + std::fs::write(&managed_path, "approval_policy = \"never\"").unwrap(); + + let api = ConfigApi::with_overrides( + tmp.path().to_path_buf(), + vec![], + LoaderOverrides { + managed_config_path: Some(managed_path), + #[cfg(target_os = "macos")] + managed_preferences_base64: None, + }, + ); + + let response = api + .read(ConfigReadParams { + include_layers: true, + }) + .await + .expect("response"); + + assert_eq!( + response.config.get("approval_policy"), + Some(&json!("never")) + ); + + assert_eq!( + response + .origins + .get("approval_policy") + .expect("origin") + .name, + ConfigLayerName::System + ); + let layers = response.layers.expect("layers present"); + assert_eq!(layers.first().unwrap().name, ConfigLayerName::System); + assert_eq!(layers.get(1).unwrap().name, ConfigLayerName::SessionFlags); + assert_eq!(layers.last().unwrap().name, ConfigLayerName::User); + } + + #[tokio::test] + async fn write_value_reports_override() { + let tmp = tempdir().expect("tempdir"); + std::fs::write( + tmp.path().join(CONFIG_FILE_NAME), + "approval_policy = \"on-request\"", + ) + .unwrap(); + + let managed_path = tmp.path().join("managed_config.toml"); + std::fs::write(&managed_path, "approval_policy = \"never\"").unwrap(); + + let api = ConfigApi::with_overrides( + tmp.path().to_path_buf(), + vec![], + LoaderOverrides { + managed_config_path: Some(managed_path), + #[cfg(target_os = "macos")] + managed_preferences_base64: None, + }, + ); + + let result = api + .write_value(ConfigValueWriteParams { + file_path: tmp.path().join(CONFIG_FILE_NAME).display().to_string(), + key_path: "approval_policy".to_string(), + value: json!("never"), + merge_strategy: MergeStrategy::Replace, + expected_version: None, + }) + .await + .expect("result"); + + let read_after = api + .read(ConfigReadParams { + include_layers: true, + }) + .await + .expect("read"); + let config_object = read_after.config.as_object().expect("object"); + assert_eq!(config_object.get("approval_policy"), Some(&json!("never"))); + assert_eq!( + read_after + .origins + .get("approval_policy") + .expect("origin") + .name, + ConfigLayerName::System + ); + assert_eq!(result.status, WriteStatus::Ok); + assert!(result.overridden_metadata.is_none()); + } + + #[tokio::test] + async fn version_conflict_rejected() { + let tmp = tempdir().expect("tempdir"); + std::fs::write(tmp.path().join(CONFIG_FILE_NAME), "model = \"user\"").unwrap(); + + let api = ConfigApi::new(tmp.path().to_path_buf(), vec![]); + let error = api + .write_value(ConfigValueWriteParams { + file_path: tmp.path().join(CONFIG_FILE_NAME).display().to_string(), + key_path: "model".to_string(), + value: json!("gpt-5"), + merge_strategy: MergeStrategy::Replace, + expected_version: Some("sha256:bogus".to_string()), + }) + .await + .expect_err("should fail"); + + assert_eq!(error.code, INVALID_REQUEST_ERROR_CODE); + assert_eq!( + error + .data + .as_ref() + .and_then(|d| d.get("config_write_error_code")) + .and_then(serde_json::Value::as_str), + Some("configVersionConflict") + ); + } + + #[tokio::test] + async fn invalid_user_value_rejected_even_if_overridden_by_managed() { + let tmp = tempdir().expect("tempdir"); + std::fs::write(tmp.path().join(CONFIG_FILE_NAME), "model = \"user\"").unwrap(); + + let managed_path = tmp.path().join("managed_config.toml"); + std::fs::write(&managed_path, "approval_policy = \"never\"").unwrap(); + + let api = ConfigApi::with_overrides( + tmp.path().to_path_buf(), + vec![], + LoaderOverrides { + managed_config_path: Some(managed_path), + #[cfg(target_os = "macos")] + managed_preferences_base64: None, + }, + ); + + let error = api + .write_value(ConfigValueWriteParams { + file_path: tmp.path().join(CONFIG_FILE_NAME).display().to_string(), + key_path: "approval_policy".to_string(), + value: json!("bogus"), + merge_strategy: MergeStrategy::Replace, + expected_version: None, + }) + .await + .expect_err("should fail validation"); + + assert_eq!(error.code, INVALID_REQUEST_ERROR_CODE); + assert_eq!( + error + .data + .as_ref() + .and_then(|d| d.get("config_write_error_code")) + .and_then(serde_json::Value::as_str), + Some("configValidationError") + ); + + let contents = + std::fs::read_to_string(tmp.path().join(CONFIG_FILE_NAME)).expect("read config"); + assert_eq!(contents.trim(), "model = \"user\""); + } + + #[tokio::test] + async fn read_reports_managed_overrides_user_and_session_flags() { + let tmp = tempdir().expect("tempdir"); + std::fs::write(tmp.path().join(CONFIG_FILE_NAME), "model = \"user\"").unwrap(); + + let managed_path = tmp.path().join("managed_config.toml"); + std::fs::write(&managed_path, "model = \"system\"").unwrap(); + + let cli_overrides = vec![( + "model".to_string(), + TomlValue::String("session".to_string()), + )]; + + let api = ConfigApi::with_overrides( + tmp.path().to_path_buf(), + cli_overrides, + LoaderOverrides { + managed_config_path: Some(managed_path), + #[cfg(target_os = "macos")] + managed_preferences_base64: None, + }, + ); + + let response = api + .read(ConfigReadParams { + include_layers: true, + }) + .await + .expect("response"); + + assert_eq!(response.config.get("model"), Some(&json!("system"))); + assert_eq!( + response.origins.get("model").expect("origin").name, + ConfigLayerName::System + ); + let layers = response.layers.expect("layers"); + assert_eq!(layers.first().unwrap().name, ConfigLayerName::System); + assert_eq!(layers.get(1).unwrap().name, ConfigLayerName::SessionFlags); + assert_eq!(layers.get(2).unwrap().name, ConfigLayerName::User); + } + + #[tokio::test] + async fn write_value_reports_managed_override() { + let tmp = tempdir().expect("tempdir"); + std::fs::write(tmp.path().join(CONFIG_FILE_NAME), "").unwrap(); + + let managed_path = tmp.path().join("managed_config.toml"); + std::fs::write(&managed_path, "approval_policy = \"never\"").unwrap(); + + let api = ConfigApi::with_overrides( + tmp.path().to_path_buf(), + vec![], + LoaderOverrides { + managed_config_path: Some(managed_path), + #[cfg(target_os = "macos")] + managed_preferences_base64: None, + }, + ); + + let result = api + .write_value(ConfigValueWriteParams { + file_path: tmp.path().join(CONFIG_FILE_NAME).display().to_string(), + key_path: "approval_policy".to_string(), + value: json!("on-request"), + merge_strategy: MergeStrategy::Replace, + expected_version: None, + }) + .await + .expect("result"); + + assert_eq!(result.status, WriteStatus::OkOverridden); + let overridden = result.overridden_metadata.expect("overridden metadata"); + assert_eq!(overridden.overriding_layer.name, ConfigLayerName::System); + assert_eq!(overridden.effective_value, json!("never")); + } +} diff --git a/codex-rs/app-server/src/lib.rs b/codex-rs/app-server/src/lib.rs index 9ad6f50b21..2aee5bc93e 100644 --- a/codex-rs/app-server/src/lib.rs +++ b/codex-rs/app-server/src/lib.rs @@ -18,6 +18,7 @@ use tokio::io::AsyncWriteExt; use tokio::io::BufReader; use tokio::io::{self}; use tokio::sync::mpsc; +use toml::Value as TomlValue; use tracing::Level; use tracing::debug; use tracing::error; @@ -30,6 +31,7 @@ use tracing_subscriber::util::SubscriberInitExt; mod bespoke_event_handling; mod codex_message_processor; +mod config_api; mod error_code; mod fuzzy_file_search; mod message_processor; @@ -80,11 +82,12 @@ pub async fn run_main( format!("error parsing -c overrides: {e}"), ) })?; - let config = Config::load_with_cli_overrides(cli_kv_overrides, ConfigOverrides::default()) - .await - .map_err(|e| { - std::io::Error::new(ErrorKind::InvalidData, format!("error loading config: {e}")) - })?; + let config = + Config::load_with_cli_overrides(cli_kv_overrides.clone(), ConfigOverrides::default()) + .await + .map_err(|e| { + std::io::Error::new(ErrorKind::InvalidData, format!("error loading config: {e}")) + })?; let feedback = CodexFeedback::new(); @@ -121,10 +124,12 @@ pub async fn run_main( // Task: process incoming messages. let processor_handle = tokio::spawn({ let outgoing_message_sender = OutgoingMessageSender::new(outgoing_tx); + let cli_overrides: Vec<(String, TomlValue)> = cli_kv_overrides.clone(); let mut processor = MessageProcessor::new( outgoing_message_sender, codex_linux_sandbox_exe, std::sync::Arc::new(config), + cli_overrides, feedback.clone(), ); async move { diff --git a/codex-rs/app-server/src/message_processor.rs b/codex-rs/app-server/src/message_processor.rs index 403263b896..90560e9b3c 100644 --- a/codex-rs/app-server/src/message_processor.rs +++ b/codex-rs/app-server/src/message_processor.rs @@ -1,16 +1,22 @@ use std::path::PathBuf; +use std::sync::Arc; use crate::codex_message_processor::CodexMessageProcessor; +use crate::config_api::ConfigApi; use crate::error_code::INVALID_REQUEST_ERROR_CODE; use crate::outgoing_message::OutgoingMessageSender; use codex_app_server_protocol::ClientInfo; use codex_app_server_protocol::ClientRequest; +use codex_app_server_protocol::ConfigBatchWriteParams; +use codex_app_server_protocol::ConfigReadParams; +use codex_app_server_protocol::ConfigValueWriteParams; use codex_app_server_protocol::InitializeResponse; use codex_app_server_protocol::JSONRPCError; use codex_app_server_protocol::JSONRPCErrorError; use codex_app_server_protocol::JSONRPCNotification; use codex_app_server_protocol::JSONRPCRequest; use codex_app_server_protocol::JSONRPCResponse; +use codex_app_server_protocol::RequestId; use codex_core::AuthManager; use codex_core::ConversationManager; use codex_core::config::Config; @@ -18,11 +24,12 @@ use codex_core::default_client::USER_AGENT_SUFFIX; use codex_core::default_client::get_codex_user_agent; use codex_feedback::CodexFeedback; use codex_protocol::protocol::SessionSource; -use std::sync::Arc; +use toml::Value as TomlValue; pub(crate) struct MessageProcessor { outgoing: Arc, codex_message_processor: CodexMessageProcessor, + config_api: ConfigApi, initialized: bool, } @@ -33,6 +40,7 @@ impl MessageProcessor { outgoing: OutgoingMessageSender, codex_linux_sandbox_exe: Option, config: Arc, + cli_overrides: Vec<(String, TomlValue)>, feedback: CodexFeedback, ) -> Self { let outgoing = Arc::new(outgoing); @@ -50,13 +58,15 @@ impl MessageProcessor { conversation_manager, outgoing.clone(), codex_linux_sandbox_exe, - config, + Arc::clone(&config), feedback, ); + let config_api = ConfigApi::new(config.codex_home.clone(), cli_overrides); Self { outgoing, codex_message_processor, + config_api, initialized: false, } } @@ -134,9 +144,20 @@ impl MessageProcessor { } } - self.codex_message_processor - .process_request(codex_request) - .await; + match codex_request { + ClientRequest::ConfigRead { request_id, params } => { + self.handle_config_read(request_id, params).await; + } + ClientRequest::ConfigValueWrite { request_id, params } => { + self.handle_config_value_write(request_id, params).await; + } + ClientRequest::ConfigBatchWrite { request_id, params } => { + self.handle_config_batch_write(request_id, params).await; + } + other => { + self.codex_message_processor.process_request(other).await; + } + } } pub(crate) async fn process_notification(&self, notification: JSONRPCNotification) { @@ -156,4 +177,33 @@ impl MessageProcessor { pub(crate) fn process_error(&mut self, err: JSONRPCError) { tracing::error!("<- error: {:?}", err); } + + async fn handle_config_read(&self, request_id: RequestId, params: ConfigReadParams) { + match self.config_api.read(params).await { + Ok(response) => self.outgoing.send_response(request_id, response).await, + Err(error) => self.outgoing.send_error(request_id, error).await, + } + } + + async fn handle_config_value_write( + &self, + request_id: RequestId, + params: ConfigValueWriteParams, + ) { + match self.config_api.write_value(params).await { + Ok(response) => self.outgoing.send_response(request_id, response).await, + Err(error) => self.outgoing.send_error(request_id, error).await, + } + } + + async fn handle_config_batch_write( + &self, + request_id: RequestId, + params: ConfigBatchWriteParams, + ) { + match self.config_api.batch_write(params).await { + Ok(response) => self.outgoing.send_response(request_id, response).await, + Err(error) => self.outgoing.send_error(request_id, error).await, + } + } } diff --git a/codex-rs/app-server/tests/common/mcp_process.rs b/codex-rs/app-server/tests/common/mcp_process.rs index 920a6fa01d..e2da40bf2d 100644 --- a/codex-rs/app-server/tests/common/mcp_process.rs +++ b/codex-rs/app-server/tests/common/mcp_process.rs @@ -18,6 +18,9 @@ use codex_app_server_protocol::CancelLoginAccountParams; use codex_app_server_protocol::CancelLoginChatGptParams; use codex_app_server_protocol::ClientInfo; use codex_app_server_protocol::ClientNotification; +use codex_app_server_protocol::ConfigBatchWriteParams; +use codex_app_server_protocol::ConfigReadParams; +use codex_app_server_protocol::ConfigValueWriteParams; use codex_app_server_protocol::FeedbackUploadParams; use codex_app_server_protocol::GetAccountParams; use codex_app_server_protocol::GetAuthStatusParams; @@ -401,6 +404,30 @@ impl McpProcess { self.send_request("logoutChatGpt", None).await } + pub async fn send_config_read_request( + &mut self, + params: ConfigReadParams, + ) -> anyhow::Result { + let params = Some(serde_json::to_value(params)?); + self.send_request("config/read", params).await + } + + pub async fn send_config_value_write_request( + &mut self, + params: ConfigValueWriteParams, + ) -> anyhow::Result { + let params = Some(serde_json::to_value(params)?); + self.send_request("config/value/write", params).await + } + + pub async fn send_config_batch_write_request( + &mut self, + params: ConfigBatchWriteParams, + ) -> anyhow::Result { + let params = Some(serde_json::to_value(params)?); + self.send_request("config/batchWrite", params).await + } + /// Send an `account/logout` JSON-RPC request. pub async fn send_logout_account_request(&mut self) -> anyhow::Result { self.send_request("account/logout", None).await diff --git a/codex-rs/app-server/tests/suite/v2/config_rpc.rs b/codex-rs/app-server/tests/suite/v2/config_rpc.rs new file mode 100644 index 0000000000..343a13c3c4 --- /dev/null +++ b/codex-rs/app-server/tests/suite/v2/config_rpc.rs @@ -0,0 +1,347 @@ +use anyhow::Result; +use app_test_support::McpProcess; +use app_test_support::to_response; +use codex_app_server_protocol::ConfigBatchWriteParams; +use codex_app_server_protocol::ConfigEdit; +use codex_app_server_protocol::ConfigLayerName; +use codex_app_server_protocol::ConfigReadParams; +use codex_app_server_protocol::ConfigReadResponse; +use codex_app_server_protocol::ConfigValueWriteParams; +use codex_app_server_protocol::ConfigWriteResponse; +use codex_app_server_protocol::JSONRPCError; +use codex_app_server_protocol::JSONRPCResponse; +use codex_app_server_protocol::MergeStrategy; +use codex_app_server_protocol::RequestId; +use codex_app_server_protocol::WriteStatus; +use pretty_assertions::assert_eq; +use serde_json::json; +use tempfile::TempDir; +use tokio::time::timeout; + +const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); + +fn write_config(codex_home: &TempDir, contents: &str) -> Result<()> { + Ok(std::fs::write( + codex_home.path().join("config.toml"), + contents, + )?) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn config_read_returns_effective_and_layers() -> Result<()> { + let codex_home = TempDir::new()?; + write_config( + &codex_home, + r#" +model = "gpt-user" +sandbox_mode = "workspace-write" +"#, + )?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let request_id = mcp + .send_config_read_request(ConfigReadParams { + include_layers: true, + }) + .await?; + let resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(request_id)), + ) + .await??; + let ConfigReadResponse { + config, + origins, + layers, + } = to_response(resp)?; + + assert_eq!(config.get("model"), Some(&json!("gpt-user"))); + assert_eq!( + origins.get("model").expect("origin").name, + ConfigLayerName::User + ); + let layers = layers.expect("layers present"); + assert_eq!(layers.len(), 2); + assert_eq!(layers[0].name, ConfigLayerName::SessionFlags); + assert_eq!(layers[1].name, ConfigLayerName::User); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn config_read_includes_system_layer_and_overrides() -> Result<()> { + let codex_home = TempDir::new()?; + write_config( + &codex_home, + r#" +model = "gpt-user" +approval_policy = "on-request" +sandbox_mode = "workspace-write" + +[sandbox_workspace_write] +writable_roots = ["/user"] +network_access = true +"#, + )?; + + let managed_path = codex_home.path().join("managed_config.toml"); + std::fs::write( + &managed_path, + r#" +model = "gpt-system" +approval_policy = "never" + +[sandbox_workspace_write] +writable_roots = ["/system"] +"#, + )?; + + let managed_path_str = managed_path.display().to_string(); + + let mut mcp = McpProcess::new_with_env( + codex_home.path(), + &[("CODEX_MANAGED_CONFIG_PATH", Some(&managed_path_str))], + ) + .await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let request_id = mcp + .send_config_read_request(ConfigReadParams { + include_layers: true, + }) + .await?; + let resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(request_id)), + ) + .await??; + let ConfigReadResponse { + config, + origins, + layers, + } = to_response(resp)?; + + assert_eq!(config.get("model"), Some(&json!("gpt-system"))); + assert_eq!( + origins.get("model").expect("origin").name, + ConfigLayerName::System + ); + + assert_eq!(config.get("approval_policy"), Some(&json!("never"))); + assert_eq!( + origins.get("approval_policy").expect("origin").name, + ConfigLayerName::System + ); + + assert_eq!(config.get("sandbox_mode"), Some(&json!("workspace-write"))); + assert_eq!( + origins.get("sandbox_mode").expect("origin").name, + ConfigLayerName::User + ); + + assert_eq!( + config + .get("sandbox_workspace_write") + .and_then(|v| v.get("writable_roots")), + Some(&json!(["/system"])) + ); + assert_eq!( + origins + .get("sandbox_workspace_write.writable_roots.0") + .expect("origin") + .name, + ConfigLayerName::System + ); + + assert_eq!( + config + .get("sandbox_workspace_write") + .and_then(|v| v.get("network_access")), + Some(&json!(true)) + ); + assert_eq!( + origins + .get("sandbox_workspace_write.network_access") + .expect("origin") + .name, + ConfigLayerName::User + ); + + let layers = layers.expect("layers present"); + assert_eq!(layers.len(), 3); + assert_eq!(layers[0].name, ConfigLayerName::System); + assert_eq!(layers[1].name, ConfigLayerName::SessionFlags); + assert_eq!(layers[2].name, ConfigLayerName::User); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn config_value_write_replaces_value() -> Result<()> { + let codex_home = TempDir::new()?; + write_config( + &codex_home, + r#" +model = "gpt-old" +"#, + )?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let read_id = mcp + .send_config_read_request(ConfigReadParams { + include_layers: false, + }) + .await?; + let read_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(read_id)), + ) + .await??; + let read: ConfigReadResponse = to_response(read_resp)?; + let expected_version = read.origins.get("model").map(|m| m.version.clone()); + + let write_id = mcp + .send_config_value_write_request(ConfigValueWriteParams { + file_path: codex_home.path().join("config.toml").display().to_string(), + key_path: "model".to_string(), + value: json!("gpt-new"), + merge_strategy: MergeStrategy::Replace, + expected_version, + }) + .await?; + let write_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(write_id)), + ) + .await??; + let write: ConfigWriteResponse = to_response(write_resp)?; + + assert_eq!(write.status, WriteStatus::Ok); + assert!(write.overridden_metadata.is_none()); + + let verify_id = mcp + .send_config_read_request(ConfigReadParams { + include_layers: false, + }) + .await?; + let verify_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(verify_id)), + ) + .await??; + let verify: ConfigReadResponse = to_response(verify_resp)?; + assert_eq!(verify.config.get("model"), Some(&json!("gpt-new"))); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn config_value_write_rejects_version_conflict() -> Result<()> { + let codex_home = TempDir::new()?; + write_config( + &codex_home, + r#" +model = "gpt-old" +"#, + )?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let write_id = mcp + .send_config_value_write_request(ConfigValueWriteParams { + file_path: codex_home.path().join("config.toml").display().to_string(), + key_path: "model".to_string(), + value: json!("gpt-new"), + merge_strategy: MergeStrategy::Replace, + expected_version: Some("sha256:stale".to_string()), + }) + .await?; + + let err: JSONRPCError = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_error_message(RequestId::Integer(write_id)), + ) + .await??; + let code = err + .error + .data + .as_ref() + .and_then(|d| d.get("config_write_error_code")) + .and_then(|v| v.as_str()); + assert_eq!(code, Some("configVersionConflict")); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn config_batch_write_applies_multiple_edits() -> Result<()> { + let codex_home = TempDir::new()?; + write_config(&codex_home, "")?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let batch_id = mcp + .send_config_batch_write_request(ConfigBatchWriteParams { + file_path: codex_home.path().join("config.toml").display().to_string(), + edits: vec![ + ConfigEdit { + key_path: "sandbox_mode".to_string(), + value: json!("workspace-write"), + merge_strategy: MergeStrategy::Replace, + }, + ConfigEdit { + key_path: "sandbox_workspace_write".to_string(), + value: json!({ + "writable_roots": ["/tmp"], + "network_access": false + }), + merge_strategy: MergeStrategy::Replace, + }, + ], + expected_version: None, + }) + .await?; + let batch_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(batch_id)), + ) + .await??; + let batch_write: ConfigWriteResponse = to_response(batch_resp)?; + assert_eq!(batch_write.status, WriteStatus::Ok); + + let read_id = mcp + .send_config_read_request(ConfigReadParams { + include_layers: false, + }) + .await?; + let read_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(read_id)), + ) + .await??; + let read: ConfigReadResponse = to_response(read_resp)?; + assert_eq!( + read.config.get("sandbox_mode"), + Some(&json!("workspace-write")) + ); + assert_eq!( + read.config + .get("sandbox_workspace_write") + .and_then(|v| v.get("writable_roots")), + Some(&json!(["/tmp"])) + ); + assert_eq!( + read.config + .get("sandbox_workspace_write") + .and_then(|v| v.get("network_access")), + Some(&json!(false)) + ); + + Ok(()) +} diff --git a/codex-rs/app-server/tests/suite/v2/mod.rs b/codex-rs/app-server/tests/suite/v2/mod.rs index a8594e7ca2..16d2142b2e 100644 --- a/codex-rs/app-server/tests/suite/v2/mod.rs +++ b/codex-rs/app-server/tests/suite/v2/mod.rs @@ -1,4 +1,5 @@ mod account; +mod config_rpc; mod model_list; mod rate_limits; mod review; diff --git a/codex-rs/core/src/config/mod.rs b/codex-rs/core/src/config/mod.rs index b15465653d..44cc651a0c 100644 --- a/codex-rs/core/src/config/mod.rs +++ b/codex-rs/core/src/config/mod.rs @@ -70,7 +70,7 @@ pub const GPT_5_CODEX_MEDIUM_MODEL: &str = "gpt-5.1-codex"; /// the context window. pub(crate) const PROJECT_DOC_MAX_BYTES: usize = 32 * 1024; // 32 KiB -pub(crate) const CONFIG_TOML_FILE: &str = "config.toml"; +pub const CONFIG_TOML_FILE: &str = "config.toml"; /// Application configuration loaded from disk and merged with overrides. #[derive(Debug, Clone, PartialEq)] diff --git a/codex-rs/core/src/config_loader/mod.rs b/codex-rs/core/src/config_loader/mod.rs index 6b55b015a3..4cd541d3e5 100644 --- a/codex-rs/core/src/config_loader/mod.rs +++ b/codex-rs/core/src/config_loader/mod.rs @@ -11,15 +11,15 @@ use toml::Value as TomlValue; #[cfg(unix)] const CODEX_MANAGED_CONFIG_SYSTEM_PATH: &str = "/etc/codex/managed_config.toml"; -#[derive(Debug)] -pub(crate) struct LoadedConfigLayers { +#[derive(Debug, Clone)] +pub struct LoadedConfigLayers { pub base: TomlValue, pub managed_config: Option, pub managed_preferences: Option, } -#[derive(Debug, Default)] -pub(crate) struct LoaderOverrides { +#[derive(Debug, Default, Clone)] +pub struct LoaderOverrides { pub managed_config_path: Option, #[cfg(target_os = "macos")] pub managed_preferences_base64: Option, @@ -47,11 +47,15 @@ pub async fn load_config_as_toml(codex_home: &Path) -> io::Result { load_config_as_toml_with_overrides(codex_home, LoaderOverrides::default()).await } +pub async fn load_config_layers(codex_home: &Path) -> io::Result { + load_config_layers_with_overrides(codex_home, LoaderOverrides::default()).await +} + fn default_empty_table() -> TomlValue { TomlValue::Table(Default::default()) } -pub(crate) async fn load_config_layers_with_overrides( +pub async fn load_config_layers_with_overrides( codex_home: &Path, overrides: LoaderOverrides, ) -> io::Result { @@ -130,7 +134,7 @@ async fn read_config_from_path( } /// Merge config `overlay` into `base`, giving `overlay` precedence. -pub(crate) fn merge_toml_values(base: &mut TomlValue, overlay: &TomlValue) { +pub fn merge_toml_values(base: &mut TomlValue, overlay: &TomlValue) { if let TomlValue::Table(overlay_table) = overlay && let TomlValue::Table(base_table) = base { @@ -147,6 +151,10 @@ pub(crate) fn merge_toml_values(base: &mut TomlValue, overlay: &TomlValue) { } fn managed_config_default_path(codex_home: &Path) -> PathBuf { + if let Ok(path) = std::env::var("CODEX_MANAGED_CONFIG_PATH") { + return PathBuf::from(path); + } + #[cfg(unix)] { let _ = codex_home; From 37d83e075e3cdc8975f905ab6cc7416d63522bd1 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Tue, 25 Nov 2025 10:35:35 +0000 Subject: [PATCH 28/61] feat: add custom env for unified exec process (#7286) --- .../core/src/unified_exec/session_manager.rs | 50 ++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/codex-rs/core/src/unified_exec/session_manager.rs b/codex-rs/core/src/unified_exec/session_manager.rs index ed489e18cd..d4ba75849b 100644 --- a/codex-rs/core/src/unified_exec/session_manager.rs +++ b/codex-rs/core/src/unified_exec/session_manager.rs @@ -48,6 +48,24 @@ use super::session::OutputBuffer; use super::session::OutputHandles; use super::session::UnifiedExecSession; +const UNIFIED_EXEC_ENV: [(&str, &str); 8] = [ + ("NO_COLOR", "1"), + ("TERM", "dumb"), + ("LANG", "C.UTF-8"), + ("LC_CTYPE", "C.UTF-8"), + ("LC_ALL", "C.UTF-8"), + ("COLORTERM", ""), + ("PAGER", "cat"), + ("GIT_PAGER", "cat"), +]; + +fn apply_unified_exec_env(mut env: HashMap) -> HashMap { + for (key, value) in UNIFIED_EXEC_ENV { + env.insert(key.to_string(), value.to_string()); + } + env +} + struct PreparedSessionHandles { writer_tx: mpsc::Sender>, output_buffer: OutputBuffer, @@ -460,12 +478,13 @@ impl UnifiedExecSessionManager { justification: Option, context: &UnifiedExecContext, ) -> Result { + let env = apply_unified_exec_env(create_env(&context.turn.shell_environment_policy)); let mut orchestrator = ToolOrchestrator::new(); let mut runtime = UnifiedExecRuntime::new(self); let req = UnifiedExecToolRequest::new( command.to_vec(), cwd, - create_env(&context.turn.shell_environment_policy), + env, with_escalated_permissions, justification, create_approval_requirement_for_command( @@ -619,6 +638,35 @@ mod tests { use tokio::time::Duration; use tokio::time::Instant; + #[test] + fn unified_exec_env_injects_defaults() { + let env = apply_unified_exec_env(HashMap::new()); + let expected = HashMap::from([ + ("NO_COLOR".to_string(), "1".to_string()), + ("TERM".to_string(), "dumb".to_string()), + ("LANG".to_string(), "C.UTF-8".to_string()), + ("LC_CTYPE".to_string(), "C.UTF-8".to_string()), + ("LC_ALL".to_string(), "C.UTF-8".to_string()), + ("COLORTERM".to_string(), String::new()), + ("PAGER".to_string(), "cat".to_string()), + ("GIT_PAGER".to_string(), "cat".to_string()), + ]); + + assert_eq!(env, expected); + } + + #[test] + fn unified_exec_env_overrides_existing_values() { + let mut base = HashMap::new(); + base.insert("NO_COLOR".to_string(), "0".to_string()); + base.insert("PATH".to_string(), "/usr/bin".to_string()); + + let env = apply_unified_exec_env(base); + + assert_eq!(env.get("NO_COLOR"), Some(&"1".to_string())); + assert_eq!(env.get("PATH"), Some(&"/usr/bin".to_string())); + } + #[test] fn pruning_prefers_exited_sessions_outside_recently_used() { let now = Instant::now(); From 157a16cefa343dfd0f093dd034df9904ad82845d Mon Sep 17 00:00:00 2001 From: Owen Lin Date: Tue, 25 Nov 2025 08:05:47 -0800 Subject: [PATCH 29/61] [app-server] feat: add thread_id and turn_id to item and error notifications (#7124) Add `thread_id` and `turn_id` to `item/started`, `item/completed`, and `error` notifications. Otherwise the client will have a hard time knowing which thread & turn (if multiple threads are running in parallel) a new item/error is for. Also add `thread_id` to `turn/started` and `turn/completed`. --- .../app-server-protocol/src/protocol/v2.rs | 8 + .../app-server/src/bespoke_event_handling.rs | 150 +++++++++++++++--- .../app-server/src/codex_message_processor.rs | 7 +- .../tests/suite/v2/turn_interrupt.rs | 4 +- .../app-server/tests/suite/v2/turn_start.rs | 2 + 5 files changed, 149 insertions(+), 22 deletions(-) diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index 71249b667f..16bdd66923 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -807,6 +807,8 @@ pub struct TurnError { #[ts(export_to = "v2/")] pub struct ErrorNotification { pub error: TurnError, + pub thread_id: String, + pub turn_id: String, } #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] @@ -1118,6 +1120,7 @@ pub struct ThreadStartedNotification { #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct TurnStartedNotification { + pub thread_id: String, pub turn: Turn, } @@ -1134,6 +1137,7 @@ pub struct Usage { #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct TurnCompletedNotification { + pub thread_id: String, pub turn: Turn, } @@ -1142,6 +1146,8 @@ pub struct TurnCompletedNotification { #[ts(export_to = "v2/")] pub struct ItemStartedNotification { pub item: ThreadItem, + pub thread_id: String, + pub turn_id: String, } #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] @@ -1149,6 +1155,8 @@ pub struct ItemStartedNotification { #[ts(export_to = "v2/")] pub struct ItemCompletedNotification { pub item: ThreadItem, + pub thread_id: String, + pub turn_id: String, } // Item-specific progress notifications diff --git a/codex-rs/app-server/src/bespoke_event_handling.rs b/codex-rs/app-server/src/bespoke_event_handling.rs index 0c2445d854..8de64880a4 100644 --- a/codex-rs/app-server/src/bespoke_event_handling.rs +++ b/codex-rs/app-server/src/bespoke_event_handling.rs @@ -117,7 +117,11 @@ pub(crate) async fn apply_bespoke_event_handling( changes: patch_changes.clone(), status: PatchApplyStatus::InProgress, }; - let notification = ItemStartedNotification { item }; + let notification = ItemStartedNotification { + thread_id: conversation_id.to_string(), + turn_id: event_id.clone(), + item, + }; outgoing .send_server_notification(ServerNotification::ItemStarted(notification)) .await; @@ -200,6 +204,7 @@ pub(crate) async fn apply_bespoke_event_handling( tokio::spawn(async move { on_command_execution_request_approval_response( event_id, + conversation_id, item_id, command_string, cwd, @@ -214,13 +219,23 @@ pub(crate) async fn apply_bespoke_event_handling( }, // TODO(celia): properly construct McpToolCall TurnItem in core. EventMsg::McpToolCallBegin(begin_event) => { - let notification = construct_mcp_tool_call_notification(begin_event).await; + let notification = construct_mcp_tool_call_notification( + begin_event, + conversation_id.to_string(), + event_id.clone(), + ) + .await; outgoing .send_server_notification(ServerNotification::ItemStarted(notification)) .await; } EventMsg::McpToolCallEnd(end_event) => { - let notification = construct_mcp_tool_call_end_notification(end_event).await; + let notification = construct_mcp_tool_call_end_notification( + end_event, + conversation_id.to_string(), + event_id.clone(), + ) + .await; outgoing .send_server_notification(ServerNotification::ItemCompleted(notification)) .await; @@ -287,6 +302,8 @@ pub(crate) async fn apply_bespoke_event_handling( outgoing .send_server_notification(ServerNotification::Error(ErrorNotification { error: turn_error, + thread_id: conversation_id.to_string(), + turn_id: event_id.clone(), })) .await; } @@ -300,11 +317,15 @@ pub(crate) async fn apply_bespoke_event_handling( outgoing .send_server_notification(ServerNotification::Error(ErrorNotification { error: turn_error, + thread_id: conversation_id.to_string(), + turn_id: event_id.clone(), })) .await; } EventMsg::EnteredReviewMode(review_request) => { let notification = ItemStartedNotification { + thread_id: conversation_id.to_string(), + turn_id: event_id.clone(), item: ThreadItem::CodeReview { id: event_id.clone(), review: review_request.user_facing_hint, @@ -316,14 +337,22 @@ pub(crate) async fn apply_bespoke_event_handling( } EventMsg::ItemStarted(item_started_event) => { let item: ThreadItem = item_started_event.item.clone().into(); - let notification = ItemStartedNotification { item }; + let notification = ItemStartedNotification { + thread_id: conversation_id.to_string(), + turn_id: event_id.clone(), + item, + }; outgoing .send_server_notification(ServerNotification::ItemStarted(notification)) .await; } EventMsg::ItemCompleted(item_completed_event) => { let item: ThreadItem = item_completed_event.item.clone().into(); - let notification = ItemCompletedNotification { item }; + let notification = ItemCompletedNotification { + thread_id: conversation_id.to_string(), + turn_id: event_id.clone(), + item, + }; outgoing .send_server_notification(ServerNotification::ItemCompleted(notification)) .await; @@ -333,9 +362,12 @@ pub(crate) async fn apply_bespoke_event_handling( Some(output) => render_review_output_text(&output), None => REVIEW_FALLBACK_MESSAGE.to_string(), }; + let review_item_id = event_id.clone(); let notification = ItemCompletedNotification { + thread_id: conversation_id.to_string(), + turn_id: event_id.clone(), item: ThreadItem::CodeReview { - id: event_id, + id: review_item_id, review: review_text, }, }; @@ -359,7 +391,11 @@ pub(crate) async fn apply_bespoke_event_handling( changes: convert_patch_changes(&patch_begin_event.changes), status: PatchApplyStatus::InProgress, }; - let notification = ItemStartedNotification { item }; + let notification = ItemStartedNotification { + thread_id: conversation_id.to_string(), + turn_id: event_id.clone(), + item, + }; outgoing .send_server_notification(ServerNotification::ItemStarted(notification)) .await; @@ -381,6 +417,7 @@ pub(crate) async fn apply_bespoke_event_handling( item_id, changes, status, + event_id.clone(), outgoing.as_ref(), &turn_summary_store, ) @@ -406,7 +443,11 @@ pub(crate) async fn apply_bespoke_event_handling( exit_code: None, duration_ms: None, }; - let notification = ItemStartedNotification { item }; + let notification = ItemStartedNotification { + thread_id: conversation_id.to_string(), + turn_id: event_id.clone(), + item, + }; outgoing .send_server_notification(ServerNotification::ItemStarted(notification)) .await; @@ -463,7 +504,11 @@ pub(crate) async fn apply_bespoke_event_handling( duration_ms: Some(duration_ms), }; - let notification = ItemCompletedNotification { item }; + let notification = ItemCompletedNotification { + thread_id: conversation_id.to_string(), + turn_id: event_id.clone(), + item, + }; outgoing .send_server_notification(ServerNotification::ItemCompleted(notification)) .await; @@ -500,11 +545,13 @@ pub(crate) async fn apply_bespoke_event_handling( } async fn emit_turn_completed_with_status( + conversation_id: ConversationId, event_id: String, status: TurnStatus, outgoing: &OutgoingMessageSender, ) { let notification = TurnCompletedNotification { + thread_id: conversation_id.to_string(), turn: Turn { id: event_id, items: vec![], @@ -521,6 +568,7 @@ async fn complete_file_change_item( item_id: String, changes: Vec, status: PatchApplyStatus, + turn_id: String, outgoing: &OutgoingMessageSender, turn_summary_store: &TurnSummaryStore, ) { @@ -536,13 +584,20 @@ async fn complete_file_change_item( changes, status, }; - let notification = ItemCompletedNotification { item }; + let notification = ItemCompletedNotification { + thread_id: conversation_id.to_string(), + turn_id, + item, + }; outgoing .send_server_notification(ServerNotification::ItemCompleted(notification)) .await; } +#[allow(clippy::too_many_arguments)] async fn complete_command_execution_item( + conversation_id: ConversationId, + turn_id: String, item_id: String, command: String, cwd: PathBuf, @@ -560,7 +615,11 @@ async fn complete_command_execution_item( exit_code: None, duration_ms: None, }; - let notification = ItemCompletedNotification { item }; + let notification = ItemCompletedNotification { + thread_id: conversation_id.to_string(), + turn_id, + item, + }; outgoing .send_server_notification(ServerNotification::ItemCompleted(notification)) .await; @@ -588,7 +647,7 @@ async fn handle_turn_complete( TurnStatus::Completed }; - emit_turn_completed_with_status(event_id, status, outgoing).await; + emit_turn_completed_with_status(conversation_id, event_id, status, outgoing).await; } async fn handle_turn_interrupted( @@ -599,7 +658,8 @@ async fn handle_turn_interrupted( ) { find_and_remove_turn_summary(conversation_id, turn_summary_store).await; - emit_turn_completed_with_status(event_id, TurnStatus::Interrupted, outgoing).await; + emit_turn_completed_with_status(conversation_id, event_id, TurnStatus::Interrupted, outgoing) + .await; } async fn handle_error( @@ -798,6 +858,7 @@ async fn on_file_change_request_approval_response( item_id, changes, status, + event_id.clone(), outgoing.as_ref(), &turn_summary_store, ) @@ -818,6 +879,7 @@ async fn on_file_change_request_approval_response( #[allow(clippy::too_many_arguments)] async fn on_command_execution_request_approval_response( event_id: String, + conversation_id: ConversationId, item_id: String, command: String, cwd: PathBuf, @@ -867,6 +929,8 @@ async fn on_command_execution_request_approval_response( if let Some(status) = completion_status { complete_command_execution_item( + conversation_id, + event_id.clone(), item_id.clone(), command.clone(), cwd.clone(), @@ -891,6 +955,8 @@ async fn on_command_execution_request_approval_response( /// similar to handle_mcp_tool_call_begin in exec async fn construct_mcp_tool_call_notification( begin_event: McpToolCallBeginEvent, + thread_id: String, + turn_id: String, ) -> ItemStartedNotification { let item = ThreadItem::McpToolCall { id: begin_event.call_id, @@ -901,12 +967,18 @@ async fn construct_mcp_tool_call_notification( result: None, error: None, }; - ItemStartedNotification { item } + ItemStartedNotification { + thread_id, + turn_id, + item, + } } /// simiilar to handle_mcp_tool_call_end in exec async fn construct_mcp_tool_call_end_notification( end_event: McpToolCallEndEvent, + thread_id: String, + turn_id: String, ) -> ItemCompletedNotification { let status = if end_event.is_success() { McpToolCallStatus::Completed @@ -939,7 +1011,11 @@ async fn construct_mcp_tool_call_end_notification( result, error, }; - ItemCompletedNotification { item } + ItemCompletedNotification { + thread_id, + turn_id, + item, + } } #[cfg(test)] @@ -1122,9 +1198,18 @@ mod tests { }, }; - let notification = construct_mcp_tool_call_notification(begin_event.clone()).await; + let thread_id = ConversationId::new().to_string(); + let turn_id = "turn_1".to_string(); + let notification = construct_mcp_tool_call_notification( + begin_event.clone(), + thread_id.clone(), + turn_id.clone(), + ) + .await; let expected = ItemStartedNotification { + thread_id, + turn_id, item: ThreadItem::McpToolCall { id: begin_event.call_id, server: begin_event.invocation.server, @@ -1267,9 +1352,18 @@ mod tests { }, }; - let notification = construct_mcp_tool_call_notification(begin_event.clone()).await; + let thread_id = ConversationId::new().to_string(); + let turn_id = "turn_2".to_string(); + let notification = construct_mcp_tool_call_notification( + begin_event.clone(), + thread_id.clone(), + turn_id.clone(), + ) + .await; let expected = ItemStartedNotification { + thread_id, + turn_id, item: ThreadItem::McpToolCall { id: begin_event.call_id, server: begin_event.invocation.server, @@ -1308,9 +1402,18 @@ mod tests { result: Ok(result), }; - let notification = construct_mcp_tool_call_end_notification(end_event.clone()).await; + let thread_id = ConversationId::new().to_string(); + let turn_id = "turn_3".to_string(); + let notification = construct_mcp_tool_call_end_notification( + end_event.clone(), + thread_id.clone(), + turn_id.clone(), + ) + .await; let expected = ItemCompletedNotification { + thread_id, + turn_id, item: ThreadItem::McpToolCall { id: end_event.call_id, server: end_event.invocation.server, @@ -1341,9 +1444,18 @@ mod tests { result: Err("boom".to_string()), }; - let notification = construct_mcp_tool_call_end_notification(end_event.clone()).await; + let thread_id = ConversationId::new().to_string(); + let turn_id = "turn_4".to_string(); + let notification = construct_mcp_tool_call_end_notification( + end_event.clone(), + thread_id.clone(), + turn_id.clone(), + ) + .await; let expected = ItemCompletedNotification { + thread_id, + turn_id, item: ThreadItem::McpToolCall { id: end_event.call_id, server: end_event.invocation.server, diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index 9dc2f2e016..b60f2d94c3 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -2478,7 +2478,10 @@ impl CodexMessageProcessor { self.outgoing.send_response(request_id, response).await; // Emit v2 turn/started notification. - let notif = TurnStartedNotification { turn }; + let notif = TurnStartedNotification { + thread_id: params.thread_id, + turn, + }; self.outgoing .send_server_notification(ServerNotification::TurnStarted(notif)) .await; @@ -2536,7 +2539,7 @@ impl CodexMessageProcessor { let response = TurnStartResponse { turn: turn.clone() }; self.outgoing.send_response(request_id, response).await; - let notif = TurnStartedNotification { turn }; + let notif = TurnStartedNotification { thread_id, turn }; self.outgoing .send_server_notification(ServerNotification::TurnStarted(notif)) .await; diff --git a/codex-rs/app-server/tests/suite/v2/turn_interrupt.rs b/codex-rs/app-server/tests/suite/v2/turn_interrupt.rs index 83389ed1d1..f68ffb899c 100644 --- a/codex-rs/app-server/tests/suite/v2/turn_interrupt.rs +++ b/codex-rs/app-server/tests/suite/v2/turn_interrupt.rs @@ -88,10 +88,11 @@ async fn turn_interrupt_aborts_running_turn() -> Result<()> { // Give the command a brief moment to start. tokio::time::sleep(std::time::Duration::from_secs(1)).await; + let thread_id = thread.id.clone(); // Interrupt the in-progress turn by id (v2 API). let interrupt_id = mcp .send_turn_interrupt_request(TurnInterruptParams { - thread_id: thread.id, + thread_id: thread_id.clone(), turn_id: turn.id, }) .await?; @@ -112,6 +113,7 @@ async fn turn_interrupt_aborts_running_turn() -> Result<()> { .params .expect("turn/completed params must be present"), )?; + assert_eq!(completed.thread_id, thread_id); assert_eq!(completed.turn.status, TurnStatus::Interrupted); Ok(()) diff --git a/codex-rs/app-server/tests/suite/v2/turn_start.rs b/codex-rs/app-server/tests/suite/v2/turn_start.rs index a3e75268c9..2c23fedd94 100644 --- a/codex-rs/app-server/tests/suite/v2/turn_start.rs +++ b/codex-rs/app-server/tests/suite/v2/turn_start.rs @@ -95,6 +95,7 @@ async fn turn_start_emits_notifications_and_accepts_model_override() -> Result<( .await??; let started: TurnStartedNotification = serde_json::from_value(notif.params.expect("params must be present"))?; + assert_eq!(started.thread_id, thread.id); assert_eq!( started.turn.status, codex_app_server_protocol::TurnStatus::InProgress @@ -138,6 +139,7 @@ async fn turn_start_emits_notifications_and_accepts_model_override() -> Result<( .params .expect("turn/completed params must be present"), )?; + assert_eq!(completed.thread_id, thread.id); assert_eq!(completed.turn.status, TurnStatus::Completed); Ok(()) From 9ba27cfa0a5eb3c35d6e552068e950e9435958e3 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Tue, 25 Nov 2025 16:12:14 +0000 Subject: [PATCH 30/61] feat: add compaction event (#7289) --- codex-rs/app-server-protocol/src/protocol/common.rs | 1 + codex-rs/app-server-protocol/src/protocol/v2.rs | 8 ++++++++ codex-rs/app-server/README.md | 1 + codex-rs/app-server/src/bespoke_event_handling.rs | 10 ++++++++++ codex-rs/core/src/compact.rs | 6 ++---- codex-rs/core/src/compact_remote.rs | 6 ++---- codex-rs/core/src/rollout/policy.rs | 1 + codex-rs/core/tests/suite/compact_remote.rs | 4 ++-- codex-rs/exec/src/event_processor_with_human_output.rs | 3 +++ codex-rs/mcp-server/src/codex_tool_runner.rs | 1 + codex-rs/protocol/src/protocol.rs | 6 ++++++ codex-rs/tui/src/chatwidget.rs | 1 + 12 files changed, 38 insertions(+), 10 deletions(-) diff --git a/codex-rs/app-server-protocol/src/protocol/common.rs b/codex-rs/app-server-protocol/src/protocol/common.rs index 721ffe2555..6c417caa87 100644 --- a/codex-rs/app-server-protocol/src/protocol/common.rs +++ b/codex-rs/app-server-protocol/src/protocol/common.rs @@ -514,6 +514,7 @@ server_notification_definitions! { ReasoningSummaryTextDelta => "item/reasoning/summaryTextDelta" (v2::ReasoningSummaryTextDeltaNotification), ReasoningSummaryPartAdded => "item/reasoning/summaryPartAdded" (v2::ReasoningSummaryPartAddedNotification), ReasoningTextDelta => "item/reasoning/textDelta" (v2::ReasoningTextDeltaNotification), + ContextCompacted => "thread/compacted" (v2::ContextCompactedNotification), /// Notifies the user of world-writable directories on Windows, which cannot be protected by the sandbox. WindowsWorldWritableWarning => "windows/worldWritableWarning" (v2::WindowsWorldWritableWarningNotification), diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index 16bdd66923..0d5ea0e2c5 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -1219,6 +1219,14 @@ pub struct WindowsWorldWritableWarningNotification { pub failed_scan: bool, } +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ContextCompactedNotification { + pub thread_id: String, + pub turn_id: String, +} + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index 7a137de266..fa0a4baaa0 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -257,6 +257,7 @@ Today both notifications carry an empty `items` array even when item events were - `fileChange` — `{id, changes, status}` describing proposed edits; `changes` list `{path, kind, diff}` and `status` is `inProgress`, `completed`, `failed`, or `declined`. - `mcpToolCall` — `{id, server, tool, status, arguments, result?, error?}` describing MCP calls; `status` is `inProgress`, `completed`, or `failed`. - `webSearch` — `{id, query}` for a web search request issued by the agent. +- `compacted` - `{threadId, turnId}` when codex compacts the conversation history. This can happen automatically. All items emit two shared lifecycle events: - `item/started` — emits the full `item` when a new unit of work begins so the UI can render it immediately; the `item.id` in this payload matches the `itemId` used by deltas. diff --git a/codex-rs/app-server/src/bespoke_event_handling.rs b/codex-rs/app-server/src/bespoke_event_handling.rs index 8de64880a4..a0d35a453a 100644 --- a/codex-rs/app-server/src/bespoke_event_handling.rs +++ b/codex-rs/app-server/src/bespoke_event_handling.rs @@ -14,6 +14,7 @@ use codex_app_server_protocol::CommandExecutionOutputDeltaNotification; use codex_app_server_protocol::CommandExecutionRequestApprovalParams; use codex_app_server_protocol::CommandExecutionRequestApprovalResponse; use codex_app_server_protocol::CommandExecutionStatus; +use codex_app_server_protocol::ContextCompactedNotification; use codex_app_server_protocol::ErrorNotification; use codex_app_server_protocol::ExecCommandApprovalParams; use codex_app_server_protocol::ExecCommandApprovalResponse; @@ -249,6 +250,15 @@ pub(crate) async fn apply_bespoke_event_handling( .send_server_notification(ServerNotification::AgentMessageDelta(notification)) .await; } + EventMsg::ContextCompacted(..) => { + let notification = ContextCompactedNotification { + thread_id: conversation_id.to_string(), + turn_id: event_id.clone(), + }; + outgoing + .send_server_notification(ServerNotification::ContextCompacted(notification)) + .await; + } EventMsg::ReasoningContentDelta(event) => { let notification = ReasoningSummaryTextDeltaNotification { item_id: event.item_id, diff --git a/codex-rs/core/src/compact.rs b/codex-rs/core/src/compact.rs index 1b3937b9fe..fb5c187b7f 100644 --- a/codex-rs/core/src/compact.rs +++ b/codex-rs/core/src/compact.rs @@ -8,8 +8,8 @@ use crate::codex::get_last_assistant_message_from_turn; use crate::error::CodexErr; use crate::error::Result as CodexResult; use crate::features::Feature; -use crate::protocol::AgentMessageEvent; use crate::protocol::CompactedItem; +use crate::protocol::ContextCompactedEvent; use crate::protocol::EventMsg; use crate::protocol::TaskStartedEvent; use crate::protocol::TurnContextItem; @@ -175,9 +175,7 @@ async fn run_compact_task_inner( }); sess.persist_rollout_items(&[rollout_item]).await; - let event = EventMsg::AgentMessage(AgentMessageEvent { - message: "Compact task completed".to_string(), - }); + let event = EventMsg::ContextCompacted(ContextCompactedEvent {}); sess.send_event(&turn_context, event).await; let warning = EventMsg::Warning(WarningEvent { diff --git a/codex-rs/core/src/compact_remote.rs b/codex-rs/core/src/compact_remote.rs index 534d794f0c..b855f28d39 100644 --- a/codex-rs/core/src/compact_remote.rs +++ b/codex-rs/core/src/compact_remote.rs @@ -4,8 +4,8 @@ use crate::Prompt; use crate::codex::Session; use crate::codex::TurnContext; use crate::error::Result as CodexResult; -use crate::protocol::AgentMessageEvent; use crate::protocol::CompactedItem; +use crate::protocol::ContextCompactedEvent; use crate::protocol::EventMsg; use crate::protocol::RolloutItem; use crate::protocol::TaskStartedEvent; @@ -74,9 +74,7 @@ async fn run_remote_compact_task_inner_impl( sess.persist_rollout_items(&[RolloutItem::Compacted(compacted_item)]) .await; - let event = EventMsg::AgentMessage(AgentMessageEvent { - message: "Compact task completed".to_string(), - }); + let event = EventMsg::ContextCompacted(ContextCompactedEvent {}); sess.send_event(turn_context, event).await; Ok(()) diff --git a/codex-rs/core/src/rollout/policy.rs b/codex-rs/core/src/rollout/policy.rs index 4d5f709d25..58072f9336 100644 --- a/codex-rs/core/src/rollout/policy.rs +++ b/codex-rs/core/src/rollout/policy.rs @@ -42,6 +42,7 @@ pub(crate) fn should_persist_event_msg(ev: &EventMsg) -> bool { | EventMsg::AgentReasoning(_) | EventMsg::AgentReasoningRawContent(_) | EventMsg::TokenCount(_) + | EventMsg::ContextCompacted(_) | EventMsg::EnteredReviewMode(_) | EventMsg::ExitedReviewMode(_) | EventMsg::UndoCompleted(_) diff --git a/codex-rs/core/tests/suite/compact_remote.rs b/codex-rs/core/tests/suite/compact_remote.rs index dc88bc5747..49fd5791c8 100644 --- a/codex-rs/core/tests/suite/compact_remote.rs +++ b/codex-rs/core/tests/suite/compact_remote.rs @@ -180,13 +180,13 @@ async fn remote_compact_runs_automatically() -> Result<()> { }) .await?; let message = wait_for_event_match(&codex, |ev| match ev { - EventMsg::AgentMessage(ev) => Some(ev.message.clone()), + EventMsg::ContextCompacted(_) => Some(true), _ => None, }) .await; wait_for_event(&codex, |ev| matches!(ev, EventMsg::TaskComplete(_))).await; - assert_eq!(message, "Compact task completed"); + assert!(message); assert_eq!(compact_mock.requests().len(), 1); let follow_up_body = responses_mock.single_request().body_json().to_string(); assert!(follow_up_body.contains("REMOTE_COMPACTED_SUMMARY")); diff --git a/codex-rs/exec/src/event_processor_with_human_output.rs b/codex-rs/exec/src/event_processor_with_human_output.rs index fabbabe659..64a5358f35 100644 --- a/codex-rs/exec/src/event_processor_with_human_output.rs +++ b/codex-rs/exec/src/event_processor_with_human_output.rs @@ -559,6 +559,9 @@ impl EventProcessor for EventProcessorWithHumanOutput { ts_msg!(self, "task aborted: review ended"); } }, + EventMsg::ContextCompacted(_) => { + ts_msg!(self, "context compacted"); + } EventMsg::ShutdownComplete => return CodexStatus::Shutdown, EventMsg::WebSearchBegin(_) | EventMsg::ExecApprovalRequest(_) diff --git a/codex-rs/mcp-server/src/codex_tool_runner.rs b/codex-rs/mcp-server/src/codex_tool_runner.rs index cceb8efc5d..55808f17ca 100644 --- a/codex-rs/mcp-server/src/codex_tool_runner.rs +++ b/codex-rs/mcp-server/src/codex_tool_runner.rs @@ -306,6 +306,7 @@ async fn run_codex_tool_session_inner( | EventMsg::UndoStarted(_) | EventMsg::UndoCompleted(_) | EventMsg::ExitedReviewMode(_) + | EventMsg::ContextCompacted(_) | EventMsg::DeprecationNotice(_) => { // For now, we do not do anything extra for these // events. Note that diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index e5dbb26b90..2fc47a50c9 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -455,6 +455,9 @@ pub enum EventMsg { /// indicates the task continued but the user should still be notified. Warning(WarningEvent), + /// Conversation history was compacted (either automatically or manually). + ContextCompacted(ContextCompactedEvent), + /// Agent has started a task TaskStarted(TaskStartedEvent), @@ -739,6 +742,9 @@ pub struct WarningEvent { pub message: String, } +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, TS)] +pub struct ContextCompactedEvent; + #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, TS)] pub struct TaskCompleteEvent { pub last_agent_message: Option, diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 980bdf7f4a..53c77f82be 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -1796,6 +1796,7 @@ impl ChatWidget { self.on_entered_review_mode(review_request) } EventMsg::ExitedReviewMode(review) => self.on_exited_review_mode(review), + EventMsg::ContextCompacted(_) => self.on_agent_message("Context compacted".to_owned()), EventMsg::RawResponseItem(_) | EventMsg::ItemStarted(_) | EventMsg::ItemCompleted(_) From caf2749d5b02a44bb0e5f48c59d86c990629adb8 Mon Sep 17 00:00:00 2001 From: Owen Lin Date: Tue, 25 Nov 2025 08:21:08 -0800 Subject: [PATCH 31/61] [app-server] feat: add turn/diff/updated event (#7279) This is the V2 version of `EventMsg::TurnDiff`. I decided to expose this as a `turn/*` notification as opposed to an Item to make it more explicit that the diff is accumulated throughout a turn (every `apply_patch` call updates the running diff). Also, I don't think it's worth persisting this diff as an Item because it can always be recomputed from the actual `FileChange` Items. --- .../src/protocol/common.rs | 1 + .../app-server-protocol/src/protocol/v2.rs | 10 +++ .../app-server/src/bespoke_event_handling.rs | 74 +++++++++++++++++++ 3 files changed, 85 insertions(+) diff --git a/codex-rs/app-server-protocol/src/protocol/common.rs b/codex-rs/app-server-protocol/src/protocol/common.rs index 6c417caa87..885cfae26c 100644 --- a/codex-rs/app-server-protocol/src/protocol/common.rs +++ b/codex-rs/app-server-protocol/src/protocol/common.rs @@ -504,6 +504,7 @@ server_notification_definitions! { ThreadStarted => "thread/started" (v2::ThreadStartedNotification), TurnStarted => "turn/started" (v2::TurnStartedNotification), TurnCompleted => "turn/completed" (v2::TurnCompletedNotification), + TurnDiffUpdated => "turn/diff/updated" (v2::TurnDiffUpdatedNotification), ItemStarted => "item/started" (v2::ItemStartedNotification), ItemCompleted => "item/completed" (v2::ItemCompletedNotification), AgentMessageDelta => "item/agentMessage/delta" (v2::AgentMessageDeltaNotification), diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index 0d5ea0e2c5..8c8e71a2ca 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -1141,6 +1141,16 @@ pub struct TurnCompletedNotification { pub turn: Turn, } +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +/// Notification that the turn-level unified diff has changed. +/// Contains the latest aggregated diff across all file changes in the turn. +pub struct TurnDiffUpdatedNotification { + pub turn_id: String, + pub diff: String, +} + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] diff --git a/codex-rs/app-server/src/bespoke_event_handling.rs b/codex-rs/app-server/src/bespoke_event_handling.rs index a0d35a453a..d5037358b3 100644 --- a/codex-rs/app-server/src/bespoke_event_handling.rs +++ b/codex-rs/app-server/src/bespoke_event_handling.rs @@ -38,6 +38,7 @@ use codex_app_server_protocol::ServerRequestPayload; use codex_app_server_protocol::ThreadItem; use codex_app_server_protocol::Turn; use codex_app_server_protocol::TurnCompletedNotification; +use codex_app_server_protocol::TurnDiffUpdatedNotification; use codex_app_server_protocol::TurnError; use codex_app_server_protocol::TurnInterruptResponse; use codex_app_server_protocol::TurnStatus; @@ -53,6 +54,7 @@ use codex_core::protocol::McpToolCallBeginEvent; use codex_core::protocol::McpToolCallEndEvent; use codex_core::protocol::Op; use codex_core::protocol::ReviewDecision; +use codex_core::protocol::TurnDiffEvent; use codex_core::review_format::format_review_findings_block; use codex_protocol::ConversationId; use codex_protocol::protocol::ReviewOutputEvent; @@ -549,11 +551,31 @@ pub(crate) async fn apply_bespoke_event_handling( handle_turn_interrupted(conversation_id, event_id, &outgoing, &turn_summary_store) .await; } + EventMsg::TurnDiff(turn_diff_event) => { + handle_turn_diff(&event_id, turn_diff_event, api_version, outgoing.as_ref()).await; + } _ => {} } } +async fn handle_turn_diff( + event_id: &str, + turn_diff_event: TurnDiffEvent, + api_version: ApiVersion, + outgoing: &OutgoingMessageSender, +) { + if let ApiVersion::V2 = api_version { + let notification = TurnDiffUpdatedNotification { + turn_id: event_id.to_string(), + diff: turn_diff_event.unified_diff, + }; + outgoing + .send_server_notification(ServerNotification::TurnDiffUpdated(notification)) + .await; + } +} + async fn emit_turn_completed_with_status( conversation_id: ConversationId, event_id: String, @@ -1481,4 +1503,56 @@ mod tests { assert_eq!(notification, expected); } + + #[tokio::test] + async fn test_handle_turn_diff_emits_v2_notification() -> Result<()> { + let (tx, mut rx) = mpsc::channel(CHANNEL_CAPACITY); + let outgoing = OutgoingMessageSender::new(tx); + let unified_diff = "--- a\n+++ b\n".to_string(); + + handle_turn_diff( + "turn-1", + TurnDiffEvent { + unified_diff: unified_diff.clone(), + }, + ApiVersion::V2, + &outgoing, + ) + .await; + + let msg = rx + .recv() + .await + .ok_or_else(|| anyhow!("should send one notification"))?; + match msg { + OutgoingMessage::AppServerNotification(ServerNotification::TurnDiffUpdated( + notification, + )) => { + assert_eq!(notification.turn_id, "turn-1"); + assert_eq!(notification.diff, unified_diff); + } + other => bail!("unexpected message: {other:?}"), + } + assert!(rx.try_recv().is_err(), "no extra messages expected"); + Ok(()) + } + + #[tokio::test] + async fn test_handle_turn_diff_is_noop_for_v1() -> Result<()> { + let (tx, mut rx) = mpsc::channel(CHANNEL_CAPACITY); + let outgoing = OutgoingMessageSender::new(tx); + + handle_turn_diff( + "turn-1", + TurnDiffEvent { + unified_diff: "diff".to_string(), + }, + ApiVersion::V1, + &outgoing, + ) + .await; + + assert!(rx.try_recv().is_err(), "no messages expected"); + Ok(()) + } } From d8a7a63959f37d12666057c3e8837d61758b4a23 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Tue, 25 Nov 2025 16:25:44 +0000 Subject: [PATCH 32/61] fix: Drop MacOS 13 (#7295) --- .github/workflows/shell-tool-mcp.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/shell-tool-mcp.yml b/.github/workflows/shell-tool-mcp.yml index 14a6a8bcd5..6a0f68ff0e 100644 --- a/.github/workflows/shell-tool-mcp.yml +++ b/.github/workflows/shell-tool-mcp.yml @@ -236,9 +236,6 @@ jobs: - runner: macos-14 target: aarch64-apple-darwin variant: macos-14 - - runner: macos-13 - target: x86_64-apple-darwin - variant: macos-13 steps: - name: Checkout repository uses: actions/checkout@v6 From 2845e2c0065c1b04253180735c59df24dd21b8ca Mon Sep 17 00:00:00 2001 From: jif-oai Date: Tue, 25 Nov 2025 17:20:25 +0000 Subject: [PATCH 33/61] fix: drop conversation when `/new` (#7297) --- codex-rs/core/src/codex.rs | 4 + .../core/src/unified_exec/session_manager.rs | 5 + codex-rs/tui/src/app.rs | 101 +++++++++++++++++- 3 files changed, 109 insertions(+), 1 deletion(-) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 799374d6d6..e3c5de5762 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -1745,6 +1745,10 @@ mod handlers { pub async fn shutdown(sess: &Arc, sub_id: String) -> bool { sess.abort_all_tasks(TurnAbortReason::Interrupted).await; + sess.services + .unified_exec_manager + .terminate_all_sessions() + .await; info!("Shutting down Codex instance"); // Gracefully flush and shutdown rollout recorder on session end so tests diff --git a/codex-rs/core/src/unified_exec/session_manager.rs b/codex-rs/core/src/unified_exec/session_manager.rs index d4ba75849b..515bb607f6 100644 --- a/codex-rs/core/src/unified_exec/session_manager.rs +++ b/codex-rs/core/src/unified_exec/session_manager.rs @@ -617,6 +617,11 @@ impl UnifiedExecSessionManager { .find(|(session_id, _, _)| !protected.contains(session_id)) .map(|(session_id, _, _)| session_id) } + + pub(crate) async fn terminate_all_sessions(&self) { + let mut sessions = self.sessions.lock().await; + sessions.clear(); + } } enum SessionStatus { diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index 42dadd96c9..d0e057102c 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -30,8 +30,8 @@ use codex_core::config::edit::ConfigEditsBuilder; #[cfg(target_os = "windows")] use codex_core::features::Feature; use codex_core::model_family::find_family_for_model; +use codex_core::protocol::EventMsg; use codex_core::protocol::FinalOutput; -#[cfg(target_os = "windows")] use codex_core::protocol::Op; use codex_core::protocol::SessionSource; use codex_core::protocol::TokenUsage; @@ -222,11 +222,23 @@ pub(crate) struct App { /// Set when the user confirms an update; propagated on exit. pub(crate) pending_update_action: Option, + /// Ignore the next ShutdownComplete event when we're intentionally + /// stopping a conversation (e.g., before starting a new one). + suppress_shutdown_complete: bool, + // One-shot suppression of the next world-writable scan after user confirmation. skip_world_writable_scan_once: bool, } impl App { + async fn shutdown_current_conversation(&mut self) { + if let Some(conversation_id) = self.chat_widget.conversation_id() { + self.suppress_shutdown_complete = true; + self.chat_widget.submit_op(Op::Shutdown); + self.server.remove_conversation(&conversation_id).await; + } + } + #[allow(clippy::too_many_arguments)] pub async fn run( tui: &mut tui::Tui, @@ -323,6 +335,7 @@ impl App { backtrack: BacktrackState::default(), feedback: feedback.clone(), pending_update_action: None, + suppress_shutdown_complete: false, skip_world_writable_scan_once: false, }; @@ -433,6 +446,7 @@ impl App { self.chat_widget.token_usage(), self.chat_widget.conversation_id(), ); + self.shutdown_current_conversation().await; let init = crate::chatwidget::ChatWidgetInit { config: self.config.clone(), frame_requester: tui.frame_requester(), @@ -503,6 +517,12 @@ impl App { self.chat_widget.on_commit_tick(); } AppEvent::CodexEvent(event) => { + if self.suppress_shutdown_complete + && matches!(event.msg, EventMsg::ShutdownComplete) + { + self.suppress_shutdown_complete = false; + return Ok(true); + } self.chat_widget.handle_codex_event(event); } AppEvent::ConversationHistory(ev) => { @@ -999,6 +1019,8 @@ mod tests { use codex_core::CodexAuth; use codex_core::ConversationManager; use codex_core::protocol::AskForApproval; + use codex_core::protocol::Event; + use codex_core::protocol::EventMsg; use codex_core::protocol::SandboxPolicy; use codex_core::protocol::SessionConfiguredEvent; use codex_protocol::ConversationId; @@ -1034,10 +1056,51 @@ mod tests { backtrack: BacktrackState::default(), feedback: codex_feedback::CodexFeedback::new(), pending_update_action: None, + suppress_shutdown_complete: false, skip_world_writable_scan_once: false, } } + fn make_test_app_with_channels() -> ( + App, + tokio::sync::mpsc::UnboundedReceiver, + tokio::sync::mpsc::UnboundedReceiver, + ) { + let (chat_widget, app_event_tx, rx, op_rx) = make_chatwidget_manual_with_sender(); + let config = chat_widget.config_ref().clone(); + let server = Arc::new(ConversationManager::with_auth(CodexAuth::from_api_key( + "Test API Key", + ))); + let auth_manager = + AuthManager::from_auth_for_testing(CodexAuth::from_api_key("Test API Key")); + let file_search = FileSearchManager::new(config.cwd.clone(), app_event_tx.clone()); + + ( + App { + server, + app_event_tx, + chat_widget, + auth_manager, + config, + active_profile: None, + file_search, + transcript_cells: Vec::new(), + overlay: None, + deferred_history_lines: Vec::new(), + has_emitted_history_lines: false, + enhanced_keys_supported: false, + commit_anim_running: Arc::new(AtomicBool::new(false)), + backtrack: BacktrackState::default(), + feedback: codex_feedback::CodexFeedback::new(), + pending_update_action: None, + suppress_shutdown_complete: false, + skip_world_writable_scan_once: false, + }, + rx, + op_rx, + ) + } + #[test] fn model_migration_prompt_only_shows_for_deprecated_models() { assert!(should_show_model_migration_prompt("gpt-5", "gpt-5.1", None)); @@ -1160,6 +1223,42 @@ mod tests { assert_eq!(prefill, "follow-up (edited)"); } + #[tokio::test] + async fn new_session_requests_shutdown_for_previous_conversation() { + let (mut app, mut app_event_rx, mut op_rx) = make_test_app_with_channels(); + + let conversation_id = ConversationId::new(); + let event = SessionConfiguredEvent { + session_id: conversation_id, + model: "gpt-test".to_string(), + model_provider_id: "test-provider".to_string(), + approval_policy: AskForApproval::Never, + sandbox_policy: SandboxPolicy::ReadOnly, + cwd: PathBuf::from("/home/user/project"), + reasoning_effort: None, + history_log_id: 0, + history_entry_count: 0, + initial_messages: None, + rollout_path: PathBuf::new(), + }; + + app.chat_widget.handle_codex_event(Event { + id: String::new(), + msg: EventMsg::SessionConfigured(event), + }); + + while app_event_rx.try_recv().is_ok() {} + while op_rx.try_recv().is_ok() {} + + app.shutdown_current_conversation().await; + + match op_rx.try_recv() { + Ok(Op::Shutdown) => {} + Ok(other) => panic!("expected Op::Shutdown, got {other:?}"), + Err(_) => panic!("expected shutdown op to be sent"), + } + } + #[test] fn session_summary_skip_zero_usage() { assert!(session_summary(TokenUsage::default(), None).is_none()); From 4502b1b263edb21988e76e52f5cde004ee98e0f1 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Tue, 25 Nov 2025 18:06:12 +0000 Subject: [PATCH 34/61] chore: proper client extraction (#6996) --- codex-rs/Cargo.lock | 43 +- codex-rs/Cargo.toml | 5 + .../app-server/tests/suite/send_message.rs | 63 +- codex-rs/client.md | 206 ++ codex-rs/codex-api/Cargo.toml | 30 + codex-rs/codex-api/README.md | 32 + codex-rs/codex-api/src/auth.rs | 27 + codex-rs/codex-api/src/common.rs | 167 ++ codex-rs/codex-api/src/endpoint/chat.rs | 266 +++ codex-rs/codex-api/src/endpoint/compact.rs | 162 ++ codex-rs/codex-api/src/endpoint/mod.rs | 4 + codex-rs/codex-api/src/endpoint/responses.rs | 107 + codex-rs/codex-api/src/endpoint/streaming.rs | 82 + codex-rs/codex-api/src/error.rs | 34 + codex-rs/codex-api/src/lib.rs | 35 + codex-rs/codex-api/src/provider.rs | 118 ++ codex-rs/codex-api/src/rate_limits.rs | 105 + codex-rs/codex-api/src/requests/chat.rs | 388 ++++ codex-rs/codex-api/src/requests/headers.rs | 36 + codex-rs/codex-api/src/requests/mod.rs | 8 + codex-rs/codex-api/src/requests/responses.rs | 247 +++ codex-rs/codex-api/src/sse/chat.rs | 504 +++++ codex-rs/codex-api/src/sse/mod.rs | 6 + codex-rs/codex-api/src/sse/responses.rs | 672 +++++++ codex-rs/codex-api/src/telemetry.rs | 84 + codex-rs/codex-api/tests/clients.rs | 315 +++ codex-rs/codex-api/tests/sse_end_to_end.rs | 229 +++ codex-rs/codex-client/Cargo.toml | 21 + codex-rs/codex-client/README.md | 8 + codex-rs/codex-client/src/error.rs | 29 + codex-rs/codex-client/src/lib.rs | 21 + codex-rs/codex-client/src/request.rs | 39 + codex-rs/codex-client/src/retry.rs | 73 + codex-rs/codex-client/src/sse.rs | 48 + codex-rs/codex-client/src/telemetry.rs | 14 + codex-rs/codex-client/src/transport.rs | 107 + codex-rs/core/Cargo.toml | 2 +- codex-rs/core/src/api_bridge.rs | 154 ++ codex-rs/core/src/chat_completions.rs | 981 --------- codex-rs/core/src/client.rs | 1755 +++-------------- codex-rs/core/src/client_common.rs | 128 +- codex-rs/core/src/default_client.rs | 8 +- codex-rs/core/src/lib.rs | 2 +- codex-rs/core/src/model_provider_info.rs | 263 +-- codex-rs/otel/src/otel_event_manager.rs | 13 +- 45 files changed, 4893 insertions(+), 2748 deletions(-) create mode 100644 codex-rs/client.md create mode 100644 codex-rs/codex-api/Cargo.toml create mode 100644 codex-rs/codex-api/README.md create mode 100644 codex-rs/codex-api/src/auth.rs create mode 100644 codex-rs/codex-api/src/common.rs create mode 100644 codex-rs/codex-api/src/endpoint/chat.rs create mode 100644 codex-rs/codex-api/src/endpoint/compact.rs create mode 100644 codex-rs/codex-api/src/endpoint/mod.rs create mode 100644 codex-rs/codex-api/src/endpoint/responses.rs create mode 100644 codex-rs/codex-api/src/endpoint/streaming.rs create mode 100644 codex-rs/codex-api/src/error.rs create mode 100644 codex-rs/codex-api/src/lib.rs create mode 100644 codex-rs/codex-api/src/provider.rs create mode 100644 codex-rs/codex-api/src/rate_limits.rs create mode 100644 codex-rs/codex-api/src/requests/chat.rs create mode 100644 codex-rs/codex-api/src/requests/headers.rs create mode 100644 codex-rs/codex-api/src/requests/mod.rs create mode 100644 codex-rs/codex-api/src/requests/responses.rs create mode 100644 codex-rs/codex-api/src/sse/chat.rs create mode 100644 codex-rs/codex-api/src/sse/mod.rs create mode 100644 codex-rs/codex-api/src/sse/responses.rs create mode 100644 codex-rs/codex-api/src/telemetry.rs create mode 100644 codex-rs/codex-api/tests/clients.rs create mode 100644 codex-rs/codex-api/tests/sse_end_to_end.rs create mode 100644 codex-rs/codex-client/Cargo.toml create mode 100644 codex-rs/codex-client/README.md create mode 100644 codex-rs/codex-client/src/error.rs create mode 100644 codex-rs/codex-client/src/lib.rs create mode 100644 codex-rs/codex-client/src/request.rs create mode 100644 codex-rs/codex-client/src/retry.rs create mode 100644 codex-rs/codex-client/src/sse.rs create mode 100644 codex-rs/codex-client/src/telemetry.rs create mode 100644 codex-rs/codex-client/src/transport.rs create mode 100644 codex-rs/core/src/api_bridge.rs delete mode 100644 codex-rs/core/src/chat_completions.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 800ef1ce73..34ea8c8773 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -843,6 +843,30 @@ dependencies = [ "tracing", ] +[[package]] +name = "codex-api" +version = "0.0.0" +dependencies = [ + "anyhow", + "assert_matches", + "async-trait", + "bytes", + "codex-client", + "codex-protocol", + "eventsource-stream", + "futures", + "http", + "pretty_assertions", + "regex-lite", + "serde", + "serde_json", + "thiserror 2.0.17", + "tokio", + "tokio-test", + "tokio-util", + "tracing", +] + [[package]] name = "codex-app-server" version = "0.0.0" @@ -1029,6 +1053,23 @@ dependencies = [ "tracing", ] +[[package]] +name = "codex-client" +version = "0.0.0" +dependencies = [ + "async-trait", + "bytes", + "eventsource-stream", + "futures", + "http", + "rand 0.9.2", + "reqwest", + "serde", + "serde_json", + "thiserror 2.0.17", + "tokio", +] + [[package]] name = "codex-cloud-tasks" version = "0.0.0" @@ -1096,9 +1137,9 @@ dependencies = [ "async-channel", "async-trait", "base64", - "bytes", "chardetng", "chrono", + "codex-api", "codex-app-server-protocol", "codex-apply-patch", "codex-arg0", diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index 7d57b62ec1..053d79d7e0 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -41,6 +41,8 @@ members = [ "utils/pty", "utils/readiness", "utils/string", + "codex-client", + "codex-api", ] resolver = "2" @@ -63,6 +65,8 @@ codex-apply-patch = { path = "apply-patch" } codex-arg0 = { path = "arg0" } codex-async-utils = { path = "async-utils" } codex-backend-client = { path = "backend-client" } +codex-api = { path = "codex-api" } +codex-client = { path = "codex-client" } codex-chatgpt = { path = "chatgpt" } codex-common = { path = "common" } codex-core = { path = "core" } @@ -171,6 +175,7 @@ reqwest = "0.12" rmcp = { version = "0.9.0", default-features = false } schemars = "0.8.22" seccompiler = "0.5.0" +sentry = "0.34.0" serde = "1" serde_json = "1" serde_with = "3.14" diff --git a/codex-rs/app-server/tests/suite/send_message.rs b/codex-rs/app-server/tests/suite/send_message.rs index 8d2b36af2f..39b3a31a8a 100644 --- a/codex-rs/app-server/tests/suite/send_message.rs +++ b/codex-rs/app-server/tests/suite/send_message.rs @@ -272,40 +272,45 @@ async fn read_raw_response_item( mcp: &mut McpProcess, conversation_id: ConversationId, ) -> ResponseItem { - let raw_notification: JSONRPCNotification = timeout( - DEFAULT_READ_TIMEOUT, - mcp.read_stream_until_notification_message("codex/event/raw_response_item"), - ) - .await - .expect("codex/event/raw_response_item notification timeout") - .expect("codex/event/raw_response_item notification resp"); + loop { + let raw_notification: JSONRPCNotification = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("codex/event/raw_response_item"), + ) + .await + .expect("codex/event/raw_response_item notification timeout") + .expect("codex/event/raw_response_item notification resp"); - let serde_json::Value::Object(params) = raw_notification - .params - .expect("codex/event/raw_response_item should have params") - else { - panic!("codex/event/raw_response_item should have params"); - }; + let serde_json::Value::Object(params) = raw_notification + .params + .expect("codex/event/raw_response_item should have params") + else { + panic!("codex/event/raw_response_item should have params"); + }; - let conversation_id_value = params - .get("conversationId") - .and_then(|value| value.as_str()) - .expect("raw response item should include conversationId"); + let conversation_id_value = params + .get("conversationId") + .and_then(|value| value.as_str()) + .expect("raw response item should include conversationId"); - assert_eq!( - conversation_id_value, - conversation_id.to_string(), - "raw response item conversation mismatch" - ); + assert_eq!( + conversation_id_value, + conversation_id.to_string(), + "raw response item conversation mismatch" + ); - let msg_value = params - .get("msg") - .cloned() - .expect("raw response item should include msg payload"); + let msg_value = params + .get("msg") + .cloned() + .expect("raw response item should include msg payload"); - let event: RawResponseItemEvent = - serde_json::from_value(msg_value).expect("deserialize raw response item"); - event.item + // Ghost snapshots are produced concurrently and may arrive before the model reply. + let event: RawResponseItemEvent = + serde_json::from_value(msg_value).expect("deserialize raw response item"); + if !matches!(event.item, ResponseItem::GhostSnapshot { .. }) { + return event.item; + } + } } fn assert_instructions_message(item: &ResponseItem) { diff --git a/codex-rs/client.md b/codex-rs/client.md new file mode 100644 index 0000000000..4c9027ccba --- /dev/null +++ b/codex-rs/client.md @@ -0,0 +1,206 @@ +# Client Extraction Plan + +## Goals +- Split the HTTP transport/client code out of `codex-core` into a reusable crate that is agnostic of Codex/OpenAI business logic and API schemas. +- Create a separate API library crate that houses typed requests/responses for well-known APIs (Responses, Chat Completions, Compact) and plugs into the transport crate via minimal traits. +- Preserve current behaviour (auth headers, retries, SSE handling, rate-limit parsing, compaction, fixtures) while making the APIs symmetric and avoiding code duplication. +- Keep existing consumers (`codex-core`, tests, and tools) stable by providing a small compatibility layer during the transition. + +## Snapshot of Today +- `core/src/client.rs (ModelClient)` owns config/auth/session state, chooses wire API, builds payloads, drives retries, parses SSE, compaction, and rate-limit headers. +- `core/src/chat_completions.rs` implements the Chat Completions call + SSE parser + aggregation helper. +- `core/src/client_common.rs` holds `Prompt`, tool specs, shared request structs (`ResponsesApiRequest`, `TextControls`), and `ResponseEvent`/`ResponseStream`. +- `core/src/default_client.rs` wraps `reqwest` with Codex UA/originator defaults. +- `core/src/model_provider_info.rs` models providers (base URL, headers, env keys, retry/timeout tuning) and builds `CodexRequestBuilder`s. + - Current retry logic is co-located with API handling; streaming SSE parsing is duplicated across Responses/Chat. + +## Target Crates (with interfaces) + +- `codex-client` (generic transport) + - Owns the generic HTTP machinery: a `CodexHttpClient`/`CodexRequestBuilder`-style wrapper, retry/backoff hooks, streaming connector (SSE framing + idle timeout), header injection, and optional telemetry callbacks. + - Does **not** know about OpenAI/Codex-specific paths, headers, or error codes; it only exposes HTTP-level concepts (status, headers, bodies, connection errors). + - Minimal surface: + ```rust + pub trait HttpTransport { + fn execute(&self, req: Request) -> Result; + fn stream(&self, req: Request) -> Result; + } + + pub struct Request { + pub method: Method, + pub url: String, + pub headers: HeaderMap, + pub body: Option, + pub timeout: Option, + } + ``` + - Generic client traits (request/response/chunk are abstract over the transport): + ```rust + #[async_trait::async_trait] + pub trait UnaryClient { + async fn run(&self, req: Req) -> Result; + } + + #[async_trait::async_trait] + pub trait StreamClient { + async fn run(&self, req: Req) -> Result, TransportError>; + } + + pub struct RetryPolicy { + pub max_attempts: u64, + pub base_delay: Duration, + pub retry_on: RetryOn, // e.g., transport errors + 429/5xx + } + ``` + - `RetryOn` lives in `codex-client` and captures HTTP status classes and transport failures that qualify for retry. + - Implementations in `codex-api` plug in their own request types, parsers, and retry policies while reusing the transport’s backoff and error types. + - Planned runtime helper: + ```rust + pub async fn run_with_retry( + policy: RetryPolicy, + make_req: impl Fn() -> Request, + op: F, + ) -> Result + where + F: Fn(Request) -> Fut, + Fut: Future>, + { + for attempt in 0..=policy.max_attempts { + let req = make_req(); + match op(req).await { + Ok(resp) => return Ok(resp), + Err(err) if policy.retry_on.should_retry(&err, attempt) => { + tokio::time::sleep(backoff(policy.base_delay, attempt + 1)).await; + } + Err(err) => return Err(err), + } + } + Err(TransportError::RetryLimit) + } + ``` + - Unary clients wrap `transport.execute` with this helper and then deserialize. + - Stream clients wrap the **initial** `transport.stream` call with this helper. Mid-stream disconnects are surfaced as `StreamError`s; automatic resume/reconnect can be added later on top of this primitive if we introduce cursor support. + - Common helpers: `retry::backoff(attempt)`, `errors::{TransportError, StreamError}`. + - Streaming utility (SSE framing only): + ```rust + pub fn sse_stream( + bytes: S, + idle_timeout: Duration, + tx: mpsc::Sender>, + telemetry: Option>, + ) + where + S: Stream> + Unpin + Send + 'static; + ``` + - `sse_stream` is responsible for timeouts, connection-level errors, and emitting raw `data:` chunks as UTF-8 strings; parsing those strings into structured events is done in `codex-api`. + +- `codex-api` (OpenAI/Codex API library) + - Owns typed models for Responses/Chat/Compact plus shared helpers (`Prompt`, tool specs, text controls, `ResponsesApiRequest`, etc.). + - Knows about OpenAI/Codex semantics: + - URL shapes (`/v1/responses`, `/v1/chat/completions`, `/responses/compact`). + - Provider configuration (`WireApi`, base URLs, query params, per-provider retry knobs). + - Rate-limit headers (`x-codex-*`) and their mapping into `RateLimitSnapshot` / `CreditsSnapshot`. + - Error body formats (`{ error: { type, code, message, plan_type, resets_at } }`) and how they become API errors (context window exceeded, quota/usage limit, etc.). + - SSE event names (`response.output_item.done`, `response.completed`, `response.failed`, etc.) and their mapping into high-level events. + - Provides a provider abstraction (conceptually similar to `ModelProviderInfo`): + ```rust + pub struct Provider { + pub name: String, + pub base_url: String, + pub wire: WireApi, // Responses | Chat + pub headers: HeaderMap, + pub retry: RetryConfig, + pub stream_idle_timeout: Duration, + } + + pub trait AuthProvider { + /// Returns a bearer token to use for this request (if any). + /// Implementations are expected to be cheap and to surface already-refreshed tokens; + /// higher layers (`codex-core`) remain responsible for token refresh flows. + fn bearer_token(&self) -> Option; + + /// Optional ChatGPT account id header for Chat mode. + fn account_id(&self) -> Option; + } + ``` + - Ready-made clients built on `HttpTransport`: + ```rust + pub struct ResponsesClient { /* ... */ } + impl ResponsesClient { + pub async fn stream(&self, prompt: &Prompt) -> ApiResult>; + pub async fn compact(&self, prompt: &Prompt) -> ApiResult>; + } + + pub struct ChatClient { /* ... */ } + impl ChatClient { + pub async fn stream(&self, prompt: &Prompt) -> ApiResult>; + } + + pub struct CompactClient { /* ... */ } + impl CompactClient { + pub async fn compact(&self, prompt: &Prompt) -> ApiResult>; + } + ``` + - Streaming events unified across wire APIs (this can closely mirror `ResponseEvent` today, and we may type-alias one to the other during migration): + ```rust + pub enum ApiEvent { + Created, + OutputItemAdded(ResponseItem), + OutputItemDone(ResponseItem), + OutputTextDelta(String), + ReasoningContentDelta { delta: String, content_index: i64 }, + ReasoningSummaryDelta { delta: String, summary_index: i64 }, + RateLimits(RateLimitSnapshot), + Completed { response_id: String, token_usage: Option }, + } + ``` + - Error layering: + - `codex-client`: defines `TransportError` / `StreamError` (status codes, IO, timeouts). + - `codex-api`: defines `ApiError` that wraps `TransportError` plus API-specific errors parsed from bodies and headers. + - `codex-core`: maps `ApiError` into existing `CodexErr` variants so downstream callers remain unchanged. + - Aggregation strategies (today’s `AggregateStreamExt`) live here as adapters (`Aggregated`, `Streaming`) that transform `ResponseStream` into the higher-level views used by `codex-core`. + +## Implementation Steps + +1. **Create crates**: add `codex-client` and `codex-api` (names keep the `codex-` prefix). Stub lib files with feature flags/tests wired into the workspace; wire them into `Cargo.toml`. +2. **Extract API-level SSE + rate limits into `codex-api`**: + - Move the Responses SSE parser (`process_sse`), rate-limit parsing, and related tests from `core/src/client.rs` into `codex-api`, keeping the behavior identical. + - Introduce `ApiEvent` (initially equivalent to `ResponseEvent`) and `ApiError`, and adjust the parser to emit those. + - Provide test-only helpers for fixture streams (replacement for `CODEX_RS_SSE_FIXTURE`) in `codex-api`. +3. **Lift transport layer into `codex-client`**: + - Move `CodexHttpClient`/`CodexRequestBuilder`, UA/originator plumbing, and backoff helpers from `core/src/default_client.rs` into `codex-client` (or a thin wrapper on top of it). + - Introduce `HttpTransport`, `Request`, `RetryPolicy`, `RetryOn`, and `run_with_retry` as described above. + - Keep sandbox/no-proxy toggles behind injected configuration so `codex-client` stays generic and does not depend on Codex-specific env vars. +4. **Model provider abstraction in `codex-api`**: + - Relocate `ModelProviderInfo` (base URL, env/header resolution, retry knobs, wire API enum) into `codex-api`, expressed in terms of `Provider` and `AuthProvider`. + - Ensure provider logic handles: + - URL building for Responses/Chat/Compact (including Azure special cases). + - Static and env-based headers. + - Per-provider retry and idle-timeout settings that map cleanly into `RetryPolicy`/`RetryOn`. +5. **API crate wiring**: + - Move `Prompt`, tool specs, `ResponsesApiRequest`, `TextControls`, and `ResponseEvent/ResponseStream` into `codex-api` under modules (`common`, `responses`, `chat`, `compact`), keeping public types stable or re-exported through `codex-core` as needed. + - Rebuild Responses and Chat clients on top of `HttpTransport` + `StreamClient`, reusing shared retry + SSE helpers; keep aggregation adapters as reusable strategies instead of `ModelClient`-local logic. + - Implement Compact on top of `UnaryClient` and the unary `execute` path with JSON deserialization, sharing the same retry policy. + - Keep request builders symmetric: each client prepares a `Request`, attaches headers/auth via `AuthProvider`, and plugs in its parser (streaming clients) or deserializer (unary) while sharing retry/backoff configuration derived from `Provider`. +6. **Core integration layer**: + - Replace `core::ModelClient` internals with thin adapters that construct `codex-api` clients using `Config`, `AuthManager`, and `OtelEventManager`. + - Keep the public `ModelClient` API and `ResponseEvent`/`ResponseStream` types stable by re-exporting `codex-api` types or providing type aliases. + - Preserve existing auth flows (including ChatGPT token refresh) inside `codex-core` or a thin adapter, using `AuthProvider` to surface bearer tokens to `codex-api` and handling 401/refresh semantics at this layer. +7. **Tests/migration**: + - Move unit tests for SSE parsing, retry/backoff decisions, and provider/header behavior into the new crates; keep integration tests in `core` using the compatibility layer. + - Update fixtures to be consumed via test-only adapters in `codex-api`. + - Run targeted `just fmt`, `just fix -p` for the touched crates, and scoped `cargo test -p codex-client`, `-p codex-api`, and existing `codex-core` suites. + +## Design Decisions + +- **UA construction** + - `codex-client` exposes an optional UA suffix/provider hook (tiny feature) and remains unaware of the CLI; `codex-core` / the CLI compute the full UA (including `terminal::user_agent()`) and pass the suffix or builder down. +- **Config vs provider** + - Most configuration stays in `codex-core`. `codex-api::Provider` only contains what is strictly required for HTTP (base URLs, query params, retry/timeout knobs, wire API), while higher-level knobs (reasoning defaults, verbosity flags, etc.) remain core concerns. +- **Auth flow ownership** + - Auth flows (including ChatGPT token refresh) remain in `codex-core`. `AuthProvider` simply exposes already-fresh tokens/account IDs; 401 handling and refresh retries stay in the existing auth layer. +- **Error enums** + - `codex-client` continues to define `TransportError` / `StreamError`. `codex-api` defines an `ApiError` (deriving `thiserror::Error`) that wraps `TransportError` and API-specific failures, and `codex-core` maps `ApiError` into existing `CodexErr` variants for callers. +- **Streaming reconnection semantics** + - For now, mid-stream SSE failures are surfaced as errors and only the initial connection is retried via `run_with_retry`. We will revisit mid-stream reconnect/resume once the underlying APIs support cursor/idempotent event semantics. + diff --git a/codex-rs/codex-api/Cargo.toml b/codex-rs/codex-api/Cargo.toml new file mode 100644 index 0000000000..f79416c96e --- /dev/null +++ b/codex-rs/codex-api/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "codex-api" +version.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +async-trait = { workspace = true } +bytes = { workspace = true } +codex-client = { workspace = true } +codex-protocol = { workspace = true } +futures = { workspace = true } +http = { workspace = true } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["macros", "rt", "sync", "time"] } +tracing = { workspace = true } +eventsource-stream = { workspace = true } +regex-lite = { workspace = true } +tokio-util = { workspace = true, features = ["codec"] } + +[dev-dependencies] +anyhow = { workspace = true } +assert_matches = { workspace = true } +pretty_assertions = { workspace = true } +tokio-test = { workspace = true } + +[lints] +workspace = true diff --git a/codex-rs/codex-api/README.md b/codex-rs/codex-api/README.md new file mode 100644 index 0000000000..98db0bec65 --- /dev/null +++ b/codex-rs/codex-api/README.md @@ -0,0 +1,32 @@ +# codex-api + +Typed clients for Codex/OpenAI APIs built on top of the generic transport in `codex-client`. + +- Hosts the request/response models and prompt helpers for Responses, Chat Completions, and Compact APIs. +- Owns provider configuration (base URLs, headers, query params), auth header injection, retry tuning, and stream idle settings. +- Parses SSE streams into `ResponseEvent`/`ResponseStream`, including rate-limit snapshots and API-specific error mapping. +- Serves as the wire-level layer consumed by `codex-core`; higher layers handle auth refresh and business logic. + +## Core interface + +The public interface of this crate is intentionally small and uniform: + +- **Prompted endpoints (Chat + Responses)** + - Input: a single `Prompt` plus endpoint-specific options. + - `Prompt` (re-exported as `codex_api::Prompt`) carries: + - `instructions: String` – the fully-resolved system prompt for this turn. + - `input: Vec` – conversation history and user/tool messages. + - `tools: Vec` – JSON tools compatible with the target API. + - `parallel_tool_calls: bool`. + - `output_schema: Option` – used to build `text.format` when present. + - Output: a `ResponseStream` of `ResponseEvent` (both re-exported from `common`). + +- **Compaction endpoint** + - Input: `CompactionInput<'a>` (re-exported as `codex_api::CompactionInput`): + - `model: &str`. + - `input: &[ResponseItem]` – history to compact. + - `instructions: &str` – fully-resolved compaction instructions. + - Output: `Vec`. + - `CompactClient::compact_input(&CompactionInput, extra_headers)` wraps the JSON encoding and retry/telemetry wiring. + +All HTTP details (URLs, headers, retry/backoff policies, SSE framing) are encapsulated in `codex-api` and `codex-client`. Callers construct prompts/inputs using protocol types and work with typed streams of `ResponseEvent` or compacted `ResponseItem` values. diff --git a/codex-rs/codex-api/src/auth.rs b/codex-rs/codex-api/src/auth.rs new file mode 100644 index 0000000000..6c26963cba --- /dev/null +++ b/codex-rs/codex-api/src/auth.rs @@ -0,0 +1,27 @@ +use codex_client::Request; + +/// Provides bearer and account identity information for API requests. +/// +/// Implementations should be cheap and non-blocking; any asynchronous +/// refresh or I/O should be handled by higher layers before requests +/// reach this interface. +pub trait AuthProvider: Send + Sync { + fn bearer_token(&self) -> Option; + fn account_id(&self) -> Option { + None + } +} + +pub(crate) fn add_auth_headers(auth: &A, mut req: Request) -> Request { + if let Some(token) = auth.bearer_token() + && let Ok(header) = format!("Bearer {token}").parse() + { + let _ = req.headers.insert(http::header::AUTHORIZATION, header); + } + if let Some(account_id) = auth.account_id() + && let Ok(header) = account_id.parse() + { + let _ = req.headers.insert("ChatGPT-Account-ID", header); + } + req +} diff --git a/codex-rs/codex-api/src/common.rs b/codex-rs/codex-api/src/common.rs new file mode 100644 index 0000000000..addab02dc7 --- /dev/null +++ b/codex-rs/codex-api/src/common.rs @@ -0,0 +1,167 @@ +use crate::error::ApiError; +use codex_protocol::config_types::ReasoningEffort as ReasoningEffortConfig; +use codex_protocol::config_types::ReasoningSummary as ReasoningSummaryConfig; +use codex_protocol::config_types::Verbosity as VerbosityConfig; +use codex_protocol::models::ResponseItem; +use codex_protocol::protocol::RateLimitSnapshot; +use codex_protocol::protocol::TokenUsage; +use futures::Stream; +use serde::Serialize; +use serde_json::Value; +use std::pin::Pin; +use std::task::Context; +use std::task::Poll; +use tokio::sync::mpsc; + +/// Canonical prompt input for Chat and Responses endpoints. +#[derive(Debug, Clone)] +pub struct Prompt { + /// Fully-resolved system instructions for this turn. + pub instructions: String, + /// Conversation history and user/tool messages. + pub input: Vec, + /// JSON-encoded tool definitions compatible with the target API. + // TODO(jif) have a proper type here + pub tools: Vec, + /// Whether parallel tool calls are permitted. + pub parallel_tool_calls: bool, + /// Optional output schema used to build the `text.format` controls. + pub output_schema: Option, +} + +/// Canonical input payload for the compaction endpoint. +#[derive(Debug, Clone, Serialize)] +pub struct CompactionInput<'a> { + pub model: &'a str, + pub input: &'a [ResponseItem], + pub instructions: &'a str, +} + +#[derive(Debug)] +pub enum ResponseEvent { + Created, + OutputItemDone(ResponseItem), + OutputItemAdded(ResponseItem), + Completed { + response_id: String, + token_usage: Option, + }, + OutputTextDelta(String), + ReasoningSummaryDelta { + delta: String, + summary_index: i64, + }, + ReasoningContentDelta { + delta: String, + content_index: i64, + }, + ReasoningSummaryPartAdded { + summary_index: i64, + }, + RateLimits(RateLimitSnapshot), +} + +#[derive(Debug, Serialize, Clone)] +pub struct Reasoning { + #[serde(skip_serializing_if = "Option::is_none")] + pub effort: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub summary: Option, +} + +#[derive(Debug, Serialize, Default, Clone)] +#[serde(rename_all = "snake_case")] +pub enum TextFormatType { + #[default] + JsonSchema, +} + +#[derive(Debug, Serialize, Default, Clone)] +pub struct TextFormat { + /// Format type used by the OpenAI text controls. + pub r#type: TextFormatType, + /// When true, the server is expected to strictly validate responses. + pub strict: bool, + /// JSON schema for the desired output. + pub schema: Value, + /// Friendly name for the format, used in telemetry/debugging. + pub name: String, +} + +/// Controls the `text` field for the Responses API, combining verbosity and +/// optional JSON schema output formatting. +#[derive(Debug, Serialize, Default, Clone)] +pub struct TextControls { + #[serde(skip_serializing_if = "Option::is_none")] + pub verbosity: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub format: Option, +} + +#[derive(Debug, Serialize, Default, Clone)] +#[serde(rename_all = "lowercase")] +pub enum OpenAiVerbosity { + Low, + #[default] + Medium, + High, +} + +impl From for OpenAiVerbosity { + fn from(v: VerbosityConfig) -> Self { + match v { + VerbosityConfig::Low => OpenAiVerbosity::Low, + VerbosityConfig::Medium => OpenAiVerbosity::Medium, + VerbosityConfig::High => OpenAiVerbosity::High, + } + } +} + +#[derive(Debug, Serialize)] +pub struct ResponsesApiRequest<'a> { + pub model: &'a str, + pub instructions: &'a str, + pub input: &'a [ResponseItem], + pub tools: &'a [serde_json::Value], + pub tool_choice: &'static str, + pub parallel_tool_calls: bool, + pub reasoning: Option, + pub store: bool, + pub stream: bool, + pub include: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub prompt_cache_key: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub text: Option, +} + +pub fn create_text_param_for_request( + verbosity: Option, + output_schema: &Option, +) -> Option { + if verbosity.is_none() && output_schema.is_none() { + return None; + } + + Some(TextControls { + verbosity: verbosity.map(std::convert::Into::into), + format: output_schema.as_ref().map(|schema| TextFormat { + r#type: TextFormatType::JsonSchema, + strict: true, + schema: schema.clone(), + name: "codex_output_schema".to_string(), + }), + }) +} + +pub struct ResponseStream { + pub rx_event: mpsc::Receiver>, +} + +impl Stream for ResponseStream { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.rx_event.poll_recv(cx) + } +} diff --git a/codex-rs/codex-api/src/endpoint/chat.rs b/codex-rs/codex-api/src/endpoint/chat.rs new file mode 100644 index 0000000000..4ad133dda4 --- /dev/null +++ b/codex-rs/codex-api/src/endpoint/chat.rs @@ -0,0 +1,266 @@ +use crate::ChatRequest; +use crate::auth::AuthProvider; +use crate::common::Prompt as ApiPrompt; +use crate::common::ResponseEvent; +use crate::common::ResponseStream; +use crate::endpoint::streaming::StreamingClient; +use crate::error::ApiError; +use crate::provider::Provider; +use crate::provider::WireApi; +use crate::sse::chat::spawn_chat_stream; +use crate::telemetry::SseTelemetry; +use codex_client::HttpTransport; +use codex_client::RequestTelemetry; +use codex_protocol::models::ContentItem; +use codex_protocol::models::ReasoningItemContent; +use codex_protocol::models::ResponseItem; +use codex_protocol::protocol::SessionSource; +use futures::Stream; +use http::HeaderMap; +use serde_json::Value; +use std::collections::VecDeque; +use std::pin::Pin; +use std::sync::Arc; +use std::task::Context; +use std::task::Poll; + +pub struct ChatClient { + streaming: StreamingClient, +} + +impl ChatClient { + pub fn new(transport: T, provider: Provider, auth: A) -> Self { + Self { + streaming: StreamingClient::new(transport, provider, auth), + } + } + + pub fn with_telemetry( + self, + request: Option>, + sse: Option>, + ) -> Self { + Self { + streaming: self.streaming.with_telemetry(request, sse), + } + } + + pub async fn stream_request(&self, request: ChatRequest) -> Result { + self.stream(request.body, request.headers).await + } + + pub async fn stream_prompt( + &self, + model: &str, + prompt: &ApiPrompt, + conversation_id: Option, + session_source: Option, + ) -> Result { + use crate::requests::ChatRequestBuilder; + + let request = + ChatRequestBuilder::new(model, &prompt.instructions, &prompt.input, &prompt.tools) + .conversation_id(conversation_id) + .session_source(session_source) + .build(self.streaming.provider())?; + + self.stream_request(request).await + } + + fn path(&self) -> &'static str { + match self.streaming.provider().wire { + WireApi::Chat => "chat/completions", + _ => "responses", + } + } + + pub async fn stream( + &self, + body: Value, + extra_headers: HeaderMap, + ) -> Result { + self.streaming + .stream(self.path(), body, extra_headers, spawn_chat_stream) + .await + } +} + +#[derive(Copy, Clone, Eq, PartialEq)] +pub enum AggregateMode { + AggregatedOnly, + Streaming, +} + +/// Stream adapter that merges token deltas into a single assistant message per turn. +pub struct AggregatedStream { + inner: ResponseStream, + cumulative: String, + cumulative_reasoning: String, + pending: VecDeque, + mode: AggregateMode, +} + +impl Stream for AggregatedStream { + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.get_mut(); + + if let Some(ev) = this.pending.pop_front() { + return Poll::Ready(Some(Ok(ev))); + } + + loop { + match Pin::new(&mut this.inner).poll_next(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(None) => return Poll::Ready(None), + Poll::Ready(Some(Err(e))) => return Poll::Ready(Some(Err(e))), + Poll::Ready(Some(Ok(ResponseEvent::OutputItemDone(item)))) => { + let is_assistant_message = matches!( + &item, + ResponseItem::Message { role, .. } if role == "assistant" + ); + + if is_assistant_message { + match this.mode { + AggregateMode::AggregatedOnly => { + if this.cumulative.is_empty() + && let ResponseItem::Message { content, .. } = &item + && let Some(text) = content.iter().find_map(|c| match c { + ContentItem::OutputText { text } => Some(text), + _ => None, + }) + { + this.cumulative.push_str(text); + } + continue; + } + AggregateMode::Streaming => { + if this.cumulative.is_empty() { + return Poll::Ready(Some(Ok(ResponseEvent::OutputItemDone( + item, + )))); + } else { + continue; + } + } + } + } + + return Poll::Ready(Some(Ok(ResponseEvent::OutputItemDone(item)))); + } + Poll::Ready(Some(Ok(ResponseEvent::RateLimits(snapshot)))) => { + return Poll::Ready(Some(Ok(ResponseEvent::RateLimits(snapshot)))); + } + Poll::Ready(Some(Ok(ResponseEvent::Completed { + response_id, + token_usage, + }))) => { + let mut emitted_any = false; + + if !this.cumulative_reasoning.is_empty() { + let aggregated_reasoning = ResponseItem::Reasoning { + id: String::new(), + summary: Vec::new(), + content: Some(vec![ReasoningItemContent::ReasoningText { + text: std::mem::take(&mut this.cumulative_reasoning), + }]), + encrypted_content: None, + }; + this.pending + .push_back(ResponseEvent::OutputItemDone(aggregated_reasoning)); + emitted_any = true; + } + + if !this.cumulative.is_empty() { + let aggregated_message = ResponseItem::Message { + id: None, + role: "assistant".to_string(), + content: vec![ContentItem::OutputText { + text: std::mem::take(&mut this.cumulative), + }], + }; + this.pending + .push_back(ResponseEvent::OutputItemDone(aggregated_message)); + emitted_any = true; + } + + if emitted_any { + this.pending.push_back(ResponseEvent::Completed { + response_id: response_id.clone(), + token_usage: token_usage.clone(), + }); + if let Some(ev) = this.pending.pop_front() { + return Poll::Ready(Some(Ok(ev))); + } + } + + return Poll::Ready(Some(Ok(ResponseEvent::Completed { + response_id, + token_usage, + }))); + } + Poll::Ready(Some(Ok(ResponseEvent::Created))) => { + continue; + } + Poll::Ready(Some(Ok(ResponseEvent::OutputTextDelta(delta)))) => { + this.cumulative.push_str(&delta); + if matches!(this.mode, AggregateMode::Streaming) { + return Poll::Ready(Some(Ok(ResponseEvent::OutputTextDelta(delta)))); + } else { + continue; + } + } + Poll::Ready(Some(Ok(ResponseEvent::ReasoningContentDelta { + delta, + content_index, + }))) => { + this.cumulative_reasoning.push_str(&delta); + if matches!(this.mode, AggregateMode::Streaming) { + return Poll::Ready(Some(Ok(ResponseEvent::ReasoningContentDelta { + delta, + content_index, + }))); + } else { + continue; + } + } + Poll::Ready(Some(Ok(ResponseEvent::ReasoningSummaryDelta { .. }))) => continue, + Poll::Ready(Some(Ok(ResponseEvent::ReasoningSummaryPartAdded { .. }))) => { + continue; + } + Poll::Ready(Some(Ok(ResponseEvent::OutputItemAdded(item)))) => { + return Poll::Ready(Some(Ok(ResponseEvent::OutputItemAdded(item)))); + } + } + } + } +} + +pub trait AggregateStreamExt { + fn aggregate(self) -> AggregatedStream; + + fn streaming_mode(self) -> ResponseStream; +} + +impl AggregateStreamExt for ResponseStream { + fn aggregate(self) -> AggregatedStream { + AggregatedStream::new(self, AggregateMode::AggregatedOnly) + } + + fn streaming_mode(self) -> ResponseStream { + self + } +} + +impl AggregatedStream { + fn new(inner: ResponseStream, mode: AggregateMode) -> Self { + AggregatedStream { + inner, + cumulative: String::new(), + cumulative_reasoning: String::new(), + pending: VecDeque::new(), + mode, + } + } +} diff --git a/codex-rs/codex-api/src/endpoint/compact.rs b/codex-rs/codex-api/src/endpoint/compact.rs new file mode 100644 index 0000000000..2b02ebd0f0 --- /dev/null +++ b/codex-rs/codex-api/src/endpoint/compact.rs @@ -0,0 +1,162 @@ +use crate::auth::AuthProvider; +use crate::auth::add_auth_headers; +use crate::common::CompactionInput; +use crate::error::ApiError; +use crate::provider::Provider; +use crate::provider::WireApi; +use crate::telemetry::run_with_request_telemetry; +use codex_client::HttpTransport; +use codex_client::RequestTelemetry; +use codex_protocol::models::ResponseItem; +use http::HeaderMap; +use http::Method; +use serde::Deserialize; +use serde_json::to_value; +use std::sync::Arc; + +pub struct CompactClient { + transport: T, + provider: Provider, + auth: A, + request_telemetry: Option>, +} + +impl CompactClient { + pub fn new(transport: T, provider: Provider, auth: A) -> Self { + Self { + transport, + provider, + auth, + request_telemetry: None, + } + } + + pub fn with_telemetry(mut self, request: Option>) -> Self { + self.request_telemetry = request; + self + } + + fn path(&self) -> Result<&'static str, ApiError> { + match self.provider.wire { + WireApi::Compact | WireApi::Responses => Ok("responses/compact"), + WireApi::Chat => Err(ApiError::Stream( + "compact endpoint requires responses wire api".to_string(), + )), + } + } + + pub async fn compact( + &self, + body: serde_json::Value, + extra_headers: HeaderMap, + ) -> Result, ApiError> { + let path = self.path()?; + let builder = || { + let mut req = self.provider.build_request(Method::POST, path); + req.headers.extend(extra_headers.clone()); + req.body = Some(body.clone()); + add_auth_headers(&self.auth, req) + }; + + let resp = run_with_request_telemetry( + self.provider.retry.to_policy(), + self.request_telemetry.clone(), + builder, + |req| self.transport.execute(req), + ) + .await?; + let parsed: CompactHistoryResponse = + serde_json::from_slice(&resp.body).map_err(|e| ApiError::Stream(e.to_string()))?; + Ok(parsed.output) + } + + pub async fn compact_input( + &self, + input: &CompactionInput<'_>, + extra_headers: HeaderMap, + ) -> Result, ApiError> { + let body = to_value(input) + .map_err(|e| ApiError::Stream(format!("failed to encode compaction input: {e}")))?; + self.compact(body, extra_headers).await + } +} + +#[derive(Debug, Deserialize)] +struct CompactHistoryResponse { + output: Vec, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::provider::RetryConfig; + use async_trait::async_trait; + use codex_client::Request; + use codex_client::Response; + use codex_client::StreamResponse; + use codex_client::TransportError; + use http::HeaderMap; + use std::time::Duration; + + #[derive(Clone, Default)] + struct DummyTransport; + + #[async_trait] + impl HttpTransport for DummyTransport { + async fn execute(&self, _req: Request) -> Result { + Err(TransportError::Build("execute should not run".to_string())) + } + + async fn stream(&self, _req: Request) -> Result { + Err(TransportError::Build("stream should not run".to_string())) + } + } + + #[derive(Clone, Default)] + struct DummyAuth; + + impl AuthProvider for DummyAuth { + fn bearer_token(&self) -> Option { + None + } + } + + fn provider(wire: WireApi) -> Provider { + Provider { + name: "test".to_string(), + base_url: "https://example.com/v1".to_string(), + query_params: None, + wire, + headers: HeaderMap::new(), + retry: RetryConfig { + max_attempts: 1, + base_delay: Duration::from_millis(1), + retry_429: false, + retry_5xx: true, + retry_transport: true, + }, + stream_idle_timeout: Duration::from_secs(1), + } + } + + #[tokio::test] + async fn errors_when_wire_is_chat() { + let client = CompactClient::new(DummyTransport, provider(WireApi::Chat), DummyAuth); + let input = CompactionInput { + model: "gpt-test", + input: &[], + instructions: "inst", + }; + let err = client + .compact_input(&input, HeaderMap::new()) + .await + .expect_err("expected wire mismatch to fail"); + + match err { + ApiError::Stream(msg) => { + assert_eq!(msg, "compact endpoint requires responses wire api"); + } + other => panic!("unexpected error: {other:?}"), + } + } +} diff --git a/codex-rs/codex-api/src/endpoint/mod.rs b/codex-rs/codex-api/src/endpoint/mod.rs new file mode 100644 index 0000000000..104b4c2640 --- /dev/null +++ b/codex-rs/codex-api/src/endpoint/mod.rs @@ -0,0 +1,4 @@ +pub mod chat; +pub mod compact; +pub mod responses; +mod streaming; diff --git a/codex-rs/codex-api/src/endpoint/responses.rs b/codex-rs/codex-api/src/endpoint/responses.rs new file mode 100644 index 0000000000..d3a314d763 --- /dev/null +++ b/codex-rs/codex-api/src/endpoint/responses.rs @@ -0,0 +1,107 @@ +use crate::auth::AuthProvider; +use crate::common::Prompt as ApiPrompt; +use crate::common::Reasoning; +use crate::common::ResponseStream; +use crate::common::TextControls; +use crate::endpoint::streaming::StreamingClient; +use crate::error::ApiError; +use crate::provider::Provider; +use crate::provider::WireApi; +use crate::requests::ResponsesRequest; +use crate::requests::ResponsesRequestBuilder; +use crate::sse::spawn_response_stream; +use crate::telemetry::SseTelemetry; +use codex_client::HttpTransport; +use codex_client::RequestTelemetry; +use codex_protocol::protocol::SessionSource; +use http::HeaderMap; +use serde_json::Value; +use std::sync::Arc; + +pub struct ResponsesClient { + streaming: StreamingClient, +} + +#[derive(Default)] +pub struct ResponsesOptions { + pub reasoning: Option, + pub include: Vec, + pub prompt_cache_key: Option, + pub text: Option, + pub store_override: Option, + pub conversation_id: Option, + pub session_source: Option, +} + +impl ResponsesClient { + pub fn new(transport: T, provider: Provider, auth: A) -> Self { + Self { + streaming: StreamingClient::new(transport, provider, auth), + } + } + + pub fn with_telemetry( + self, + request: Option>, + sse: Option>, + ) -> Self { + Self { + streaming: self.streaming.with_telemetry(request, sse), + } + } + + pub async fn stream_request( + &self, + request: ResponsesRequest, + ) -> Result { + self.stream(request.body, request.headers).await + } + + pub async fn stream_prompt( + &self, + model: &str, + prompt: &ApiPrompt, + options: ResponsesOptions, + ) -> Result { + let ResponsesOptions { + reasoning, + include, + prompt_cache_key, + text, + store_override, + conversation_id, + session_source, + } = options; + + let request = ResponsesRequestBuilder::new(model, &prompt.instructions, &prompt.input) + .tools(&prompt.tools) + .parallel_tool_calls(prompt.parallel_tool_calls) + .reasoning(reasoning) + .include(include) + .prompt_cache_key(prompt_cache_key) + .text(text) + .conversation(conversation_id) + .session_source(session_source) + .store_override(store_override) + .build(self.streaming.provider())?; + + self.stream_request(request).await + } + + fn path(&self) -> &'static str { + match self.streaming.provider().wire { + WireApi::Responses | WireApi::Compact => "responses", + WireApi::Chat => "chat/completions", + } + } + + pub async fn stream( + &self, + body: Value, + extra_headers: HeaderMap, + ) -> Result { + self.streaming + .stream(self.path(), body, extra_headers, spawn_response_stream) + .await + } +} diff --git a/codex-rs/codex-api/src/endpoint/streaming.rs b/codex-rs/codex-api/src/endpoint/streaming.rs new file mode 100644 index 0000000000..156d4084bc --- /dev/null +++ b/codex-rs/codex-api/src/endpoint/streaming.rs @@ -0,0 +1,82 @@ +use crate::auth::AuthProvider; +use crate::auth::add_auth_headers; +use crate::common::ResponseStream; +use crate::error::ApiError; +use crate::provider::Provider; +use crate::telemetry::SseTelemetry; +use crate::telemetry::run_with_request_telemetry; +use codex_client::HttpTransport; +use codex_client::RequestTelemetry; +use codex_client::StreamResponse; +use http::HeaderMap; +use http::Method; +use serde_json::Value; +use std::sync::Arc; +use std::time::Duration; + +pub(crate) struct StreamingClient { + transport: T, + provider: Provider, + auth: A, + request_telemetry: Option>, + sse_telemetry: Option>, +} + +impl StreamingClient { + pub(crate) fn new(transport: T, provider: Provider, auth: A) -> Self { + Self { + transport, + provider, + auth, + request_telemetry: None, + sse_telemetry: None, + } + } + + pub(crate) fn with_telemetry( + mut self, + request: Option>, + sse: Option>, + ) -> Self { + self.request_telemetry = request; + self.sse_telemetry = sse; + self + } + + pub(crate) fn provider(&self) -> &Provider { + &self.provider + } + + pub(crate) async fn stream( + &self, + path: &str, + body: Value, + extra_headers: HeaderMap, + spawner: fn(StreamResponse, Duration, Option>) -> ResponseStream, + ) -> Result { + let builder = || { + let mut req = self.provider.build_request(Method::POST, path); + req.headers.extend(extra_headers.clone()); + req.headers.insert( + http::header::ACCEPT, + http::HeaderValue::from_static("text/event-stream"), + ); + req.body = Some(body.clone()); + add_auth_headers(&self.auth, req) + }; + + let stream_response = run_with_request_telemetry( + self.provider.retry.to_policy(), + self.request_telemetry.clone(), + builder, + |req| self.transport.stream(req), + ) + .await?; + + Ok(spawner( + stream_response, + self.provider.stream_idle_timeout, + self.sse_telemetry.clone(), + )) + } +} diff --git a/codex-rs/codex-api/src/error.rs b/codex-rs/codex-api/src/error.rs new file mode 100644 index 0000000000..60118e8723 --- /dev/null +++ b/codex-rs/codex-api/src/error.rs @@ -0,0 +1,34 @@ +use crate::rate_limits::RateLimitError; +use codex_client::TransportError; +use http::StatusCode; +use std::time::Duration; +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum ApiError { + #[error(transparent)] + Transport(#[from] TransportError), + #[error("api error {status}: {message}")] + Api { status: StatusCode, message: String }, + #[error("stream error: {0}")] + Stream(String), + #[error("context window exceeded")] + ContextWindowExceeded, + #[error("quota exceeded")] + QuotaExceeded, + #[error("usage not included")] + UsageNotIncluded, + #[error("retryable error: {message}")] + Retryable { + message: String, + delay: Option, + }, + #[error("rate limit: {0}")] + RateLimit(String), +} + +impl From for ApiError { + fn from(err: RateLimitError) -> Self { + Self::RateLimit(err.to_string()) + } +} diff --git a/codex-rs/codex-api/src/lib.rs b/codex-rs/codex-api/src/lib.rs new file mode 100644 index 0000000000..acde4b4589 --- /dev/null +++ b/codex-rs/codex-api/src/lib.rs @@ -0,0 +1,35 @@ +pub mod auth; +pub mod common; +pub mod endpoint; +pub mod error; +pub mod provider; +pub mod rate_limits; +pub mod requests; +pub mod sse; +pub mod telemetry; + +pub use codex_client::RequestTelemetry; +pub use codex_client::ReqwestTransport; +pub use codex_client::TransportError; + +pub use crate::auth::AuthProvider; +pub use crate::common::CompactionInput; +pub use crate::common::Prompt; +pub use crate::common::ResponseEvent; +pub use crate::common::ResponseStream; +pub use crate::common::ResponsesApiRequest; +pub use crate::common::create_text_param_for_request; +pub use crate::endpoint::chat::AggregateStreamExt; +pub use crate::endpoint::chat::ChatClient; +pub use crate::endpoint::compact::CompactClient; +pub use crate::endpoint::responses::ResponsesClient; +pub use crate::endpoint::responses::ResponsesOptions; +pub use crate::error::ApiError; +pub use crate::provider::Provider; +pub use crate::provider::WireApi; +pub use crate::requests::ChatRequest; +pub use crate::requests::ChatRequestBuilder; +pub use crate::requests::ResponsesRequest; +pub use crate::requests::ResponsesRequestBuilder; +pub use crate::sse::stream_from_fixture; +pub use crate::telemetry::SseTelemetry; diff --git a/codex-rs/codex-api/src/provider.rs b/codex-rs/codex-api/src/provider.rs new file mode 100644 index 0000000000..8bd5fc9093 --- /dev/null +++ b/codex-rs/codex-api/src/provider.rs @@ -0,0 +1,118 @@ +use codex_client::Request; +use codex_client::RetryOn; +use codex_client::RetryPolicy; +use http::Method; +use http::header::HeaderMap; +use std::collections::HashMap; +use std::time::Duration; + +/// Wire-level APIs supported by a `Provider`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum WireApi { + Responses, + Chat, + Compact, +} + +/// High-level retry configuration for a provider. +/// +/// This is converted into a `RetryPolicy` used by `codex-client` to drive +/// transport-level retries for both unary and streaming calls. +#[derive(Debug, Clone)] +pub struct RetryConfig { + pub max_attempts: u64, + pub base_delay: Duration, + pub retry_429: bool, + pub retry_5xx: bool, + pub retry_transport: bool, +} + +impl RetryConfig { + pub fn to_policy(&self) -> RetryPolicy { + RetryPolicy { + max_attempts: self.max_attempts, + base_delay: self.base_delay, + retry_on: RetryOn { + retry_429: self.retry_429, + retry_5xx: self.retry_5xx, + retry_transport: self.retry_transport, + }, + } + } +} + +/// HTTP endpoint configuration used to talk to a concrete API deployment. +/// +/// Encapsulates base URL, default headers, query params, retry policy, and +/// stream idle timeout, plus helper methods for building requests. +#[derive(Debug, Clone)] +pub struct Provider { + pub name: String, + pub base_url: String, + pub query_params: Option>, + pub wire: WireApi, + pub headers: HeaderMap, + pub retry: RetryConfig, + pub stream_idle_timeout: Duration, +} + +impl Provider { + pub fn url_for_path(&self, path: &str) -> String { + let base = self.base_url.trim_end_matches('/'); + let path = path.trim_start_matches('/'); + let mut url = if path.is_empty() { + base.to_string() + } else { + format!("{base}/{path}") + }; + + if let Some(params) = &self.query_params + && !params.is_empty() + { + let qs = params + .iter() + .map(|(k, v)| format!("{k}={v}")) + .collect::>() + .join("&"); + url.push('?'); + url.push_str(&qs); + } + + url + } + + pub fn build_request(&self, method: Method, path: &str) -> Request { + Request { + method, + url: self.url_for_path(path), + headers: self.headers.clone(), + body: None, + timeout: None, + } + } + + pub fn is_azure_responses_endpoint(&self) -> bool { + if self.wire != WireApi::Responses { + return false; + } + + if self.name.eq_ignore_ascii_case("azure") { + return true; + } + + self.base_url.to_ascii_lowercase().contains("openai.azure.") + || matches_azure_responses_base_url(&self.base_url) + } +} + +fn matches_azure_responses_base_url(base_url: &str) -> bool { + const AZURE_MARKERS: [&str; 5] = [ + "cognitiveservices.azure.", + "aoai.azure.", + "azure-api.", + "azurefd.", + "windows.net/openai", + ]; + let base = base_url.to_ascii_lowercase(); + AZURE_MARKERS.iter().any(|marker| base.contains(marker)) +} diff --git a/codex-rs/codex-api/src/rate_limits.rs b/codex-rs/codex-api/src/rate_limits.rs new file mode 100644 index 0000000000..69092063f6 --- /dev/null +++ b/codex-rs/codex-api/src/rate_limits.rs @@ -0,0 +1,105 @@ +use codex_protocol::protocol::CreditsSnapshot; +use codex_protocol::protocol::RateLimitSnapshot; +use codex_protocol::protocol::RateLimitWindow; +use http::HeaderMap; +use std::fmt::Display; + +#[derive(Debug)] +pub struct RateLimitError { + pub message: String, +} + +impl Display for RateLimitError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.message) + } +} + +/// Parses the bespoke Codex rate-limit headers into a `RateLimitSnapshot`. +pub fn parse_rate_limit(headers: &HeaderMap) -> Option { + let primary = parse_rate_limit_window( + headers, + "x-codex-primary-used-percent", + "x-codex-primary-window-minutes", + "x-codex-primary-reset-at", + ); + + let secondary = parse_rate_limit_window( + headers, + "x-codex-secondary-used-percent", + "x-codex-secondary-window-minutes", + "x-codex-secondary-reset-at", + ); + + let credits = parse_credits_snapshot(headers); + + Some(RateLimitSnapshot { + primary, + secondary, + credits, + }) +} + +fn parse_rate_limit_window( + headers: &HeaderMap, + used_percent_header: &str, + window_minutes_header: &str, + resets_at_header: &str, +) -> Option { + let used_percent: Option = parse_header_f64(headers, used_percent_header); + + used_percent.and_then(|used_percent| { + let window_minutes = parse_header_i64(headers, window_minutes_header); + let resets_at = parse_header_i64(headers, resets_at_header); + + let has_data = used_percent != 0.0 + || window_minutes.is_some_and(|minutes| minutes != 0) + || resets_at.is_some(); + + has_data.then_some(RateLimitWindow { + used_percent, + window_minutes, + resets_at, + }) + }) +} + +fn parse_credits_snapshot(headers: &HeaderMap) -> Option { + let has_credits = parse_header_bool(headers, "x-codex-credits-has-credits")?; + let unlimited = parse_header_bool(headers, "x-codex-credits-unlimited")?; + let balance = parse_header_str(headers, "x-codex-credits-balance") + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(std::string::ToString::to_string); + Some(CreditsSnapshot { + has_credits, + unlimited, + balance, + }) +} + +fn parse_header_f64(headers: &HeaderMap, name: &str) -> Option { + parse_header_str(headers, name)? + .parse::() + .ok() + .filter(|v| v.is_finite()) +} + +fn parse_header_i64(headers: &HeaderMap, name: &str) -> Option { + parse_header_str(headers, name)?.parse::().ok() +} + +fn parse_header_bool(headers: &HeaderMap, name: &str) -> Option { + let raw = parse_header_str(headers, name)?; + if raw.eq_ignore_ascii_case("true") || raw == "1" { + Some(true) + } else if raw.eq_ignore_ascii_case("false") || raw == "0" { + Some(false) + } else { + None + } +} + +fn parse_header_str<'a>(headers: &'a HeaderMap, name: &str) -> Option<&'a str> { + headers.get(name)?.to_str().ok() +} diff --git a/codex-rs/codex-api/src/requests/chat.rs b/codex-rs/codex-api/src/requests/chat.rs new file mode 100644 index 0000000000..c1ba89f37a --- /dev/null +++ b/codex-rs/codex-api/src/requests/chat.rs @@ -0,0 +1,388 @@ +use crate::error::ApiError; +use crate::provider::Provider; +use crate::requests::headers::build_conversation_headers; +use crate::requests::headers::insert_header; +use crate::requests::headers::subagent_header; +use codex_protocol::models::ContentItem; +use codex_protocol::models::FunctionCallOutputContentItem; +use codex_protocol::models::ReasoningItemContent; +use codex_protocol::models::ResponseItem; +use codex_protocol::protocol::SessionSource; +use http::HeaderMap; +use serde_json::Value; +use serde_json::json; +use std::collections::HashMap; + +/// Assembled request body plus headers for Chat Completions streaming calls. +pub struct ChatRequest { + pub body: Value, + pub headers: HeaderMap, +} + +pub struct ChatRequestBuilder<'a> { + model: &'a str, + instructions: &'a str, + input: &'a [ResponseItem], + tools: &'a [Value], + conversation_id: Option, + session_source: Option, +} + +impl<'a> ChatRequestBuilder<'a> { + pub fn new( + model: &'a str, + instructions: &'a str, + input: &'a [ResponseItem], + tools: &'a [Value], + ) -> Self { + Self { + model, + instructions, + input, + tools, + conversation_id: None, + session_source: None, + } + } + + pub fn conversation_id(mut self, id: Option) -> Self { + self.conversation_id = id; + self + } + + pub fn session_source(mut self, source: Option) -> Self { + self.session_source = source; + self + } + + pub fn build(self, _provider: &Provider) -> Result { + let mut messages = Vec::::new(); + messages.push(json!({"role": "system", "content": self.instructions})); + + let input = self.input; + let mut reasoning_by_anchor_index: HashMap = HashMap::new(); + let mut last_emitted_role: Option<&str> = None; + for item in input { + match item { + ResponseItem::Message { role, .. } => last_emitted_role = Some(role.as_str()), + ResponseItem::FunctionCall { .. } | ResponseItem::LocalShellCall { .. } => { + last_emitted_role = Some("assistant") + } + ResponseItem::FunctionCallOutput { .. } => last_emitted_role = Some("tool"), + ResponseItem::Reasoning { .. } | ResponseItem::Other => {} + ResponseItem::CustomToolCall { .. } => {} + ResponseItem::CustomToolCallOutput { .. } => {} + ResponseItem::WebSearchCall { .. } => {} + ResponseItem::GhostSnapshot { .. } => {} + ResponseItem::CompactionSummary { .. } => {} + } + } + + let mut last_user_index: Option = None; + for (idx, item) in input.iter().enumerate() { + if let ResponseItem::Message { role, .. } = item + && role == "user" + { + last_user_index = Some(idx); + } + } + + if !matches!(last_emitted_role, Some("user")) { + for (idx, item) in input.iter().enumerate() { + if let Some(u_idx) = last_user_index + && idx <= u_idx + { + continue; + } + + if let ResponseItem::Reasoning { + content: Some(items), + .. + } = item + { + let mut text = String::new(); + for entry in items { + match entry { + ReasoningItemContent::ReasoningText { text: segment } + | ReasoningItemContent::Text { text: segment } => { + text.push_str(segment) + } + } + } + if text.trim().is_empty() { + continue; + } + + let mut attached = false; + if idx > 0 + && let ResponseItem::Message { role, .. } = &input[idx - 1] + && role == "assistant" + { + reasoning_by_anchor_index + .entry(idx - 1) + .and_modify(|v| v.push_str(&text)) + .or_insert(text.clone()); + attached = true; + } + + if !attached && idx + 1 < input.len() { + match &input[idx + 1] { + ResponseItem::FunctionCall { .. } + | ResponseItem::LocalShellCall { .. } => { + reasoning_by_anchor_index + .entry(idx + 1) + .and_modify(|v| v.push_str(&text)) + .or_insert(text.clone()); + } + ResponseItem::Message { role, .. } if role == "assistant" => { + reasoning_by_anchor_index + .entry(idx + 1) + .and_modify(|v| v.push_str(&text)) + .or_insert(text.clone()); + } + _ => {} + } + } + } + } + } + + let mut last_assistant_text: Option = None; + + for (idx, item) in input.iter().enumerate() { + match item { + ResponseItem::Message { role, content, .. } => { + let mut text = String::new(); + let mut items: Vec = Vec::new(); + let mut saw_image = false; + + for c in content { + match c { + ContentItem::InputText { text: t } + | ContentItem::OutputText { text: t } => { + text.push_str(t); + items.push(json!({"type":"text","text": t})); + } + ContentItem::InputImage { image_url } => { + saw_image = true; + items.push( + json!({"type":"image_url","image_url": {"url": image_url}}), + ); + } + } + } + + if role == "assistant" { + if let Some(prev) = &last_assistant_text + && prev == &text + { + continue; + } + last_assistant_text = Some(text.clone()); + } + + let content_value = if role == "assistant" { + json!(text) + } else if saw_image { + json!(items) + } else { + json!(text) + }; + + let mut msg = json!({"role": role, "content": content_value}); + if role == "assistant" + && let Some(reasoning) = reasoning_by_anchor_index.get(&idx) + && let Some(obj) = msg.as_object_mut() + { + obj.insert("reasoning".to_string(), json!(reasoning)); + } + messages.push(msg); + } + ResponseItem::FunctionCall { + name, + arguments, + call_id, + .. + } => { + let mut msg = json!({ + "role": "assistant", + "content": null, + "tool_calls": [{ + "id": call_id, + "type": "function", + "function": { + "name": name, + "arguments": arguments, + } + }] + }); + if let Some(reasoning) = reasoning_by_anchor_index.get(&idx) + && let Some(obj) = msg.as_object_mut() + { + obj.insert("reasoning".to_string(), json!(reasoning)); + } + messages.push(msg); + } + ResponseItem::LocalShellCall { + id, + call_id: _, + status, + action, + } => { + let mut msg = json!({ + "role": "assistant", + "content": null, + "tool_calls": [{ + "id": id.clone().unwrap_or_default(), + "type": "local_shell_call", + "status": status, + "action": action, + }] + }); + if let Some(reasoning) = reasoning_by_anchor_index.get(&idx) + && let Some(obj) = msg.as_object_mut() + { + obj.insert("reasoning".to_string(), json!(reasoning)); + } + messages.push(msg); + } + ResponseItem::FunctionCallOutput { call_id, output } => { + let content_value = if let Some(items) = &output.content_items { + let mapped: Vec = items + .iter() + .map(|it| match it { + FunctionCallOutputContentItem::InputText { text } => { + json!({"type":"text","text": text}) + } + FunctionCallOutputContentItem::InputImage { image_url } => { + json!({"type":"image_url","image_url": {"url": image_url}}) + } + }) + .collect(); + json!(mapped) + } else { + json!(output.content) + }; + + messages.push(json!({ + "role": "tool", + "tool_call_id": call_id, + "content": content_value, + })); + } + ResponseItem::CustomToolCall { + id, + call_id: _, + name, + input, + status: _, + } => { + messages.push(json!({ + "role": "assistant", + "content": null, + "tool_calls": [{ + "id": id, + "type": "custom", + "custom": { + "name": name, + "input": input, + } + }] + })); + } + ResponseItem::CustomToolCallOutput { call_id, output } => { + messages.push(json!({ + "role": "tool", + "tool_call_id": call_id, + "content": output, + })); + } + ResponseItem::GhostSnapshot { .. } => { + continue; + } + ResponseItem::Reasoning { .. } + | ResponseItem::WebSearchCall { .. } + | ResponseItem::Other + | ResponseItem::CompactionSummary { .. } => { + continue; + } + } + } + + let payload = json!({ + "model": self.model, + "messages": messages, + "stream": true, + "tools": self.tools, + }); + + let mut headers = build_conversation_headers(self.conversation_id); + if let Some(subagent) = subagent_header(&self.session_source) { + insert_header(&mut headers, "x-openai-subagent", &subagent); + } + + Ok(ChatRequest { + body: payload, + headers, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::provider::RetryConfig; + use crate::provider::WireApi; + use codex_protocol::protocol::SessionSource; + use codex_protocol::protocol::SubAgentSource; + use http::HeaderValue; + use pretty_assertions::assert_eq; + use std::time::Duration; + + fn provider() -> Provider { + Provider { + name: "openai".to_string(), + base_url: "https://api.openai.com/v1".to_string(), + query_params: None, + wire: WireApi::Chat, + headers: HeaderMap::new(), + retry: RetryConfig { + max_attempts: 1, + base_delay: Duration::from_millis(10), + retry_429: false, + retry_5xx: true, + retry_transport: true, + }, + stream_idle_timeout: Duration::from_secs(1), + } + } + + #[test] + fn attaches_conversation_and_subagent_headers() { + let prompt_input = vec![ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::InputText { + text: "hi".to_string(), + }], + }]; + let req = ChatRequestBuilder::new("gpt-test", "inst", &prompt_input, &[]) + .conversation_id(Some("conv-1".into())) + .session_source(Some(SessionSource::SubAgent(SubAgentSource::Review))) + .build(&provider()) + .expect("request"); + + assert_eq!( + req.headers.get("conversation_id"), + Some(&HeaderValue::from_static("conv-1")) + ); + assert_eq!( + req.headers.get("session_id"), + Some(&HeaderValue::from_static("conv-1")) + ); + assert_eq!( + req.headers.get("x-openai-subagent"), + Some(&HeaderValue::from_static("review")) + ); + } +} diff --git a/codex-rs/codex-api/src/requests/headers.rs b/codex-rs/codex-api/src/requests/headers.rs new file mode 100644 index 0000000000..4d8a17d18d --- /dev/null +++ b/codex-rs/codex-api/src/requests/headers.rs @@ -0,0 +1,36 @@ +use codex_protocol::protocol::SessionSource; +use http::HeaderMap; +use http::HeaderValue; + +pub(crate) fn build_conversation_headers(conversation_id: Option) -> HeaderMap { + let mut headers = HeaderMap::new(); + if let Some(id) = conversation_id { + insert_header(&mut headers, "conversation_id", &id); + insert_header(&mut headers, "session_id", &id); + } + headers +} + +pub(crate) fn subagent_header(source: &Option) -> Option { + let SessionSource::SubAgent(sub) = source.as_ref()? else { + return None; + }; + match sub { + codex_protocol::protocol::SubAgentSource::Other(label) => Some(label.clone()), + other => Some( + serde_json::to_value(other) + .ok() + .and_then(|v| v.as_str().map(std::string::ToString::to_string)) + .unwrap_or_else(|| "other".to_string()), + ), + } +} + +pub(crate) fn insert_header(headers: &mut HeaderMap, name: &str, value: &str) { + if let (Ok(header_name), Ok(header_value)) = ( + name.parse::(), + HeaderValue::from_str(value), + ) { + headers.insert(header_name, header_value); + } +} diff --git a/codex-rs/codex-api/src/requests/mod.rs b/codex-rs/codex-api/src/requests/mod.rs new file mode 100644 index 0000000000..f0ab23a25f --- /dev/null +++ b/codex-rs/codex-api/src/requests/mod.rs @@ -0,0 +1,8 @@ +pub mod chat; +pub(crate) mod headers; +pub mod responses; + +pub use chat::ChatRequest; +pub use chat::ChatRequestBuilder; +pub use responses::ResponsesRequest; +pub use responses::ResponsesRequestBuilder; diff --git a/codex-rs/codex-api/src/requests/responses.rs b/codex-rs/codex-api/src/requests/responses.rs new file mode 100644 index 0000000000..543b79bbe9 --- /dev/null +++ b/codex-rs/codex-api/src/requests/responses.rs @@ -0,0 +1,247 @@ +use crate::common::Reasoning; +use crate::common::ResponsesApiRequest; +use crate::common::TextControls; +use crate::error::ApiError; +use crate::provider::Provider; +use crate::requests::headers::build_conversation_headers; +use crate::requests::headers::insert_header; +use crate::requests::headers::subagent_header; +use codex_protocol::models::ResponseItem; +use codex_protocol::protocol::SessionSource; +use http::HeaderMap; +use serde_json::Value; + +/// Assembled request body plus headers for a Responses stream request. +pub struct ResponsesRequest { + pub body: Value, + pub headers: HeaderMap, +} + +#[derive(Default)] +pub struct ResponsesRequestBuilder<'a> { + model: Option<&'a str>, + instructions: Option<&'a str>, + input: Option<&'a [ResponseItem]>, + tools: Option<&'a [Value]>, + parallel_tool_calls: bool, + reasoning: Option, + include: Vec, + prompt_cache_key: Option, + text: Option, + conversation_id: Option, + session_source: Option, + store_override: Option, + headers: HeaderMap, +} + +impl<'a> ResponsesRequestBuilder<'a> { + pub fn new(model: &'a str, instructions: &'a str, input: &'a [ResponseItem]) -> Self { + Self { + model: Some(model), + instructions: Some(instructions), + input: Some(input), + ..Default::default() + } + } + + pub fn tools(mut self, tools: &'a [Value]) -> Self { + self.tools = Some(tools); + self + } + + pub fn parallel_tool_calls(mut self, enabled: bool) -> Self { + self.parallel_tool_calls = enabled; + self + } + + pub fn reasoning(mut self, reasoning: Option) -> Self { + self.reasoning = reasoning; + self + } + + pub fn include(mut self, include: Vec) -> Self { + self.include = include; + self + } + + pub fn prompt_cache_key(mut self, key: Option) -> Self { + self.prompt_cache_key = key; + self + } + + pub fn text(mut self, text: Option) -> Self { + self.text = text; + self + } + + pub fn conversation(mut self, conversation_id: Option) -> Self { + self.conversation_id = conversation_id; + self + } + + pub fn session_source(mut self, source: Option) -> Self { + self.session_source = source; + self + } + + pub fn store_override(mut self, store: Option) -> Self { + self.store_override = store; + self + } + + pub fn extra_headers(mut self, headers: HeaderMap) -> Self { + self.headers = headers; + self + } + + pub fn build(self, provider: &Provider) -> Result { + let model = self + .model + .ok_or_else(|| ApiError::Stream("missing model for responses request".into()))?; + let instructions = self + .instructions + .ok_or_else(|| ApiError::Stream("missing instructions for responses request".into()))?; + let input = self + .input + .ok_or_else(|| ApiError::Stream("missing input for responses request".into()))?; + let tools = self.tools.unwrap_or_default(); + + let store = self + .store_override + .unwrap_or_else(|| provider.is_azure_responses_endpoint()); + + let req = ResponsesApiRequest { + model, + instructions, + input, + tools, + tool_choice: "auto", + parallel_tool_calls: self.parallel_tool_calls, + reasoning: self.reasoning, + store, + stream: true, + include: self.include, + prompt_cache_key: self.prompt_cache_key, + text: self.text, + }; + + let mut body = serde_json::to_value(&req) + .map_err(|e| ApiError::Stream(format!("failed to encode responses request: {e}")))?; + + if store && provider.is_azure_responses_endpoint() { + attach_item_ids(&mut body, input); + } + + let mut headers = self.headers; + headers.extend(build_conversation_headers(self.conversation_id)); + if let Some(subagent) = subagent_header(&self.session_source) { + insert_header(&mut headers, "x-openai-subagent", &subagent); + } + + Ok(ResponsesRequest { body, headers }) + } +} + +fn attach_item_ids(payload_json: &mut Value, original_items: &[ResponseItem]) { + let Some(input_value) = payload_json.get_mut("input") else { + return; + }; + let Value::Array(items) = input_value else { + return; + }; + + for (value, item) in items.iter_mut().zip(original_items.iter()) { + if let ResponseItem::Reasoning { id, .. } + | ResponseItem::Message { id: Some(id), .. } + | ResponseItem::WebSearchCall { id: Some(id), .. } + | ResponseItem::FunctionCall { id: Some(id), .. } + | ResponseItem::LocalShellCall { id: Some(id), .. } + | ResponseItem::CustomToolCall { id: Some(id), .. } = item + { + if id.is_empty() { + continue; + } + + if let Some(obj) = value.as_object_mut() { + obj.insert("id".to_string(), Value::String(id.clone())); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::provider::RetryConfig; + use crate::provider::WireApi; + use codex_protocol::protocol::SubAgentSource; + use http::HeaderValue; + use pretty_assertions::assert_eq; + use std::time::Duration; + + fn provider(name: &str, base_url: &str) -> Provider { + Provider { + name: name.to_string(), + base_url: base_url.to_string(), + query_params: None, + wire: WireApi::Responses, + headers: HeaderMap::new(), + retry: RetryConfig { + max_attempts: 1, + base_delay: Duration::from_millis(50), + retry_429: false, + retry_5xx: true, + retry_transport: true, + }, + stream_idle_timeout: Duration::from_secs(5), + } + } + + #[test] + fn azure_default_store_attaches_ids_and_headers() { + let provider = provider("azure", "https://example.openai.azure.com/v1"); + let input = vec![ + ResponseItem::Message { + id: Some("m1".into()), + role: "assistant".into(), + content: Vec::new(), + }, + ResponseItem::Message { + id: None, + role: "assistant".into(), + content: Vec::new(), + }, + ]; + + let request = ResponsesRequestBuilder::new("gpt-test", "inst", &input) + .conversation(Some("conv-1".into())) + .session_source(Some(SessionSource::SubAgent(SubAgentSource::Review))) + .build(&provider) + .expect("request"); + + assert_eq!(request.body.get("store"), Some(&Value::Bool(true))); + + let ids: Vec> = request + .body + .get("input") + .and_then(|v| v.as_array()) + .into_iter() + .flatten() + .map(|item| item.get("id").and_then(|v| v.as_str().map(str::to_string))) + .collect(); + assert_eq!(ids, vec![Some("m1".to_string()), None]); + + assert_eq!( + request.headers.get("conversation_id"), + Some(&HeaderValue::from_static("conv-1")) + ); + assert_eq!( + request.headers.get("session_id"), + Some(&HeaderValue::from_static("conv-1")) + ); + assert_eq!( + request.headers.get("x-openai-subagent"), + Some(&HeaderValue::from_static("review")) + ); + } +} diff --git a/codex-rs/codex-api/src/sse/chat.rs b/codex-rs/codex-api/src/sse/chat.rs new file mode 100644 index 0000000000..7f50bb634a --- /dev/null +++ b/codex-rs/codex-api/src/sse/chat.rs @@ -0,0 +1,504 @@ +use crate::common::ResponseEvent; +use crate::common::ResponseStream; +use crate::error::ApiError; +use crate::telemetry::SseTelemetry; +use codex_client::StreamResponse; +use codex_protocol::models::ContentItem; +use codex_protocol::models::ReasoningItemContent; +use codex_protocol::models::ResponseItem; +use eventsource_stream::Eventsource; +use futures::Stream; +use futures::StreamExt; +use std::collections::HashMap; +use std::time::Duration; +use tokio::sync::mpsc; +use tokio::time::Instant; +use tokio::time::timeout; +use tracing::debug; +use tracing::trace; + +pub(crate) fn spawn_chat_stream( + stream_response: StreamResponse, + idle_timeout: Duration, + telemetry: Option>, +) -> ResponseStream { + let (tx_event, rx_event) = mpsc::channel::>(1600); + tokio::spawn(async move { + process_chat_sse(stream_response.bytes, tx_event, idle_timeout, telemetry).await; + }); + ResponseStream { rx_event } +} + +pub async fn process_chat_sse( + stream: S, + tx_event: mpsc::Sender>, + idle_timeout: Duration, + telemetry: Option>, +) where + S: Stream> + Unpin, +{ + let mut stream = stream.eventsource(); + + #[derive(Default, Debug)] + struct ToolCallState { + name: Option, + arguments: String, + } + + let mut tool_calls: HashMap = HashMap::new(); + let mut tool_call_order: Vec = Vec::new(); + let mut assistant_item: Option = None; + let mut reasoning_item: Option = None; + let mut completed_sent = false; + + loop { + let start = Instant::now(); + let response = timeout(idle_timeout, stream.next()).await; + if let Some(t) = telemetry.as_ref() { + t.on_sse_poll(&response, start.elapsed()); + } + let sse = match response { + Ok(Some(Ok(sse))) => sse, + Ok(Some(Err(e))) => { + let _ = tx_event.send(Err(ApiError::Stream(e.to_string()))).await; + return; + } + Ok(None) => { + if let Some(reasoning) = reasoning_item { + let _ = tx_event + .send(Ok(ResponseEvent::OutputItemDone(reasoning))) + .await; + } + + if let Some(assistant) = assistant_item { + let _ = tx_event + .send(Ok(ResponseEvent::OutputItemDone(assistant))) + .await; + } + if !completed_sent { + let _ = tx_event + .send(Ok(ResponseEvent::Completed { + response_id: String::new(), + token_usage: None, + })) + .await; + } + return; + } + Err(_) => { + let _ = tx_event + .send(Err(ApiError::Stream("idle timeout waiting for SSE".into()))) + .await; + return; + } + }; + + trace!("SSE event: {}", sse.data); + + if sse.data.trim().is_empty() { + continue; + } + + let value: serde_json::Value = match serde_json::from_str(&sse.data) { + Ok(val) => val, + Err(err) => { + debug!( + "Failed to parse ChatCompletions SSE event: {err}, data: {}", + &sse.data + ); + continue; + } + }; + + let Some(choices) = value.get("choices").and_then(|c| c.as_array()) else { + continue; + }; + + for choice in choices { + if let Some(delta) = choice.get("delta") { + if let Some(reasoning) = delta.get("reasoning") { + if let Some(text) = reasoning.as_str() { + append_reasoning_text(&tx_event, &mut reasoning_item, text.to_string()) + .await; + } else if let Some(text) = reasoning.get("text").and_then(|v| v.as_str()) { + append_reasoning_text(&tx_event, &mut reasoning_item, text.to_string()) + .await; + } else if let Some(text) = reasoning.get("content").and_then(|v| v.as_str()) { + append_reasoning_text(&tx_event, &mut reasoning_item, text.to_string()) + .await; + } + } + + if let Some(content) = delta.get("content") { + if content.is_array() { + for item in content.as_array().unwrap_or(&vec![]) { + if let Some(text) = item.get("text").and_then(|t| t.as_str()) { + append_assistant_text( + &tx_event, + &mut assistant_item, + text.to_string(), + ) + .await; + } + } + } else if let Some(text) = content.as_str() { + append_assistant_text(&tx_event, &mut assistant_item, text.to_string()) + .await; + } + } + + if let Some(tool_call_values) = delta.get("tool_calls").and_then(|c| c.as_array()) { + for tool_call in tool_call_values { + let id = tool_call + .get("id") + .and_then(|i| i.as_str()) + .map(str::to_string) + .unwrap_or_else(|| format!("tool-call-{}", tool_call_order.len())); + + let call_state = tool_calls.entry(id.clone()).or_default(); + if !tool_call_order.contains(&id) { + tool_call_order.push(id.clone()); + } + + if let Some(func) = tool_call.get("function") { + if let Some(fname) = func.get("name").and_then(|n| n.as_str()) { + call_state.name = Some(fname.to_string()); + } + if let Some(arguments) = func.get("arguments").and_then(|a| a.as_str()) + { + call_state.arguments.push_str(arguments); + } + } + } + } + } + + if let Some(message) = choice.get("message") + && let Some(reasoning) = message.get("reasoning") + { + if let Some(text) = reasoning.as_str() { + append_reasoning_text(&tx_event, &mut reasoning_item, text.to_string()).await; + } else if let Some(text) = reasoning.get("text").and_then(|v| v.as_str()) { + append_reasoning_text(&tx_event, &mut reasoning_item, text.to_string()).await; + } else if let Some(text) = reasoning.get("content").and_then(|v| v.as_str()) { + append_reasoning_text(&tx_event, &mut reasoning_item, text.to_string()).await; + } + } + + let finish_reason = choice.get("finish_reason").and_then(|r| r.as_str()); + if finish_reason == Some("stop") { + if let Some(reasoning) = reasoning_item.take() { + let _ = tx_event + .send(Ok(ResponseEvent::OutputItemDone(reasoning))) + .await; + } + + if let Some(assistant) = assistant_item.take() { + let _ = tx_event + .send(Ok(ResponseEvent::OutputItemDone(assistant))) + .await; + } + if !completed_sent { + let _ = tx_event + .send(Ok(ResponseEvent::Completed { + response_id: String::new(), + token_usage: None, + })) + .await; + completed_sent = true; + } + continue; + } + + if finish_reason == Some("length") { + let _ = tx_event.send(Err(ApiError::ContextWindowExceeded)).await; + return; + } + + if finish_reason == Some("tool_calls") { + if let Some(reasoning) = reasoning_item.take() { + let _ = tx_event + .send(Ok(ResponseEvent::OutputItemDone(reasoning))) + .await; + } + + for call_id in tool_call_order.drain(..) { + let state = tool_calls.remove(&call_id).unwrap_or_default(); + let item = ResponseItem::FunctionCall { + id: None, + name: state.name.unwrap_or_default(), + arguments: state.arguments, + call_id: call_id.clone(), + }; + let _ = tx_event.send(Ok(ResponseEvent::OutputItemDone(item))).await; + } + } + } + } +} + +async fn append_assistant_text( + tx_event: &mpsc::Sender>, + assistant_item: &mut Option, + text: String, +) { + if assistant_item.is_none() { + let item = ResponseItem::Message { + id: None, + role: "assistant".to_string(), + content: vec![], + }; + *assistant_item = Some(item.clone()); + let _ = tx_event + .send(Ok(ResponseEvent::OutputItemAdded(item))) + .await; + } + + if let Some(ResponseItem::Message { content, .. }) = assistant_item { + content.push(ContentItem::OutputText { text: text.clone() }); + let _ = tx_event + .send(Ok(ResponseEvent::OutputTextDelta(text.clone()))) + .await; + } +} + +async fn append_reasoning_text( + tx_event: &mpsc::Sender>, + reasoning_item: &mut Option, + text: String, +) { + if reasoning_item.is_none() { + let item = ResponseItem::Reasoning { + id: String::new(), + summary: Vec::new(), + content: Some(vec![]), + encrypted_content: None, + }; + *reasoning_item = Some(item.clone()); + let _ = tx_event + .send(Ok(ResponseEvent::OutputItemAdded(item))) + .await; + } + + if let Some(ResponseItem::Reasoning { + content: Some(content), + .. + }) = reasoning_item + { + let content_index = content.len() as i64; + content.push(ReasoningItemContent::ReasoningText { text: text.clone() }); + + let _ = tx_event + .send(Ok(ResponseEvent::ReasoningContentDelta { + delta: text.clone(), + content_index, + })) + .await; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use assert_matches::assert_matches; + use codex_protocol::models::ResponseItem; + use futures::TryStreamExt; + use serde_json::json; + use tokio::sync::mpsc; + use tokio_util::io::ReaderStream; + + fn build_body(events: &[serde_json::Value]) -> String { + let mut body = String::new(); + for e in events { + body.push_str(&format!("event: message\ndata: {e}\n\n")); + } + body + } + + async fn collect_events(body: &str) -> Vec { + let reader = ReaderStream::new(std::io::Cursor::new(body.to_string())) + .map_err(|err| codex_client::TransportError::Network(err.to_string())); + let (tx, mut rx) = mpsc::channel::>(16); + tokio::spawn(process_chat_sse( + reader, + tx, + Duration::from_millis(1000), + None, + )); + + let mut out = Vec::new(); + while let Some(ev) = rx.recv().await { + out.push(ev.expect("stream error")); + } + out + } + + #[tokio::test] + async fn emits_multiple_tool_calls() { + let delta_a = json!({ + "choices": [{ + "delta": { + "tool_calls": [{ + "id": "call_a", + "function": { "name": "do_a", "arguments": "{\"foo\":1}" } + }] + } + }] + }); + + let delta_b = json!({ + "choices": [{ + "delta": { + "tool_calls": [{ + "id": "call_b", + "function": { "name": "do_b", "arguments": "{\"bar\":2}" } + }] + } + }] + }); + + let finish = json!({ + "choices": [{ + "finish_reason": "tool_calls" + }] + }); + + let body = build_body(&[delta_a, delta_b, finish]); + let events = collect_events(&body).await; + assert_eq!(events.len(), 3); + + assert_matches!( + &events[0], + ResponseEvent::OutputItemDone(ResponseItem::FunctionCall { call_id, name, arguments, .. }) + if call_id == "call_a" && name == "do_a" && arguments == "{\"foo\":1}" + ); + assert_matches!( + &events[1], + ResponseEvent::OutputItemDone(ResponseItem::FunctionCall { call_id, name, arguments, .. }) + if call_id == "call_b" && name == "do_b" && arguments == "{\"bar\":2}" + ); + assert_matches!(events[2], ResponseEvent::Completed { .. }); + } + + #[tokio::test] + async fn concatenates_tool_call_arguments_across_deltas() { + let delta_name = json!({ + "choices": [{ + "delta": { + "tool_calls": [{ + "id": "call_a", + "function": { "name": "do_a" } + }] + } + }] + }); + + let delta_args_1 = json!({ + "choices": [{ + "delta": { + "tool_calls": [{ + "id": "call_a", + "function": { "arguments": "{ \"foo\":" } + }] + } + }] + }); + + let delta_args_2 = json!({ + "choices": [{ + "delta": { + "tool_calls": [{ + "id": "call_a", + "function": { "arguments": "1}" } + }] + } + }] + }); + + let finish = json!({ + "choices": [{ + "finish_reason": "tool_calls" + }] + }); + + let body = build_body(&[delta_name, delta_args_1, delta_args_2, finish]); + let events = collect_events(&body).await; + assert_matches!( + &events[..], + [ + ResponseEvent::OutputItemDone(ResponseItem::FunctionCall { call_id, name, arguments, .. }), + ResponseEvent::Completed { .. } + ] if call_id == "call_a" && name == "do_a" && arguments == "{ \"foo\":1}" + ); + } + + #[tokio::test] + async fn emits_tool_calls_even_when_content_and_reasoning_present() { + let delta_content_and_tools = json!({ + "choices": [{ + "delta": { + "content": [{"text": "hi"}], + "reasoning": "because", + "tool_calls": [{ + "id": "call_a", + "function": { "name": "do_a", "arguments": "{}" } + }] + } + }] + }); + + let finish = json!({ + "choices": [{ + "finish_reason": "tool_calls" + }] + }); + + let body = build_body(&[delta_content_and_tools, finish]); + let events = collect_events(&body).await; + + assert_matches!( + &events[..], + [ + ResponseEvent::OutputItemAdded(ResponseItem::Reasoning { .. }), + ResponseEvent::ReasoningContentDelta { .. }, + ResponseEvent::OutputItemAdded(ResponseItem::Message { .. }), + ResponseEvent::OutputTextDelta(delta), + ResponseEvent::OutputItemDone(ResponseItem::Reasoning { .. }), + ResponseEvent::OutputItemDone(ResponseItem::FunctionCall { call_id, name, .. }), + ResponseEvent::OutputItemDone(ResponseItem::Message { .. }), + ResponseEvent::Completed { .. } + ] if delta == "hi" && call_id == "call_a" && name == "do_a" + ); + } + + #[tokio::test] + async fn drops_partial_tool_calls_on_stop_finish_reason() { + let delta_tool = json!({ + "choices": [{ + "delta": { + "tool_calls": [{ + "id": "call_a", + "function": { "name": "do_a", "arguments": "{}" } + }] + } + }] + }); + + let finish_stop = json!({ + "choices": [{ + "finish_reason": "stop" + }] + }); + + let body = build_body(&[delta_tool, finish_stop]); + let events = collect_events(&body).await; + + assert!(!events.iter().any(|ev| { + matches!( + ev, + ResponseEvent::OutputItemDone(ResponseItem::FunctionCall { .. }) + ) + })); + assert_matches!(events.last(), Some(ResponseEvent::Completed { .. })); + } +} diff --git a/codex-rs/codex-api/src/sse/mod.rs b/codex-rs/codex-api/src/sse/mod.rs new file mode 100644 index 0000000000..e3ab770c43 --- /dev/null +++ b/codex-rs/codex-api/src/sse/mod.rs @@ -0,0 +1,6 @@ +pub mod chat; +pub mod responses; + +pub use responses::process_sse; +pub use responses::spawn_response_stream; +pub use responses::stream_from_fixture; diff --git a/codex-rs/codex-api/src/sse/responses.rs b/codex-rs/codex-api/src/sse/responses.rs new file mode 100644 index 0000000000..5dbec7b77a --- /dev/null +++ b/codex-rs/codex-api/src/sse/responses.rs @@ -0,0 +1,672 @@ +use crate::common::ResponseEvent; +use crate::common::ResponseStream; +use crate::error::ApiError; +use crate::rate_limits::parse_rate_limit; +use crate::telemetry::SseTelemetry; +use codex_client::ByteStream; +use codex_client::StreamResponse; +use codex_client::TransportError; +use codex_protocol::models::ResponseItem; +use codex_protocol::protocol::TokenUsage; +use eventsource_stream::Eventsource; +use futures::StreamExt; +use futures::TryStreamExt; +use serde::Deserialize; +use serde_json::Value; +use std::io::BufRead; +use std::path::Path; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::mpsc; +use tokio::time::Instant; +use tokio::time::timeout; +use tokio_util::io::ReaderStream; +use tracing::debug; +use tracing::trace; + +/// Streams SSE events from an on-disk fixture for tests. +pub fn stream_from_fixture( + path: impl AsRef, + idle_timeout: Duration, +) -> Result { + let file = + std::fs::File::open(path.as_ref()).map_err(|err| ApiError::Stream(err.to_string()))?; + let mut content = String::new(); + for line in std::io::BufReader::new(file).lines() { + let line = line.map_err(|err| ApiError::Stream(err.to_string()))?; + content.push_str(&line); + content.push_str("\n\n"); + } + + let reader = std::io::Cursor::new(content); + let stream = ReaderStream::new(reader).map_err(|err| TransportError::Network(err.to_string())); + let (tx_event, rx_event) = mpsc::channel::>(1600); + tokio::spawn(process_sse(Box::pin(stream), tx_event, idle_timeout, None)); + Ok(ResponseStream { rx_event }) +} + +pub fn spawn_response_stream( + stream_response: StreamResponse, + idle_timeout: Duration, + telemetry: Option>, +) -> ResponseStream { + let rate_limits = parse_rate_limit(&stream_response.headers); + let (tx_event, rx_event) = mpsc::channel::>(1600); + tokio::spawn(async move { + if let Some(snapshot) = rate_limits { + let _ = tx_event.send(Ok(ResponseEvent::RateLimits(snapshot))).await; + } + process_sse(stream_response.bytes, tx_event, idle_timeout, telemetry).await; + }); + + ResponseStream { rx_event } +} + +#[derive(Debug, Deserialize)] +#[allow(dead_code)] +struct Error { + r#type: Option, + code: Option, + message: Option, + plan_type: Option, + resets_at: Option, +} + +#[derive(Debug, Deserialize)] +#[allow(dead_code)] +struct ResponseCompleted { + id: String, + #[serde(default)] + usage: Option, +} + +#[derive(Debug, Deserialize)] +struct ResponseCompletedUsage { + input_tokens: i64, + input_tokens_details: Option, + output_tokens: i64, + output_tokens_details: Option, + total_tokens: i64, +} + +impl From for TokenUsage { + fn from(val: ResponseCompletedUsage) -> Self { + TokenUsage { + input_tokens: val.input_tokens, + cached_input_tokens: val + .input_tokens_details + .map(|d| d.cached_tokens) + .unwrap_or(0), + output_tokens: val.output_tokens, + reasoning_output_tokens: val + .output_tokens_details + .map(|d| d.reasoning_tokens) + .unwrap_or(0), + total_tokens: val.total_tokens, + } + } +} + +#[derive(Debug, Deserialize)] +struct ResponseCompletedInputTokensDetails { + cached_tokens: i64, +} + +#[derive(Debug, Deserialize)] +struct ResponseCompletedOutputTokensDetails { + reasoning_tokens: i64, +} + +#[derive(Deserialize, Debug)] +struct SseEvent { + #[serde(rename = "type")] + kind: String, + response: Option, + item: Option, + delta: Option, + summary_index: Option, + content_index: Option, +} + +pub async fn process_sse( + stream: ByteStream, + tx_event: mpsc::Sender>, + idle_timeout: Duration, + telemetry: Option>, +) { + let mut stream = stream.eventsource(); + let mut response_completed: Option = None; + let mut response_error: Option = None; + + loop { + let start = Instant::now(); + let response = timeout(idle_timeout, stream.next()).await; + if let Some(t) = telemetry.as_ref() { + t.on_sse_poll(&response, start.elapsed()); + } + let sse = match response { + Ok(Some(Ok(sse))) => sse, + Ok(Some(Err(e))) => { + debug!("SSE Error: {e:#}"); + let _ = tx_event.send(Err(ApiError::Stream(e.to_string()))).await; + return; + } + Ok(None) => { + match response_completed.take() { + Some(ResponseCompleted { id, usage }) => { + let event = ResponseEvent::Completed { + response_id: id, + token_usage: usage.map(Into::into), + }; + let _ = tx_event.send(Ok(event)).await; + } + None => { + let error = response_error.unwrap_or(ApiError::Stream( + "stream closed before response.completed".into(), + )); + let _ = tx_event.send(Err(error)).await; + } + } + return; + } + Err(_) => { + let _ = tx_event + .send(Err(ApiError::Stream("idle timeout waiting for SSE".into()))) + .await; + return; + } + }; + + let raw = sse.data.clone(); + trace!("SSE event: {raw}"); + + let event: SseEvent = match serde_json::from_str(&sse.data) { + Ok(event) => event, + Err(e) => { + debug!("Failed to parse SSE event: {e}, data: {}", &sse.data); + continue; + } + }; + + match event.kind.as_str() { + "response.output_item.done" => { + let Some(item_val) = event.item else { continue }; + let Ok(item) = serde_json::from_value::(item_val) else { + debug!("failed to parse ResponseItem from output_item.done"); + continue; + }; + + let event = ResponseEvent::OutputItemDone(item); + if tx_event.send(Ok(event)).await.is_err() { + return; + } + } + "response.output_text.delta" => { + if let Some(delta) = event.delta { + let event = ResponseEvent::OutputTextDelta(delta); + if tx_event.send(Ok(event)).await.is_err() { + return; + } + } + } + "response.reasoning_summary_text.delta" => { + if let (Some(delta), Some(summary_index)) = (event.delta, event.summary_index) { + let event = ResponseEvent::ReasoningSummaryDelta { + delta, + summary_index, + }; + if tx_event.send(Ok(event)).await.is_err() { + return; + } + } + } + "response.reasoning_text.delta" => { + if let (Some(delta), Some(content_index)) = (event.delta, event.content_index) { + let event = ResponseEvent::ReasoningContentDelta { + delta, + content_index, + }; + if tx_event.send(Ok(event)).await.is_err() { + return; + } + } + } + "response.created" => { + if event.response.is_some() { + let _ = tx_event.send(Ok(ResponseEvent::Created {})).await; + } + } + "response.failed" => { + if let Some(resp_val) = event.response { + response_error = + Some(ApiError::Stream("response.failed event received".into())); + + if let Some(error) = resp_val.get("error") + && let Ok(error) = serde_json::from_value::(error.clone()) + { + if is_context_window_error(&error) { + response_error = Some(ApiError::ContextWindowExceeded); + } else if is_quota_exceeded_error(&error) { + response_error = Some(ApiError::QuotaExceeded); + } else if is_usage_not_included(&error) { + response_error = Some(ApiError::UsageNotIncluded); + } else { + let delay = try_parse_retry_after(&error); + let message = error.message.clone().unwrap_or_default(); + response_error = Some(ApiError::Retryable { message, delay }); + } + } + } + } + "response.completed" => { + if let Some(resp_val) = event.response { + match serde_json::from_value::(resp_val) { + Ok(r) => { + response_completed = Some(r); + } + Err(e) => { + let error = format!("failed to parse ResponseCompleted: {e}"); + debug!(error); + response_error = Some(ApiError::Stream(error)); + continue; + } + }; + }; + } + "response.output_item.added" => { + let Some(item_val) = event.item else { continue }; + let Ok(item) = serde_json::from_value::(item_val) else { + debug!("failed to parse ResponseItem from output_item.done"); + continue; + }; + + let event = ResponseEvent::OutputItemAdded(item); + if tx_event.send(Ok(event)).await.is_err() { + return; + } + } + "response.reasoning_summary_part.added" => { + if let Some(summary_index) = event.summary_index { + let event = ResponseEvent::ReasoningSummaryPartAdded { summary_index }; + if tx_event.send(Ok(event)).await.is_err() { + return; + } + } + } + _ => {} + } + } +} + +fn try_parse_retry_after(err: &Error) -> Option { + if err.code.as_deref() != Some("rate_limit_exceeded") { + return None; + } + + let re = rate_limit_regex(); + if let Some(message) = &err.message + && let Some(captures) = re.captures(message) + { + let seconds = captures.get(1); + let unit = captures.get(2); + + if let (Some(value), Some(unit)) = (seconds, unit) { + let value = value.as_str().parse::().ok()?; + let unit = unit.as_str().to_ascii_lowercase(); + + if unit == "s" || unit.starts_with("second") { + return Some(Duration::from_secs_f64(value)); + } else if unit == "ms" { + return Some(Duration::from_millis(value as u64)); + } + } + } + None +} + +fn is_context_window_error(error: &Error) -> bool { + error.code.as_deref() == Some("context_length_exceeded") +} + +fn is_quota_exceeded_error(error: &Error) -> bool { + error.code.as_deref() == Some("insufficient_quota") +} + +fn is_usage_not_included(error: &Error) -> bool { + error.code.as_deref() == Some("usage_not_included") +} + +fn rate_limit_regex() -> &'static regex_lite::Regex { + static RE: std::sync::OnceLock = std::sync::OnceLock::new(); + #[expect(clippy::unwrap_used)] + RE.get_or_init(|| { + regex_lite::Regex::new(r"(?i)try again in\s*(\d+(?:\.\d+)?)\s*(s|ms|seconds?)").unwrap() + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use assert_matches::assert_matches; + use codex_protocol::models::ResponseItem; + use pretty_assertions::assert_eq; + use serde_json::json; + use tokio::sync::mpsc; + use tokio_test::io::Builder as IoBuilder; + + async fn collect_events(chunks: &[&[u8]]) -> Vec> { + let mut builder = IoBuilder::new(); + for chunk in chunks { + builder.read(chunk); + } + + let reader = builder.build(); + let stream = + ReaderStream::new(reader).map_err(|err| TransportError::Network(err.to_string())); + let (tx, mut rx) = mpsc::channel::>(16); + tokio::spawn(process_sse(Box::pin(stream), tx, idle_timeout(), None)); + + let mut events = Vec::new(); + while let Some(ev) = rx.recv().await { + events.push(ev); + } + events + } + + async fn run_sse(events: Vec) -> Vec { + let mut body = String::new(); + for e in events { + let kind = e + .get("type") + .and_then(|v| v.as_str()) + .expect("fixture event missing type"); + if e.as_object().map(|o| o.len() == 1).unwrap_or(false) { + body.push_str(&format!("event: {kind}\n\n")); + } else { + body.push_str(&format!("event: {kind}\ndata: {e}\n\n")); + } + } + + let (tx, mut rx) = mpsc::channel::>(8); + let stream = ReaderStream::new(std::io::Cursor::new(body)) + .map_err(|err| TransportError::Network(err.to_string())); + tokio::spawn(process_sse(Box::pin(stream), tx, idle_timeout(), None)); + + let mut out = Vec::new(); + while let Some(ev) = rx.recv().await { + out.push(ev.expect("channel closed")); + } + out + } + + fn idle_timeout() -> Duration { + Duration::from_millis(1000) + } + + #[tokio::test] + async fn parses_items_and_completed() { + let item1 = json!({ + "type": "response.output_item.done", + "item": { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "Hello"}] + } + }) + .to_string(); + + let item2 = json!({ + "type": "response.output_item.done", + "item": { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "World"}] + } + }) + .to_string(); + + let completed = json!({ + "type": "response.completed", + "response": { "id": "resp1" } + }) + .to_string(); + + let sse1 = format!("event: response.output_item.done\ndata: {item1}\n\n"); + let sse2 = format!("event: response.output_item.done\ndata: {item2}\n\n"); + let sse3 = format!("event: response.completed\ndata: {completed}\n\n"); + + let events = collect_events(&[sse1.as_bytes(), sse2.as_bytes(), sse3.as_bytes()]).await; + + assert_eq!(events.len(), 3); + + assert_matches!( + &events[0], + Ok(ResponseEvent::OutputItemDone(ResponseItem::Message { role, .. })) + if role == "assistant" + ); + + assert_matches!( + &events[1], + Ok(ResponseEvent::OutputItemDone(ResponseItem::Message { role, .. })) + if role == "assistant" + ); + + match &events[2] { + Ok(ResponseEvent::Completed { + response_id, + token_usage, + }) => { + assert_eq!(response_id, "resp1"); + assert!(token_usage.is_none()); + } + other => panic!("unexpected third event: {other:?}"), + } + } + + #[tokio::test] + async fn error_when_missing_completed() { + let item1 = json!({ + "type": "response.output_item.done", + "item": { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "Hello"}] + } + }) + .to_string(); + + let sse1 = format!("event: response.output_item.done\ndata: {item1}\n\n"); + + let events = collect_events(&[sse1.as_bytes()]).await; + + assert_eq!(events.len(), 2); + + assert_matches!(events[0], Ok(ResponseEvent::OutputItemDone(_))); + + match &events[1] { + Err(ApiError::Stream(msg)) => { + assert_eq!(msg, "stream closed before response.completed") + } + other => panic!("unexpected second event: {other:?}"), + } + } + + #[tokio::test] + async fn error_when_error_event() { + let raw_error = r#"{"type":"response.failed","sequence_number":3,"response":{"id":"resp_689bcf18d7f08194bf3440ba62fe05d803fee0cdac429894","object":"response","created_at":1755041560,"status":"failed","background":false,"error":{"code":"rate_limit_exceeded","message":"Rate limit reached for gpt-5.1 in organization org-AAA on tokens per min (TPM): Limit 30000, Used 22999, Requested 12528. Please try again in 11.054s. Visit https://platform.openai.com/account/rate-limits to learn more."}, "usage":null,"user":null,"metadata":{}}}"#; + + let sse1 = format!("event: response.failed\ndata: {raw_error}\n\n"); + + let events = collect_events(&[sse1.as_bytes()]).await; + + assert_eq!(events.len(), 1); + + match &events[0] { + Err(ApiError::Retryable { message, delay }) => { + assert_eq!( + message, + "Rate limit reached for gpt-5.1 in organization org-AAA on tokens per min (TPM): Limit 30000, Used 22999, Requested 12528. Please try again in 11.054s. Visit https://platform.openai.com/account/rate-limits to learn more." + ); + assert_eq!(*delay, Some(Duration::from_secs_f64(11.054))); + } + other => panic!("unexpected second event: {other:?}"), + } + } + + #[tokio::test] + async fn context_window_error_is_fatal() { + let raw_error = r#"{"type":"response.failed","sequence_number":3,"response":{"id":"resp_5c66275b97b9baef1ed95550adb3b7ec13b17aafd1d2f11b","object":"response","created_at":1759510079,"status":"failed","background":false,"error":{"code":"context_length_exceeded","message":"Your input exceeds the context window of this model. Please adjust your input and try again."},"usage":null,"user":null,"metadata":{}}}"#; + + let sse1 = format!("event: response.failed\ndata: {raw_error}\n\n"); + + let events = collect_events(&[sse1.as_bytes()]).await; + + assert_eq!(events.len(), 1); + + assert_matches!(events[0], Err(ApiError::ContextWindowExceeded)); + } + + #[tokio::test] + async fn context_window_error_with_newline_is_fatal() { + let raw_error = r#"{"type":"response.failed","sequence_number":4,"response":{"id":"resp_fatal_newline","object":"response","created_at":1759510080,"status":"failed","background":false,"error":{"code":"context_length_exceeded","message":"Your input exceeds the context window of this model. Please adjust your input and try\nagain."},"usage":null,"user":null,"metadata":{}}}"#; + + let sse1 = format!("event: response.failed\ndata: {raw_error}\n\n"); + + let events = collect_events(&[sse1.as_bytes()]).await; + + assert_eq!(events.len(), 1); + + assert_matches!(events[0], Err(ApiError::ContextWindowExceeded)); + } + + #[tokio::test] + async fn quota_exceeded_error_is_fatal() { + let raw_error = r#"{"type":"response.failed","sequence_number":3,"response":{"id":"resp_fatal_quota","object":"response","created_at":1759771626,"status":"failed","background":false,"error":{"code":"insufficient_quota","message":"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."},"incomplete_details":null}}"#; + + let sse1 = format!("event: response.failed\ndata: {raw_error}\n\n"); + + let events = collect_events(&[sse1.as_bytes()]).await; + + assert_eq!(events.len(), 1); + + assert_matches!(events[0], Err(ApiError::QuotaExceeded)); + } + + #[tokio::test] + async fn table_driven_event_kinds() { + struct TestCase { + name: &'static str, + event: serde_json::Value, + expect_first: fn(&ResponseEvent) -> bool, + expected_len: usize, + } + + fn is_created(ev: &ResponseEvent) -> bool { + matches!(ev, ResponseEvent::Created) + } + fn is_output(ev: &ResponseEvent) -> bool { + matches!(ev, ResponseEvent::OutputItemDone(_)) + } + fn is_completed(ev: &ResponseEvent) -> bool { + matches!(ev, ResponseEvent::Completed { .. }) + } + + let completed = json!({ + "type": "response.completed", + "response": { + "id": "c", + "usage": { + "input_tokens": 0, + "input_tokens_details": null, + "output_tokens": 0, + "output_tokens_details": null, + "total_tokens": 0 + }, + "output": [] + } + }); + + let cases = vec![ + TestCase { + name: "created", + event: json!({"type": "response.created", "response": {}}), + expect_first: is_created, + expected_len: 2, + }, + TestCase { + name: "output_item.done", + event: json!({ + "type": "response.output_item.done", + "item": { + "type": "message", + "role": "assistant", + "content": [ + {"type": "output_text", "text": "hi"} + ] + } + }), + expect_first: is_output, + expected_len: 2, + }, + TestCase { + name: "unknown", + event: json!({"type": "response.new_tool_event"}), + expect_first: is_completed, + expected_len: 1, + }, + ]; + + for case in cases { + let mut evs = vec![case.event]; + evs.push(completed.clone()); + + let out = run_sse(evs).await; + assert_eq!(out.len(), case.expected_len, "case {}", case.name); + assert!( + (case.expect_first)(&out[0]), + "first event mismatch in case {}", + case.name + ); + } + } + + #[test] + fn test_try_parse_retry_after() { + let err = Error { + r#type: None, + message: Some("Rate limit reached for gpt-5.1 in organization org- on tokens per min (TPM): Limit 1, Used 1, Requested 19304. Please try again in 28ms. Visit https://platform.openai.com/account/rate-limits to learn more.".to_string()), + code: Some("rate_limit_exceeded".to_string()), + plan_type: None, + resets_at: None, + }; + + let delay = try_parse_retry_after(&err); + assert_eq!(delay, Some(Duration::from_millis(28))); + } + + #[test] + fn test_try_parse_retry_after_no_delay() { + let err = Error { + r#type: None, + message: Some("Rate limit reached for gpt-5.1 in organization on tokens per min (TPM): Limit 30000, Used 6899, Requested 24050. Please try again in 1.898s. Visit https://platform.openai.com/account/rate-limits to learn more.".to_string()), + code: Some("rate_limit_exceeded".to_string()), + plan_type: None, + resets_at: None, + }; + let delay = try_parse_retry_after(&err); + assert_eq!(delay, Some(Duration::from_secs_f64(1.898))); + } + + #[test] + fn test_try_parse_retry_after_azure() { + let err = Error { + r#type: None, + message: Some("Rate limit exceeded. Try again in 35 seconds.".to_string()), + code: Some("rate_limit_exceeded".to_string()), + plan_type: None, + resets_at: None, + }; + let delay = try_parse_retry_after(&err); + assert_eq!(delay, Some(Duration::from_secs(35))); + } +} diff --git a/codex-rs/codex-api/src/telemetry.rs b/codex-rs/codex-api/src/telemetry.rs new file mode 100644 index 0000000000..d6a38b2af3 --- /dev/null +++ b/codex-rs/codex-api/src/telemetry.rs @@ -0,0 +1,84 @@ +use codex_client::Request; +use codex_client::RequestTelemetry; +use codex_client::Response; +use codex_client::RetryPolicy; +use codex_client::StreamResponse; +use codex_client::TransportError; +use codex_client::run_with_retry; +use http::StatusCode; +use std::future::Future; +use std::sync::Arc; +use std::time::Duration; +use tokio::time::Instant; + +/// Generic telemetry. +pub trait SseTelemetry: Send + Sync { + fn on_sse_poll( + &self, + result: &Result< + Option< + Result< + eventsource_stream::Event, + eventsource_stream::EventStreamError, + >, + >, + tokio::time::error::Elapsed, + >, + duration: Duration, + ); +} + +pub(crate) trait WithStatus { + fn status(&self) -> StatusCode; +} + +fn http_status(err: &TransportError) -> Option { + match err { + TransportError::Http { status, .. } => Some(*status), + _ => None, + } +} + +impl WithStatus for Response { + fn status(&self) -> StatusCode { + self.status + } +} + +impl WithStatus for StreamResponse { + fn status(&self) -> StatusCode { + self.status + } +} + +pub(crate) async fn run_with_request_telemetry( + policy: RetryPolicy, + telemetry: Option>, + make_request: impl FnMut() -> Request, + send: F, +) -> Result +where + T: WithStatus, + F: Clone + Fn(Request) -> Fut, + Fut: Future>, +{ + // Wraps `run_with_retry` to attach per-attempt request telemetry for both + // unary and streaming HTTP calls. + run_with_retry(policy, make_request, move |req, attempt| { + let telemetry = telemetry.clone(); + let send = send.clone(); + async move { + let start = Instant::now(); + let result = send(req).await; + if let Some(t) = telemetry.as_ref() { + let (status, err) = match &result { + Ok(resp) => (Some(resp.status()), None), + Err(err) => (http_status(err), Some(err)), + }; + t.on_request(attempt, status, err, start.elapsed()); + } + result + } + }) + .await +} diff --git a/codex-rs/codex-api/tests/clients.rs b/codex-rs/codex-api/tests/clients.rs new file mode 100644 index 0000000000..3dafaf74fa --- /dev/null +++ b/codex-rs/codex-api/tests/clients.rs @@ -0,0 +1,315 @@ +use std::sync::Arc; +use std::sync::Mutex; +use std::time::Duration; + +use anyhow::Result; +use async_trait::async_trait; +use bytes::Bytes; +use codex_api::AuthProvider; +use codex_api::ChatClient; +use codex_api::Provider; +use codex_api::ResponsesClient; +use codex_api::ResponsesOptions; +use codex_api::WireApi; +use codex_client::HttpTransport; +use codex_client::Request; +use codex_client::Response; +use codex_client::StreamResponse; +use codex_client::TransportError; +use codex_protocol::models::ContentItem; +use codex_protocol::models::ResponseItem; +use http::HeaderMap; +use http::StatusCode; +use pretty_assertions::assert_eq; +use serde_json::Value; + +fn assert_path_ends_with(requests: &[Request], suffix: &str) { + assert_eq!(requests.len(), 1); + let url = &requests[0].url; + assert!( + url.ends_with(suffix), + "expected url to end with {suffix}, got {url}" + ); +} + +#[derive(Debug, Default, Clone)] +struct RecordingState { + stream_requests: Arc>>, +} + +impl RecordingState { + fn record(&self, req: Request) { + let mut guard = self + .stream_requests + .lock() + .unwrap_or_else(|err| panic!("mutex poisoned: {err}")); + guard.push(req); + } + + fn take_stream_requests(&self) -> Vec { + let mut guard = self + .stream_requests + .lock() + .unwrap_or_else(|err| panic!("mutex poisoned: {err}")); + std::mem::take(&mut *guard) + } +} + +#[derive(Clone)] +struct RecordingTransport { + state: RecordingState, +} + +impl RecordingTransport { + fn new(state: RecordingState) -> Self { + Self { state } + } +} + +#[async_trait] +impl HttpTransport for RecordingTransport { + async fn execute(&self, _req: Request) -> Result { + Err(TransportError::Build("execute should not run".to_string())) + } + + async fn stream(&self, req: Request) -> Result { + self.state.record(req); + + let stream = futures::stream::iter(Vec::>::new()); + Ok(StreamResponse { + status: StatusCode::OK, + headers: HeaderMap::new(), + bytes: Box::pin(stream), + }) + } +} + +#[derive(Clone, Default)] +struct NoAuth; + +impl AuthProvider for NoAuth { + fn bearer_token(&self) -> Option { + None + } +} + +#[derive(Clone)] +struct StaticAuth { + token: String, + account_id: String, +} + +impl StaticAuth { + fn new(token: &str, account_id: &str) -> Self { + Self { + token: token.to_string(), + account_id: account_id.to_string(), + } + } +} + +impl AuthProvider for StaticAuth { + fn bearer_token(&self) -> Option { + Some(self.token.clone()) + } + + fn account_id(&self) -> Option { + Some(self.account_id.clone()) + } +} + +fn provider(name: &str, wire: WireApi) -> Provider { + Provider { + name: name.to_string(), + base_url: "https://example.com/v1".to_string(), + query_params: None, + wire, + headers: HeaderMap::new(), + retry: codex_api::provider::RetryConfig { + max_attempts: 1, + base_delay: Duration::from_millis(1), + retry_429: false, + retry_5xx: false, + retry_transport: true, + }, + stream_idle_timeout: Duration::from_millis(10), + } +} + +#[derive(Clone)] +struct FlakyTransport { + state: Arc>, +} + +impl Default for FlakyTransport { + fn default() -> Self { + Self::new() + } +} + +impl FlakyTransport { + fn new() -> Self { + Self { + state: Arc::new(Mutex::new(0)), + } + } + + fn attempts(&self) -> i64 { + *self + .state + .lock() + .unwrap_or_else(|err| panic!("mutex poisoned: {err}")) + } +} + +#[async_trait] +impl HttpTransport for FlakyTransport { + async fn execute(&self, _req: Request) -> Result { + Err(TransportError::Build("execute should not run".to_string())) + } + + async fn stream(&self, _req: Request) -> Result { + let mut attempts = self + .state + .lock() + .unwrap_or_else(|err| panic!("mutex poisoned: {err}")); + *attempts += 1; + + if *attempts == 1 { + return Err(TransportError::Network("first attempt fails".to_string())); + } + + let stream = futures::stream::iter(vec![Ok(Bytes::from( + r#"event: message +data: {"id":"resp-1","output":[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"hi"}]}]} + +"#, + ))]); + + Ok(StreamResponse { + status: StatusCode::OK, + headers: HeaderMap::new(), + bytes: Box::pin(stream), + }) + } +} + +#[tokio::test] +async fn chat_client_uses_chat_completions_path_for_chat_wire() -> Result<()> { + let state = RecordingState::default(); + let transport = RecordingTransport::new(state.clone()); + let client = ChatClient::new(transport, provider("openai", WireApi::Chat), NoAuth); + + let body = serde_json::json!({ "echo": true }); + let _stream = client.stream(body, HeaderMap::new()).await?; + + let requests = state.take_stream_requests(); + assert_path_ends_with(&requests, "/chat/completions"); + Ok(()) +} + +#[tokio::test] +async fn chat_client_uses_responses_path_for_responses_wire() -> Result<()> { + let state = RecordingState::default(); + let transport = RecordingTransport::new(state.clone()); + let client = ChatClient::new(transport, provider("openai", WireApi::Responses), NoAuth); + + let body = serde_json::json!({ "echo": true }); + let _stream = client.stream(body, HeaderMap::new()).await?; + + let requests = state.take_stream_requests(); + assert_path_ends_with(&requests, "/responses"); + Ok(()) +} + +#[tokio::test] +async fn responses_client_uses_responses_path_for_responses_wire() -> Result<()> { + let state = RecordingState::default(); + let transport = RecordingTransport::new(state.clone()); + let client = ResponsesClient::new(transport, provider("openai", WireApi::Responses), NoAuth); + + let body = serde_json::json!({ "echo": true }); + let _stream = client.stream(body, HeaderMap::new()).await?; + + let requests = state.take_stream_requests(); + assert_path_ends_with(&requests, "/responses"); + Ok(()) +} + +#[tokio::test] +async fn responses_client_uses_chat_path_for_chat_wire() -> Result<()> { + let state = RecordingState::default(); + let transport = RecordingTransport::new(state.clone()); + let client = ResponsesClient::new(transport, provider("openai", WireApi::Chat), NoAuth); + + let body = serde_json::json!({ "echo": true }); + let _stream = client.stream(body, HeaderMap::new()).await?; + + let requests = state.take_stream_requests(); + assert_path_ends_with(&requests, "/chat/completions"); + Ok(()) +} + +#[tokio::test] +async fn streaming_client_adds_auth_headers() -> Result<()> { + let state = RecordingState::default(); + let transport = RecordingTransport::new(state.clone()); + let auth = StaticAuth::new("secret-token", "acct-1"); + let client = ResponsesClient::new(transport, provider("openai", WireApi::Responses), auth); + + let body = serde_json::json!({ "model": "gpt-test" }); + let _stream = client.stream(body, HeaderMap::new()).await?; + + let requests = state.take_stream_requests(); + assert_eq!(requests.len(), 1); + let req = &requests[0]; + + let auth_header = req.headers.get(http::header::AUTHORIZATION); + assert!(auth_header.is_some(), "missing auth header"); + assert_eq!( + auth_header.unwrap().to_str().ok(), + Some("Bearer secret-token") + ); + + let account_header = req.headers.get("ChatGPT-Account-ID"); + assert!(account_header.is_some(), "missing account header"); + assert_eq!(account_header.unwrap().to_str().ok(), Some("acct-1")); + + let accept_header = req.headers.get(http::header::ACCEPT); + assert!(accept_header.is_some(), "missing Accept header"); + assert_eq!( + accept_header.unwrap().to_str().ok(), + Some("text/event-stream") + ); + Ok(()) +} + +#[tokio::test] +async fn streaming_client_retries_on_transport_error() -> Result<()> { + let transport = FlakyTransport::new(); + + let mut provider = provider("openai", WireApi::Responses); + provider.retry.max_attempts = 2; + + let client = ResponsesClient::new(transport.clone(), provider, NoAuth); + + let prompt = codex_api::Prompt { + instructions: "Say hi".to_string(), + input: vec![ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::InputText { + text: "hi".to_string(), + }], + }], + tools: Vec::::new(), + parallel_tool_calls: false, + output_schema: None, + }; + + let options = ResponsesOptions::default(); + + let _stream = client.stream_prompt("gpt-test", &prompt, options).await?; + assert_eq!(transport.attempts(), 2); + Ok(()) +} diff --git a/codex-rs/codex-api/tests/sse_end_to_end.rs b/codex-rs/codex-api/tests/sse_end_to_end.rs new file mode 100644 index 0000000000..b91cf3a5d8 --- /dev/null +++ b/codex-rs/codex-api/tests/sse_end_to_end.rs @@ -0,0 +1,229 @@ +use std::time::Duration; + +use anyhow::Result; +use async_trait::async_trait; +use bytes::Bytes; +use codex_api::AggregateStreamExt; +use codex_api::AuthProvider; +use codex_api::Provider; +use codex_api::ResponseEvent; +use codex_api::ResponsesClient; +use codex_api::WireApi; +use codex_client::HttpTransport; +use codex_client::Request; +use codex_client::Response; +use codex_client::StreamResponse; +use codex_client::TransportError; +use codex_protocol::models::ContentItem; +use codex_protocol::models::ResponseItem; +use futures::StreamExt; +use http::HeaderMap; +use http::StatusCode; +use pretty_assertions::assert_eq; +use serde_json::Value; + +#[derive(Clone)] +struct FixtureSseTransport { + body: String, +} + +impl FixtureSseTransport { + fn new(body: String) -> Self { + Self { body } + } +} + +#[async_trait] +impl HttpTransport for FixtureSseTransport { + async fn execute(&self, _req: Request) -> Result { + Err(TransportError::Build("execute should not run".to_string())) + } + + async fn stream(&self, _req: Request) -> Result { + let stream = futures::stream::iter(vec![Ok::(Bytes::from( + self.body.clone(), + ))]); + Ok(StreamResponse { + status: StatusCode::OK, + headers: HeaderMap::new(), + bytes: Box::pin(stream), + }) + } +} + +#[derive(Clone, Default)] +struct NoAuth; + +impl AuthProvider for NoAuth { + fn bearer_token(&self) -> Option { + None + } +} + +fn provider(name: &str, wire: WireApi) -> Provider { + Provider { + name: name.to_string(), + base_url: "https://example.com/v1".to_string(), + query_params: None, + wire, + headers: HeaderMap::new(), + retry: codex_api::provider::RetryConfig { + max_attempts: 1, + base_delay: Duration::from_millis(1), + retry_429: false, + retry_5xx: false, + retry_transport: true, + }, + stream_idle_timeout: Duration::from_millis(50), + } +} + +fn build_responses_body(events: Vec) -> String { + let mut body = String::new(); + for e in events { + let kind = e + .get("type") + .and_then(|v| v.as_str()) + .unwrap_or_else(|| panic!("fixture event missing type in SSE fixture: {e}")); + if e.as_object().map(|o| o.len() == 1).unwrap_or(false) { + body.push_str(&format!("event: {kind}\n\n")); + } else { + body.push_str(&format!("event: {kind}\ndata: {e}\n\n")); + } + } + body +} + +#[tokio::test] +async fn responses_stream_parses_items_and_completed_end_to_end() -> Result<()> { + let item1 = serde_json::json!({ + "type": "response.output_item.done", + "item": { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "Hello"}] + } + }); + + let item2 = serde_json::json!({ + "type": "response.output_item.done", + "item": { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "World"}] + } + }); + + let completed = serde_json::json!({ + "type": "response.completed", + "response": { "id": "resp1" } + }); + + let body = build_responses_body(vec![item1, item2, completed]); + let transport = FixtureSseTransport::new(body); + let client = ResponsesClient::new(transport, provider("openai", WireApi::Responses), NoAuth); + + let mut stream = client + .stream(serde_json::json!({"echo": true}), HeaderMap::new()) + .await?; + + let mut events = Vec::new(); + while let Some(ev) = stream.next().await { + events.push(ev?); + } + + let events: Vec = events + .into_iter() + .filter(|ev| !matches!(ev, ResponseEvent::RateLimits(_))) + .collect(); + + assert_eq!(events.len(), 3); + + match &events[0] { + ResponseEvent::OutputItemDone(ResponseItem::Message { role, .. }) => { + assert_eq!(role, "assistant"); + } + other => panic!("unexpected first event: {other:?}"), + } + + match &events[1] { + ResponseEvent::OutputItemDone(ResponseItem::Message { role, .. }) => { + assert_eq!(role, "assistant"); + } + other => panic!("unexpected second event: {other:?}"), + } + + match &events[2] { + ResponseEvent::Completed { + response_id, + token_usage, + } => { + assert_eq!(response_id, "resp1"); + assert!(token_usage.is_none()); + } + other => panic!("unexpected third event: {other:?}"), + } + + Ok(()) +} + +#[tokio::test] +async fn responses_stream_aggregates_output_text_deltas() -> Result<()> { + let delta1 = serde_json::json!({ + "type": "response.output_text.delta", + "delta": "Hello, " + }); + + let delta2 = serde_json::json!({ + "type": "response.output_text.delta", + "delta": "world" + }); + + let completed = serde_json::json!({ + "type": "response.completed", + "response": { "id": "resp-agg" } + }); + + let body = build_responses_body(vec![delta1, delta2, completed]); + let transport = FixtureSseTransport::new(body); + let client = ResponsesClient::new(transport, provider("openai", WireApi::Responses), NoAuth); + + let stream = client + .stream(serde_json::json!({"echo": true}), HeaderMap::new()) + .await?; + + let mut stream = stream.aggregate(); + let mut events = Vec::new(); + while let Some(ev) = stream.next().await { + events.push(ev?); + } + + let events: Vec = events + .into_iter() + .filter(|ev| !matches!(ev, ResponseEvent::RateLimits(_))) + .collect(); + + assert_eq!(events.len(), 2); + + match &events[0] { + ResponseEvent::OutputItemDone(ResponseItem::Message { content, .. }) => { + let mut aggregated = String::new(); + for item in content { + if let ContentItem::OutputText { text } = item { + aggregated.push_str(text); + } + } + assert_eq!(aggregated, "Hello, world"); + } + other => panic!("unexpected first event: {other:?}"), + } + + match &events[1] { + ResponseEvent::Completed { response_id, .. } => { + assert_eq!(response_id, "resp-agg"); + } + other => panic!("unexpected second event: {other:?}"), + } + + Ok(()) +} diff --git a/codex-rs/codex-client/Cargo.toml b/codex-rs/codex-client/Cargo.toml new file mode 100644 index 0000000000..2defe12fda --- /dev/null +++ b/codex-rs/codex-client/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "codex-client" +version.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +async-trait = { workspace = true } +bytes = { workspace = true } +futures = { workspace = true } +http = { workspace = true } +reqwest = { workspace = true, features = ["json", "stream"] } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["macros", "rt", "time", "sync"] } +rand = { workspace = true } +eventsource-stream = { workspace = true } + +[lints] +workspace = true diff --git a/codex-rs/codex-client/README.md b/codex-rs/codex-client/README.md new file mode 100644 index 0000000000..045ee7b343 --- /dev/null +++ b/codex-rs/codex-client/README.md @@ -0,0 +1,8 @@ +# codex-client + +Generic transport layer that wraps HTTP requests, retries, and streaming primitives without any Codex/OpenAI awareness. + +- Defines `HttpTransport` and a default `ReqwestTransport` plus thin `Request`/`Response` types. +- Provides retry utilities (`RetryPolicy`, `RetryOn`, `run_with_retry`, `backoff`) that callers plug into for unary and streaming calls. +- Supplies the `sse_stream` helper to turn byte streams into raw SSE `data:` frames with idle timeouts and surfaced stream errors. +- Consumed by higher-level crates like `codex-api`; it stays neutral on endpoints, headers, or API-specific error shapes. diff --git a/codex-rs/codex-client/src/error.rs b/codex-rs/codex-client/src/error.rs new file mode 100644 index 0000000000..086b91a505 --- /dev/null +++ b/codex-rs/codex-client/src/error.rs @@ -0,0 +1,29 @@ +use http::HeaderMap; +use http::StatusCode; +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum TransportError { + #[error("http {status}: {body:?}")] + Http { + status: StatusCode, + headers: Option, + body: Option, + }, + #[error("retry limit reached")] + RetryLimit, + #[error("timeout")] + Timeout, + #[error("network error: {0}")] + Network(String), + #[error("request build error: {0}")] + Build(String), +} + +#[derive(Debug, Error)] +pub enum StreamError { + #[error("stream failed: {0}")] + Stream(String), + #[error("timeout")] + Timeout, +} diff --git a/codex-rs/codex-client/src/lib.rs b/codex-rs/codex-client/src/lib.rs new file mode 100644 index 0000000000..3ac00a21a8 --- /dev/null +++ b/codex-rs/codex-client/src/lib.rs @@ -0,0 +1,21 @@ +mod error; +mod request; +mod retry; +mod sse; +mod telemetry; +mod transport; + +pub use crate::error::StreamError; +pub use crate::error::TransportError; +pub use crate::request::Request; +pub use crate::request::Response; +pub use crate::retry::RetryOn; +pub use crate::retry::RetryPolicy; +pub use crate::retry::backoff; +pub use crate::retry::run_with_retry; +pub use crate::sse::sse_stream; +pub use crate::telemetry::RequestTelemetry; +pub use crate::transport::ByteStream; +pub use crate::transport::HttpTransport; +pub use crate::transport::ReqwestTransport; +pub use crate::transport::StreamResponse; diff --git a/codex-rs/codex-client/src/request.rs b/codex-rs/codex-client/src/request.rs new file mode 100644 index 0000000000..f3d205de99 --- /dev/null +++ b/codex-rs/codex-client/src/request.rs @@ -0,0 +1,39 @@ +use bytes::Bytes; +use http::Method; +use reqwest::header::HeaderMap; +use serde::Serialize; +use serde_json::Value; +use std::time::Duration; + +#[derive(Debug, Clone)] +pub struct Request { + pub method: Method, + pub url: String, + pub headers: HeaderMap, + pub body: Option, + pub timeout: Option, +} + +impl Request { + pub fn new(method: Method, url: String) -> Self { + Self { + method, + url, + headers: HeaderMap::new(), + body: None, + timeout: None, + } + } + + pub fn with_json(mut self, body: &T) -> Self { + self.body = serde_json::to_value(body).ok(); + self + } +} + +#[derive(Debug, Clone)] +pub struct Response { + pub status: http::StatusCode, + pub headers: HeaderMap, + pub body: Bytes, +} diff --git a/codex-rs/codex-client/src/retry.rs b/codex-rs/codex-client/src/retry.rs new file mode 100644 index 0000000000..c7bdd34b1e --- /dev/null +++ b/codex-rs/codex-client/src/retry.rs @@ -0,0 +1,73 @@ +use crate::error::TransportError; +use crate::request::Request; +use rand::Rng; +use std::future::Future; +use std::time::Duration; +use tokio::time::sleep; + +#[derive(Debug, Clone)] +pub struct RetryPolicy { + pub max_attempts: u64, + pub base_delay: Duration, + pub retry_on: RetryOn, +} + +#[derive(Debug, Clone)] +pub struct RetryOn { + pub retry_429: bool, + pub retry_5xx: bool, + pub retry_transport: bool, +} + +impl RetryOn { + pub fn should_retry(&self, err: &TransportError, attempt: u64, max_attempts: u64) -> bool { + if attempt >= max_attempts { + return false; + } + match err { + TransportError::Http { status, .. } => { + (self.retry_429 && status.as_u16() == 429) + || (self.retry_5xx && status.is_server_error()) + } + TransportError::Timeout | TransportError::Network(_) => self.retry_transport, + _ => false, + } + } +} + +pub fn backoff(base: Duration, attempt: u64) -> Duration { + if attempt == 0 { + return base; + } + let exp = 2u64.saturating_pow(attempt as u32 - 1); + let millis = base.as_millis() as u64; + let raw = millis.saturating_mul(exp); + let jitter: f64 = rand::rng().random_range(0.9..1.1); + Duration::from_millis((raw as f64 * jitter) as u64) +} + +pub async fn run_with_retry( + policy: RetryPolicy, + mut make_req: impl FnMut() -> Request, + op: F, +) -> Result +where + F: Fn(Request, u64) -> Fut, + Fut: Future>, +{ + for attempt in 0..=policy.max_attempts { + let req = make_req(); + match op(req, attempt).await { + Ok(resp) => return Ok(resp), + Err(err) + if policy + .retry_on + .should_retry(&err, attempt, policy.max_attempts) => + { + sleep(backoff(policy.base_delay, attempt + 1)).await; + } + Err(err) => return Err(err), + } + } + Err(TransportError::RetryLimit) +} diff --git a/codex-rs/codex-client/src/sse.rs b/codex-rs/codex-client/src/sse.rs new file mode 100644 index 0000000000..f3aba3a2c5 --- /dev/null +++ b/codex-rs/codex-client/src/sse.rs @@ -0,0 +1,48 @@ +use crate::error::StreamError; +use crate::transport::ByteStream; +use eventsource_stream::Eventsource; +use futures::StreamExt; +use tokio::sync::mpsc; +use tokio::time::Duration; +use tokio::time::timeout; + +/// Minimal SSE helper that forwards raw `data:` frames as UTF-8 strings. +/// +/// Errors and idle timeouts are sent as `Err(StreamError)` before the task exits. +pub fn sse_stream( + stream: ByteStream, + idle_timeout: Duration, + tx: mpsc::Sender>, +) { + tokio::spawn(async move { + let mut stream = stream + .map(|res| res.map_err(|e| StreamError::Stream(e.to_string()))) + .eventsource(); + + loop { + match timeout(idle_timeout, stream.next()).await { + Ok(Some(Ok(ev))) => { + if tx.send(Ok(ev.data.clone())).await.is_err() { + return; + } + } + Ok(Some(Err(e))) => { + let _ = tx.send(Err(StreamError::Stream(e.to_string()))).await; + return; + } + Ok(None) => { + let _ = tx + .send(Err(StreamError::Stream( + "stream closed before completion".into(), + ))) + .await; + return; + } + Err(_) => { + let _ = tx.send(Err(StreamError::Timeout)).await; + return; + } + } + } + }); +} diff --git a/codex-rs/codex-client/src/telemetry.rs b/codex-rs/codex-client/src/telemetry.rs new file mode 100644 index 0000000000..457d47f4fc --- /dev/null +++ b/codex-rs/codex-client/src/telemetry.rs @@ -0,0 +1,14 @@ +use crate::error::TransportError; +use http::StatusCode; +use std::time::Duration; + +/// API specific telemetry. +pub trait RequestTelemetry: Send + Sync { + fn on_request( + &self, + attempt: u64, + status: Option, + error: Option<&TransportError>, + duration: Duration, + ); +} diff --git a/codex-rs/codex-client/src/transport.rs b/codex-rs/codex-client/src/transport.rs new file mode 100644 index 0000000000..f64e72fb68 --- /dev/null +++ b/codex-rs/codex-client/src/transport.rs @@ -0,0 +1,107 @@ +use crate::error::TransportError; +use crate::request::Request; +use crate::request::Response; +use async_trait::async_trait; +use bytes::Bytes; +use futures::StreamExt; +use futures::stream::BoxStream; +use http::HeaderMap; +use http::Method; +use http::StatusCode; + +pub type ByteStream = BoxStream<'static, Result>; + +pub struct StreamResponse { + pub status: StatusCode, + pub headers: HeaderMap, + pub bytes: ByteStream, +} + +#[async_trait] +pub trait HttpTransport: Send + Sync { + async fn execute(&self, req: Request) -> Result; + async fn stream(&self, req: Request) -> Result; +} + +#[derive(Clone, Debug)] +pub struct ReqwestTransport { + client: reqwest::Client, +} + +impl ReqwestTransport { + pub fn new(client: reqwest::Client) -> Self { + Self { client } + } + + fn build(&self, req: Request) -> Result { + let mut builder = self + .client + .request( + Method::from_bytes(req.method.as_str().as_bytes()).unwrap_or(Method::GET), + &req.url, + ) + .headers(req.headers); + if let Some(timeout) = req.timeout { + builder = builder.timeout(timeout); + } + if let Some(body) = req.body { + builder = builder.json(&body); + } + Ok(builder) + } + + fn map_error(err: reqwest::Error) -> TransportError { + if err.is_timeout() { + TransportError::Timeout + } else { + TransportError::Network(err.to_string()) + } + } +} + +#[async_trait] +impl HttpTransport for ReqwestTransport { + async fn execute(&self, req: Request) -> Result { + let builder = self.build(req)?; + let resp = builder.send().await.map_err(Self::map_error)?; + let status = resp.status(); + let headers = resp.headers().clone(); + let bytes = resp.bytes().await.map_err(Self::map_error)?; + if !status.is_success() { + let body = String::from_utf8(bytes.to_vec()).ok(); + return Err(TransportError::Http { + status, + headers: Some(headers), + body, + }); + } + Ok(Response { + status, + headers, + body: bytes, + }) + } + + async fn stream(&self, req: Request) -> Result { + let builder = self.build(req)?; + let resp = builder.send().await.map_err(Self::map_error)?; + let status = resp.status(); + let headers = resp.headers().clone(); + if !status.is_success() { + let body = resp.text().await.ok(); + return Err(TransportError::Http { + status, + headers: Some(headers), + body, + }); + } + let stream = resp + .bytes_stream() + .map(|result| result.map_err(Self::map_error)); + Ok(StreamResponse { + status, + headers, + bytes: Box::pin(stream), + }) + } +} diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index d6db71973a..705bab57df 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -18,12 +18,12 @@ askama = { workspace = true } async-channel = { workspace = true } async-trait = { workspace = true } base64 = { workspace = true } -bytes = { workspace = true } chrono = { workspace = true, features = ["serde"] } chardetng = { workspace = true } codex-app-server-protocol = { workspace = true } codex-apply-patch = { workspace = true } codex-async-utils = { workspace = true } +codex-api = { workspace = true } codex-execpolicy = { workspace = true } codex-file-search = { workspace = true } codex-git = { workspace = true } diff --git a/codex-rs/core/src/api_bridge.rs b/codex-rs/core/src/api_bridge.rs new file mode 100644 index 0000000000..b9f802ae66 --- /dev/null +++ b/codex-rs/core/src/api_bridge.rs @@ -0,0 +1,154 @@ +use chrono::DateTime; +use chrono::Utc; +use codex_api::AuthProvider as ApiAuthProvider; +use codex_api::TransportError; +use codex_api::error::ApiError; +use codex_api::rate_limits::parse_rate_limit; +use http::HeaderMap; +use serde::Deserialize; + +use crate::auth::CodexAuth; +use crate::error::CodexErr; +use crate::error::RetryLimitReachedError; +use crate::error::UnexpectedResponseError; +use crate::error::UsageLimitReachedError; +use crate::model_provider_info::ModelProviderInfo; +use crate::token_data::PlanType; + +pub(crate) fn map_api_error(err: ApiError) -> CodexErr { + match err { + ApiError::ContextWindowExceeded => CodexErr::ContextWindowExceeded, + ApiError::QuotaExceeded => CodexErr::QuotaExceeded, + ApiError::UsageNotIncluded => CodexErr::UsageNotIncluded, + ApiError::Retryable { message, delay } => CodexErr::Stream(message, delay), + ApiError::Stream(msg) => CodexErr::Stream(msg, None), + ApiError::Api { status, message } => CodexErr::UnexpectedStatus(UnexpectedResponseError { + status, + body: message, + request_id: None, + }), + ApiError::Transport(transport) => match transport { + TransportError::Http { + status, + headers, + body, + } => { + if status == http::StatusCode::INTERNAL_SERVER_ERROR { + CodexErr::InternalServerError + } else if status == http::StatusCode::TOO_MANY_REQUESTS { + if let Some(body) = body + && let Ok(err) = serde_json::from_str::(&body) + { + if err.error.error_type.as_deref() == Some("usage_limit_reached") { + let rate_limits = headers.as_ref().and_then(parse_rate_limit); + let resets_at = err + .error + .resets_at + .and_then(|seconds| DateTime::::from_timestamp(seconds, 0)); + return CodexErr::UsageLimitReached(UsageLimitReachedError { + plan_type: err.error.plan_type, + resets_at, + rate_limits, + }); + } else if err.error.error_type.as_deref() == Some("usage_not_included") { + return CodexErr::UsageNotIncluded; + } + } + + CodexErr::RetryLimit(RetryLimitReachedError { + status, + request_id: extract_request_id(headers.as_ref()), + }) + } else { + CodexErr::UnexpectedStatus(UnexpectedResponseError { + status, + body: body.unwrap_or_default(), + request_id: extract_request_id(headers.as_ref()), + }) + } + } + TransportError::RetryLimit => CodexErr::RetryLimit(RetryLimitReachedError { + status: http::StatusCode::INTERNAL_SERVER_ERROR, + request_id: None, + }), + TransportError::Timeout => CodexErr::Timeout, + TransportError::Network(msg) | TransportError::Build(msg) => { + CodexErr::Stream(msg, None) + } + }, + ApiError::RateLimit(msg) => CodexErr::Stream(msg, None), + } +} + +fn extract_request_id(headers: Option<&HeaderMap>) -> Option { + headers.and_then(|map| { + ["cf-ray", "x-request-id", "x-oai-request-id"] + .iter() + .find_map(|name| { + map.get(*name) + .and_then(|v| v.to_str().ok()) + .map(str::to_string) + }) + }) +} + +pub(crate) async fn auth_provider_from_auth( + auth: Option, + provider: &ModelProviderInfo, +) -> crate::error::Result { + if let Some(api_key) = provider.api_key()? { + return Ok(CoreAuthProvider { + token: Some(api_key), + account_id: None, + }); + } + + if let Some(token) = provider.experimental_bearer_token.clone() { + return Ok(CoreAuthProvider { + token: Some(token), + account_id: None, + }); + } + + if let Some(auth) = auth { + let token = auth.get_token().await?; + Ok(CoreAuthProvider { + token: Some(token), + account_id: auth.get_account_id(), + }) + } else { + Ok(CoreAuthProvider { + token: None, + account_id: None, + }) + } +} + +#[derive(Debug, Deserialize)] +struct UsageErrorResponse { + error: UsageErrorBody, +} + +#[derive(Debug, Deserialize)] +struct UsageErrorBody { + #[serde(rename = "type")] + error_type: Option, + plan_type: Option, + resets_at: Option, +} + +#[derive(Clone, Default)] +pub(crate) struct CoreAuthProvider { + token: Option, + account_id: Option, +} + +impl ApiAuthProvider for CoreAuthProvider { + fn bearer_token(&self) -> Option { + self.token.clone() + } + + fn account_id(&self) -> Option { + self.account_id.clone() + } +} diff --git a/codex-rs/core/src/chat_completions.rs b/codex-rs/core/src/chat_completions.rs deleted file mode 100644 index 785a4d4ce5..0000000000 --- a/codex-rs/core/src/chat_completions.rs +++ /dev/null @@ -1,981 +0,0 @@ -use std::time::Duration; - -use crate::ModelProviderInfo; -use crate::client_common::Prompt; -use crate::client_common::ResponseEvent; -use crate::client_common::ResponseStream; -use crate::default_client::CodexHttpClient; -use crate::error::CodexErr; -use crate::error::ConnectionFailedError; -use crate::error::ResponseStreamFailed; -use crate::error::Result; -use crate::error::RetryLimitReachedError; -use crate::error::UnexpectedResponseError; -use crate::model_family::ModelFamily; -use crate::tools::spec::create_tools_json_for_chat_completions_api; -use crate::util::backoff; -use bytes::Bytes; -use codex_otel::otel_event_manager::OtelEventManager; -use codex_protocol::models::ContentItem; -use codex_protocol::models::FunctionCallOutputContentItem; -use codex_protocol::models::ReasoningItemContent; -use codex_protocol::models::ResponseItem; -use codex_protocol::protocol::SessionSource; -use codex_protocol::protocol::SubAgentSource; -use eventsource_stream::Eventsource; -use futures::Stream; -use futures::StreamExt; -use futures::TryStreamExt; -use reqwest::StatusCode; -use serde_json::json; -use std::pin::Pin; -use std::task::Context; -use std::task::Poll; -use tokio::sync::mpsc; -use tokio::time::timeout; -use tracing::debug; -use tracing::trace; - -/// Implementation for the classic Chat Completions API. -pub(crate) async fn stream_chat_completions( - prompt: &Prompt, - model_family: &ModelFamily, - client: &CodexHttpClient, - provider: &ModelProviderInfo, - otel_event_manager: &OtelEventManager, - session_source: &SessionSource, -) -> Result { - if prompt.output_schema.is_some() { - return Err(CodexErr::UnsupportedOperation( - "output_schema is not supported for Chat Completions API".to_string(), - )); - } - - // Build messages array - let mut messages = Vec::::new(); - - let full_instructions = prompt.get_full_instructions(model_family); - messages.push(json!({"role": "system", "content": full_instructions})); - - let input = prompt.get_formatted_input(); - - // Pre-scan: map Reasoning blocks to the adjacent assistant anchor after the last user. - // - If the last emitted message is a user message, drop all reasoning. - // - Otherwise, for each Reasoning item after the last user message, attach it - // to the immediate previous assistant message (stop turns) or the immediate - // next assistant anchor (tool-call turns: function/local shell call, or assistant message). - let mut reasoning_by_anchor_index: std::collections::HashMap = - std::collections::HashMap::new(); - - // Determine the last role that would be emitted to Chat Completions. - let mut last_emitted_role: Option<&str> = None; - for item in &input { - match item { - ResponseItem::Message { role, .. } => last_emitted_role = Some(role.as_str()), - ResponseItem::FunctionCall { .. } | ResponseItem::LocalShellCall { .. } => { - last_emitted_role = Some("assistant") - } - ResponseItem::FunctionCallOutput { .. } => last_emitted_role = Some("tool"), - ResponseItem::Reasoning { .. } | ResponseItem::Other => {} - ResponseItem::CustomToolCall { .. } => {} - ResponseItem::CustomToolCallOutput { .. } => {} - ResponseItem::WebSearchCall { .. } => {} - ResponseItem::GhostSnapshot { .. } => {} - ResponseItem::CompactionSummary { .. } => {} - } - } - - // Find the last user message index in the input. - let mut last_user_index: Option = None; - for (idx, item) in input.iter().enumerate() { - if let ResponseItem::Message { role, .. } = item - && role == "user" - { - last_user_index = Some(idx); - } - } - - // Attach reasoning only if the conversation does not end with a user message. - if !matches!(last_emitted_role, Some("user")) { - for (idx, item) in input.iter().enumerate() { - // Only consider reasoning that appears after the last user message. - if let Some(u_idx) = last_user_index - && idx <= u_idx - { - continue; - } - - if let ResponseItem::Reasoning { - content: Some(items), - .. - } = item - { - let mut text = String::new(); - for entry in items { - match entry { - ReasoningItemContent::ReasoningText { text: segment } - | ReasoningItemContent::Text { text: segment } => text.push_str(segment), - } - } - if text.trim().is_empty() { - continue; - } - - // Prefer immediate previous assistant message (stop turns) - let mut attached = false; - if idx > 0 - && let ResponseItem::Message { role, .. } = &input[idx - 1] - && role == "assistant" - { - reasoning_by_anchor_index - .entry(idx - 1) - .and_modify(|v| v.push_str(&text)) - .or_insert(text.clone()); - attached = true; - } - - // Otherwise, attach to immediate next assistant anchor (tool-calls or assistant message) - if !attached && idx + 1 < input.len() { - match &input[idx + 1] { - ResponseItem::FunctionCall { .. } | ResponseItem::LocalShellCall { .. } => { - reasoning_by_anchor_index - .entry(idx + 1) - .and_modify(|v| v.push_str(&text)) - .or_insert(text.clone()); - } - ResponseItem::Message { role, .. } if role == "assistant" => { - reasoning_by_anchor_index - .entry(idx + 1) - .and_modify(|v| v.push_str(&text)) - .or_insert(text.clone()); - } - _ => {} - } - } - } - } - } - - // Track last assistant text we emitted to avoid duplicate assistant messages - // in the outbound Chat Completions payload (can happen if a final - // aggregated assistant message was recorded alongside an earlier partial). - let mut last_assistant_text: Option = None; - - for (idx, item) in input.iter().enumerate() { - match item { - ResponseItem::Message { role, content, .. } => { - // Build content either as a plain string (typical for assistant text) - // or as an array of content items when images are present (user/tool multimodal). - let mut text = String::new(); - let mut items: Vec = Vec::new(); - let mut saw_image = false; - - for c in content { - match c { - ContentItem::InputText { text: t } - | ContentItem::OutputText { text: t } => { - text.push_str(t); - items.push(json!({"type":"text","text": t})); - } - ContentItem::InputImage { image_url } => { - saw_image = true; - items.push(json!({"type":"image_url","image_url": {"url": image_url}})); - } - } - } - - // Skip exact-duplicate assistant messages. - if role == "assistant" { - if let Some(prev) = &last_assistant_text - && prev == &text - { - continue; - } - last_assistant_text = Some(text.clone()); - } - - // For assistant messages, always send a plain string for compatibility. - // For user messages, if an image is present, send an array of content items. - let content_value = if role == "assistant" { - json!(text) - } else if saw_image { - json!(items) - } else { - json!(text) - }; - - let mut msg = json!({"role": role, "content": content_value}); - if role == "assistant" - && let Some(reasoning) = reasoning_by_anchor_index.get(&idx) - && let Some(obj) = msg.as_object_mut() - { - obj.insert("reasoning".to_string(), json!(reasoning)); - } - messages.push(msg); - } - ResponseItem::FunctionCall { - name, - arguments, - call_id, - .. - } => { - let mut msg = json!({ - "role": "assistant", - "content": null, - "tool_calls": [{ - "id": call_id, - "type": "function", - "function": { - "name": name, - "arguments": arguments, - } - }] - }); - if let Some(reasoning) = reasoning_by_anchor_index.get(&idx) - && let Some(obj) = msg.as_object_mut() - { - obj.insert("reasoning".to_string(), json!(reasoning)); - } - messages.push(msg); - } - ResponseItem::LocalShellCall { - id, - call_id: _, - status, - action, - } => { - // Confirm with API team. - let mut msg = json!({ - "role": "assistant", - "content": null, - "tool_calls": [{ - "id": id.clone().unwrap_or_else(|| "".to_string()), - "type": "local_shell_call", - "status": status, - "action": action, - }] - }); - if let Some(reasoning) = reasoning_by_anchor_index.get(&idx) - && let Some(obj) = msg.as_object_mut() - { - obj.insert("reasoning".to_string(), json!(reasoning)); - } - messages.push(msg); - } - ResponseItem::FunctionCallOutput { call_id, output } => { - // Prefer structured content items when available (e.g., images) - // otherwise fall back to the legacy plain-string content. - let content_value = if let Some(items) = &output.content_items { - let mapped: Vec = items - .iter() - .map(|it| match it { - FunctionCallOutputContentItem::InputText { text } => { - json!({"type":"text","text": text}) - } - FunctionCallOutputContentItem::InputImage { image_url } => { - json!({"type":"image_url","image_url": {"url": image_url}}) - } - }) - .collect(); - json!(mapped) - } else { - json!(output.content) - }; - - messages.push(json!({ - "role": "tool", - "tool_call_id": call_id, - "content": content_value, - })); - } - ResponseItem::CustomToolCall { - id, - call_id: _, - name, - input, - status: _, - } => { - messages.push(json!({ - "role": "assistant", - "content": null, - "tool_calls": [{ - "id": id, - "type": "custom", - "custom": { - "name": name, - "input": input, - } - }] - })); - } - ResponseItem::CustomToolCallOutput { call_id, output } => { - messages.push(json!({ - "role": "tool", - "tool_call_id": call_id, - "content": output, - })); - } - ResponseItem::GhostSnapshot { .. } => { - // Ghost snapshots annotate history but are not sent to the model. - continue; - } - ResponseItem::Reasoning { .. } - | ResponseItem::WebSearchCall { .. } - | ResponseItem::Other - | ResponseItem::CompactionSummary { .. } => { - // Omit these items from the conversation history. - continue; - } - } - } - - let tools_json = create_tools_json_for_chat_completions_api(&prompt.tools)?; - let payload = json!({ - "model": model_family.slug, - "messages": messages, - "stream": true, - "tools": tools_json, - }); - - debug!( - "POST to {}: {}", - provider.get_full_url(&None), - payload.to_string() - ); - - let mut attempt = 0; - let max_retries = provider.request_max_retries(); - loop { - attempt += 1; - - let mut req_builder = provider.create_request_builder(client, &None).await?; - - // Include subagent header only for subagent sessions. - if let SessionSource::SubAgent(sub) = session_source.clone() { - let subagent = if let SubAgentSource::Other(label) = sub { - label - } else { - serde_json::to_value(&sub) - .ok() - .and_then(|v| v.as_str().map(std::string::ToString::to_string)) - .unwrap_or_else(|| "other".to_string()) - }; - req_builder = req_builder.header("x-openai-subagent", subagent); - } - - let res = otel_event_manager - .log_request(attempt, || { - req_builder - .header(reqwest::header::ACCEPT, "text/event-stream") - .json(&payload) - .send() - }) - .await; - - match res { - Ok(resp) if resp.status().is_success() => { - let (tx_event, rx_event) = mpsc::channel::>(1600); - let stream = resp.bytes_stream().map_err(|e| { - CodexErr::ResponseStreamFailed(ResponseStreamFailed { - source: e, - request_id: None, - }) - }); - tokio::spawn(process_chat_sse( - stream, - tx_event, - provider.stream_idle_timeout(), - otel_event_manager.clone(), - )); - return Ok(ResponseStream { rx_event }); - } - Ok(res) => { - let status = res.status(); - if !(status == StatusCode::TOO_MANY_REQUESTS || status.is_server_error()) { - let body = (res.text().await).unwrap_or_default(); - return Err(CodexErr::UnexpectedStatus(UnexpectedResponseError { - status, - body, - request_id: None, - })); - } - - if attempt > max_retries { - return Err(CodexErr::RetryLimit(RetryLimitReachedError { - status, - request_id: None, - })); - } - - let retry_after_secs = res - .headers() - .get(reqwest::header::RETRY_AFTER) - .and_then(|v| v.to_str().ok()) - .and_then(|s| s.parse::().ok()); - - let delay = retry_after_secs - .map(|s| Duration::from_millis(s * 1_000)) - .unwrap_or_else(|| backoff(attempt)); - tokio::time::sleep(delay).await; - } - Err(e) => { - if attempt > max_retries { - return Err(CodexErr::ConnectionFailed(ConnectionFailedError { - source: e, - })); - } - let delay = backoff(attempt); - tokio::time::sleep(delay).await; - } - } - } -} - -async fn append_assistant_text( - tx_event: &mpsc::Sender>, - assistant_item: &mut Option, - text: String, -) { - if assistant_item.is_none() { - let item = ResponseItem::Message { - id: None, - role: "assistant".to_string(), - content: vec![], - }; - *assistant_item = Some(item.clone()); - let _ = tx_event - .send(Ok(ResponseEvent::OutputItemAdded(item))) - .await; - } - - if let Some(ResponseItem::Message { content, .. }) = assistant_item { - content.push(ContentItem::OutputText { text: text.clone() }); - let _ = tx_event - .send(Ok(ResponseEvent::OutputTextDelta(text.clone()))) - .await; - } -} - -async fn append_reasoning_text( - tx_event: &mpsc::Sender>, - reasoning_item: &mut Option, - text: String, -) { - if reasoning_item.is_none() { - let item = ResponseItem::Reasoning { - id: String::new(), - summary: Vec::new(), - content: Some(vec![]), - encrypted_content: None, - }; - *reasoning_item = Some(item.clone()); - let _ = tx_event - .send(Ok(ResponseEvent::OutputItemAdded(item))) - .await; - } - - if let Some(ResponseItem::Reasoning { - content: Some(content), - .. - }) = reasoning_item - { - let content_index = content.len() as i64; - content.push(ReasoningItemContent::ReasoningText { text: text.clone() }); - - let _ = tx_event - .send(Ok(ResponseEvent::ReasoningContentDelta { - delta: text.clone(), - content_index, - })) - .await; - } -} -/// Lightweight SSE processor for the Chat Completions streaming format. The -/// output is mapped onto Codex's internal [`ResponseEvent`] so that the rest -/// of the pipeline can stay agnostic of the underlying wire format. -async fn process_chat_sse( - stream: S, - tx_event: mpsc::Sender>, - idle_timeout: Duration, - otel_event_manager: OtelEventManager, -) where - S: Stream> + Unpin, -{ - let mut stream = stream.eventsource(); - - // State to accumulate a function call across streaming chunks. - // OpenAI may split the `arguments` string over multiple `delta` events - // until the chunk whose `finish_reason` is `tool_calls` is emitted. We - // keep collecting the pieces here and forward a single - // `ResponseItem::FunctionCall` once the call is complete. - #[derive(Default)] - struct FunctionCallState { - name: Option, - arguments: String, - call_id: Option, - active: bool, - } - - let mut fn_call_state = FunctionCallState::default(); - let mut assistant_item: Option = None; - let mut reasoning_item: Option = None; - - loop { - let start = std::time::Instant::now(); - let response = timeout(idle_timeout, stream.next()).await; - let duration = start.elapsed(); - otel_event_manager.log_sse_event(&response, duration); - - let sse = match response { - Ok(Some(Ok(ev))) => ev, - Ok(Some(Err(e))) => { - let _ = tx_event - .send(Err(CodexErr::Stream(e.to_string(), None))) - .await; - return; - } - Ok(None) => { - // Stream closed gracefully – emit Completed with dummy id. - let _ = tx_event - .send(Ok(ResponseEvent::Completed { - response_id: String::new(), - token_usage: None, - })) - .await; - return; - } - Err(_) => { - let _ = tx_event - .send(Err(CodexErr::Stream( - "idle timeout waiting for SSE".into(), - None, - ))) - .await; - return; - } - }; - - // OpenAI Chat streaming sends a literal string "[DONE]" when finished. - if sse.data.trim() == "[DONE]" { - // Emit any finalized items before closing so downstream consumers receive - // terminal events for both assistant content and raw reasoning. - if let Some(item) = assistant_item { - let _ = tx_event.send(Ok(ResponseEvent::OutputItemDone(item))).await; - } - - if let Some(item) = reasoning_item { - let _ = tx_event.send(Ok(ResponseEvent::OutputItemDone(item))).await; - } - - let _ = tx_event - .send(Ok(ResponseEvent::Completed { - response_id: String::new(), - token_usage: None, - })) - .await; - return; - } - - // Parse JSON chunk - let chunk: serde_json::Value = match serde_json::from_str(&sse.data) { - Ok(v) => v, - Err(_) => continue, - }; - trace!("chat_completions received SSE chunk: {chunk:?}"); - - let choice_opt = chunk.get("choices").and_then(|c| c.get(0)); - - if let Some(choice) = choice_opt { - // Handle assistant content tokens as streaming deltas. - if let Some(content) = choice - .get("delta") - .and_then(|d| d.get("content")) - .and_then(|c| c.as_str()) - && !content.is_empty() - { - append_assistant_text(&tx_event, &mut assistant_item, content.to_string()).await; - } - - // Forward any reasoning/thinking deltas if present. - // Some providers stream `reasoning` as a plain string while others - // nest the text under an object (e.g. `{ "reasoning": { "text": "…" } }`). - if let Some(reasoning_val) = choice.get("delta").and_then(|d| d.get("reasoning")) { - let mut maybe_text = reasoning_val - .as_str() - .map(str::to_string) - .filter(|s| !s.is_empty()); - - if maybe_text.is_none() && reasoning_val.is_object() { - if let Some(s) = reasoning_val - .get("text") - .and_then(|t| t.as_str()) - .filter(|s| !s.is_empty()) - { - maybe_text = Some(s.to_string()); - } else if let Some(s) = reasoning_val - .get("content") - .and_then(|t| t.as_str()) - .filter(|s| !s.is_empty()) - { - maybe_text = Some(s.to_string()); - } - } - - if let Some(reasoning) = maybe_text { - // Accumulate so we can emit a terminal Reasoning item at the end. - append_reasoning_text(&tx_event, &mut reasoning_item, reasoning).await; - } - } - - // Some providers only include reasoning on the final message object. - if let Some(message_reasoning) = choice.get("message").and_then(|m| m.get("reasoning")) - { - // Accept either a plain string or an object with { text | content } - if let Some(s) = message_reasoning.as_str() { - if !s.is_empty() { - append_reasoning_text(&tx_event, &mut reasoning_item, s.to_string()).await; - } - } else if let Some(obj) = message_reasoning.as_object() - && let Some(s) = obj - .get("text") - .and_then(|v| v.as_str()) - .or_else(|| obj.get("content").and_then(|v| v.as_str())) - && !s.is_empty() - { - append_reasoning_text(&tx_event, &mut reasoning_item, s.to_string()).await; - } - } - - // Handle streaming function / tool calls. - if let Some(tool_calls) = choice - .get("delta") - .and_then(|d| d.get("tool_calls")) - .and_then(|tc| tc.as_array()) - && let Some(tool_call) = tool_calls.first() - { - // Mark that we have an active function call in progress. - fn_call_state.active = true; - - // Extract call_id if present. - if let Some(id) = tool_call.get("id").and_then(|v| v.as_str()) { - fn_call_state.call_id.get_or_insert_with(|| id.to_string()); - } - - // Extract function details if present. - if let Some(function) = tool_call.get("function") { - if let Some(name) = function.get("name").and_then(|n| n.as_str()) { - fn_call_state.name.get_or_insert_with(|| name.to_string()); - } - - if let Some(args_fragment) = function.get("arguments").and_then(|a| a.as_str()) - { - fn_call_state.arguments.push_str(args_fragment); - } - } - } - - // Emit end-of-turn when finish_reason signals completion. - if let Some(finish_reason) = choice.get("finish_reason").and_then(|v| v.as_str()) - && !finish_reason.is_empty() - { - match finish_reason { - "tool_calls" if fn_call_state.active => { - // First, flush the terminal raw reasoning so UIs can finalize - // the reasoning stream before any exec/tool events begin. - if let Some(item) = reasoning_item.take() { - let _ = tx_event.send(Ok(ResponseEvent::OutputItemDone(item))).await; - } - - // Then emit the FunctionCall response item. - let item = ResponseItem::FunctionCall { - id: None, - name: fn_call_state.name.clone().unwrap_or_else(|| "".to_string()), - arguments: fn_call_state.arguments.clone(), - call_id: fn_call_state.call_id.clone().unwrap_or_else(String::new), - }; - - let _ = tx_event.send(Ok(ResponseEvent::OutputItemDone(item))).await; - } - "stop" => { - // Regular turn without tool-call. Emit the final assistant message - // as a single OutputItemDone so non-delta consumers see the result. - if let Some(item) = assistant_item.take() { - let _ = tx_event.send(Ok(ResponseEvent::OutputItemDone(item))).await; - } - // Also emit a terminal Reasoning item so UIs can finalize raw reasoning. - if let Some(item) = reasoning_item.take() { - let _ = tx_event.send(Ok(ResponseEvent::OutputItemDone(item))).await; - } - } - _ => {} - } - - // Emit Completed regardless of reason so the agent can advance. - let _ = tx_event - .send(Ok(ResponseEvent::Completed { - response_id: String::new(), - token_usage: None, - })) - .await; - - // Prepare for potential next turn (should not happen in same stream). - // fn_call_state = FunctionCallState::default(); - - return; // End processing for this SSE stream. - } - } - } -} - -/// Optional client-side aggregation helper -/// -/// Stream adapter that merges the incremental `OutputItemDone` chunks coming from -/// [`process_chat_sse`] into a *running* assistant message, **suppressing the -/// per-token deltas**. The stream stays silent while the model is thinking -/// and only emits two events per turn: -/// -/// 1. `ResponseEvent::OutputItemDone` with the *complete* assistant message -/// (fully concatenated). -/// 2. The original `ResponseEvent::Completed` right after it. -/// -/// This mirrors the behaviour the TypeScript CLI exposes to its higher layers. -/// -/// The adapter is intentionally *lossless*: callers who do **not** opt in via -/// [`AggregateStreamExt::aggregate()`] keep receiving the original unmodified -/// events. -#[derive(Copy, Clone, Eq, PartialEq)] -enum AggregateMode { - AggregatedOnly, - Streaming, -} -pub(crate) struct AggregatedChatStream { - inner: S, - cumulative: String, - cumulative_reasoning: String, - pending: std::collections::VecDeque, - mode: AggregateMode, -} - -impl Stream for AggregatedChatStream -where - S: Stream> + Unpin, -{ - type Item = Result; - - fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let this = self.get_mut(); - - // First, flush any buffered events from the previous call. - if let Some(ev) = this.pending.pop_front() { - return Poll::Ready(Some(Ok(ev))); - } - - loop { - match Pin::new(&mut this.inner).poll_next(cx) { - Poll::Pending => return Poll::Pending, - Poll::Ready(None) => return Poll::Ready(None), - Poll::Ready(Some(Err(e))) => return Poll::Ready(Some(Err(e))), - Poll::Ready(Some(Ok(ResponseEvent::OutputItemDone(item)))) => { - // If this is an incremental assistant message chunk, accumulate but - // do NOT emit yet. Forward any other item (e.g. FunctionCall) right - // away so downstream consumers see it. - - let is_assistant_message = matches!( - &item, - codex_protocol::models::ResponseItem::Message { role, .. } if role == "assistant" - ); - - if is_assistant_message { - match this.mode { - AggregateMode::AggregatedOnly => { - // Only use the final assistant message if we have not - // seen any deltas; otherwise, deltas already built the - // cumulative text and this would duplicate it. - if this.cumulative.is_empty() - && let codex_protocol::models::ResponseItem::Message { - content, - .. - } = &item - && let Some(text) = content.iter().find_map(|c| match c { - codex_protocol::models::ContentItem::OutputText { - text, - } => Some(text), - _ => None, - }) - { - this.cumulative.push_str(text); - } - // Swallow assistant message here; emit on Completed. - continue; - } - AggregateMode::Streaming => { - // In streaming mode, if we have not seen any deltas, forward - // the final assistant message directly. If deltas were seen, - // suppress the final message to avoid duplication. - if this.cumulative.is_empty() { - return Poll::Ready(Some(Ok(ResponseEvent::OutputItemDone( - item, - )))); - } else { - continue; - } - } - } - } - - // Not an assistant message – forward immediately. - return Poll::Ready(Some(Ok(ResponseEvent::OutputItemDone(item)))); - } - Poll::Ready(Some(Ok(ResponseEvent::RateLimits(snapshot)))) => { - return Poll::Ready(Some(Ok(ResponseEvent::RateLimits(snapshot)))); - } - Poll::Ready(Some(Ok(ResponseEvent::Completed { - response_id, - token_usage, - }))) => { - // Build any aggregated items in the correct order: Reasoning first, then Message. - let mut emitted_any = false; - - if !this.cumulative_reasoning.is_empty() - && matches!(this.mode, AggregateMode::AggregatedOnly) - { - let aggregated_reasoning = - codex_protocol::models::ResponseItem::Reasoning { - id: String::new(), - summary: Vec::new(), - content: Some(vec![ - codex_protocol::models::ReasoningItemContent::ReasoningText { - text: std::mem::take(&mut this.cumulative_reasoning), - }, - ]), - encrypted_content: None, - }; - this.pending - .push_back(ResponseEvent::OutputItemDone(aggregated_reasoning)); - emitted_any = true; - } - - // Always emit the final aggregated assistant message when any - // content deltas have been observed. In AggregatedOnly mode this - // is the sole assistant output; in Streaming mode this finalizes - // the streamed deltas into a terminal OutputItemDone so callers - // can persist/render the message once per turn. - if !this.cumulative.is_empty() { - let aggregated_message = codex_protocol::models::ResponseItem::Message { - id: None, - role: "assistant".to_string(), - content: vec![codex_protocol::models::ContentItem::OutputText { - text: std::mem::take(&mut this.cumulative), - }], - }; - this.pending - .push_back(ResponseEvent::OutputItemDone(aggregated_message)); - emitted_any = true; - } - - // Always emit Completed last when anything was aggregated. - if emitted_any { - this.pending.push_back(ResponseEvent::Completed { - response_id: response_id.clone(), - token_usage: token_usage.clone(), - }); - // Return the first pending event now. - if let Some(ev) = this.pending.pop_front() { - return Poll::Ready(Some(Ok(ev))); - } - } - - // Nothing aggregated – forward Completed directly. - return Poll::Ready(Some(Ok(ResponseEvent::Completed { - response_id, - token_usage, - }))); - } - Poll::Ready(Some(Ok(ResponseEvent::Created))) => { - // These events are exclusive to the Responses API and - // will never appear in a Chat Completions stream. - continue; - } - Poll::Ready(Some(Ok(ResponseEvent::OutputTextDelta(delta)))) => { - // Always accumulate deltas so we can emit a final OutputItemDone at Completed. - this.cumulative.push_str(&delta); - if matches!(this.mode, AggregateMode::Streaming) { - // In streaming mode, also forward the delta immediately. - return Poll::Ready(Some(Ok(ResponseEvent::OutputTextDelta(delta)))); - } else { - continue; - } - } - Poll::Ready(Some(Ok(ResponseEvent::ReasoningContentDelta { - delta, - content_index, - }))) => { - // Always accumulate reasoning deltas so we can emit a final Reasoning item at Completed. - this.cumulative_reasoning.push_str(&delta); - if matches!(this.mode, AggregateMode::Streaming) { - // In streaming mode, also forward the delta immediately. - return Poll::Ready(Some(Ok(ResponseEvent::ReasoningContentDelta { - delta, - content_index, - }))); - } else { - continue; - } - } - Poll::Ready(Some(Ok(ResponseEvent::ReasoningSummaryDelta { .. }))) => { - continue; - } - Poll::Ready(Some(Ok(ResponseEvent::ReasoningSummaryPartAdded { .. }))) => { - continue; - } - Poll::Ready(Some(Ok(ResponseEvent::OutputItemAdded(item)))) => { - return Poll::Ready(Some(Ok(ResponseEvent::OutputItemAdded(item)))); - } - } - } - } -} - -/// Extension trait that activates aggregation on any stream of [`ResponseEvent`]. -pub(crate) trait AggregateStreamExt: Stream> + Sized { - /// Returns a new stream that emits **only** the final assistant message - /// per turn instead of every incremental delta. The produced - /// `ResponseEvent` sequence for a typical text turn looks like: - /// - /// ```ignore - /// OutputItemDone() - /// Completed - /// ``` - /// - /// No other `OutputItemDone` events will be seen by the caller. - /// - /// Usage: - /// - /// ```ignore - /// let agg_stream = client.stream(&prompt).await?.aggregate(); - /// while let Some(event) = agg_stream.next().await { - /// // event now contains cumulative text - /// } - /// ``` - fn aggregate(self) -> AggregatedChatStream { - AggregatedChatStream::new(self, AggregateMode::AggregatedOnly) - } -} - -impl AggregateStreamExt for T where T: Stream> + Sized {} - -impl AggregatedChatStream { - fn new(inner: S, mode: AggregateMode) -> Self { - AggregatedChatStream { - inner, - cumulative: String::new(), - cumulative_reasoning: String::new(), - pending: std::collections::VecDeque::new(), - mode, - } - } - - pub(crate) fn streaming_mode(inner: S) -> Self { - Self::new(inner, AggregateMode::Streaming) - } -} diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index 68bd30f4e7..82839522c9 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -1,12 +1,22 @@ -use std::io::BufRead; -use std::path::Path; use std::sync::Arc; -use std::sync::OnceLock; -use std::time::Duration; -use bytes::Bytes; -use chrono::DateTime; -use chrono::Utc; +use crate::api_bridge::auth_provider_from_auth; +use crate::api_bridge::map_api_error; +use codex_api::AggregateStreamExt; +use codex_api::ChatClient as ApiChatClient; +use codex_api::CompactClient as ApiCompactClient; +use codex_api::CompactionInput as ApiCompactionInput; +use codex_api::Prompt as ApiPrompt; +use codex_api::RequestTelemetry; +use codex_api::ReqwestTransport; +use codex_api::ResponseStream as ApiResponseStream; +use codex_api::ResponsesClient as ApiResponsesClient; +use codex_api::ResponsesOptions as ApiResponsesOptions; +use codex_api::SseTelemetry; +use codex_api::TransportError; +use codex_api::common::Reasoning; +use codex_api::create_text_param_for_request; +use codex_api::error::ApiError; use codex_app_server_protocol::AuthMode; use codex_otel::otel_event_manager::OtelEventManager; use codex_protocol::ConversationId; @@ -14,90 +24,40 @@ use codex_protocol::config_types::ReasoningEffort as ReasoningEffortConfig; use codex_protocol::config_types::ReasoningSummary as ReasoningSummaryConfig; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::SessionSource; -use eventsource_stream::Eventsource; -use futures::prelude::*; -use regex_lite::Regex; +use eventsource_stream::Event; +use eventsource_stream::EventStreamError; +use futures::StreamExt; +use http::HeaderMap as ApiHeaderMap; +use http::HeaderValue; +use http::StatusCode as HttpStatusCode; use reqwest::StatusCode; -use reqwest::header::HeaderMap; -use serde::Deserialize; -use serde::Serialize; use serde_json::Value; +use std::time::Duration; use tokio::sync::mpsc; -use tokio::time::timeout; -use tokio_util::io::ReaderStream; -use tracing::debug; -use tracing::enabled; -use tracing::trace; use tracing::warn; use crate::AuthManager; -use crate::auth::CodexAuth; use crate::auth::RefreshTokenError; -use crate::chat_completions::AggregateStreamExt; -use crate::chat_completions::stream_chat_completions; use crate::client_common::Prompt; -use crate::client_common::Reasoning; use crate::client_common::ResponseEvent; use crate::client_common::ResponseStream; -use crate::client_common::ResponsesApiRequest; -use crate::client_common::create_text_param_for_request; use crate::config::Config; -use crate::default_client::CodexHttpClient; -use crate::default_client::create_client; +use crate::default_client::build_reqwest_client; use crate::error::CodexErr; -use crate::error::ConnectionFailedError; -use crate::error::ResponseStreamFailed; use crate::error::Result; -use crate::error::RetryLimitReachedError; -use crate::error::UnexpectedResponseError; -use crate::error::UsageLimitReachedError; use crate::flags::CODEX_RS_SSE_FIXTURE; use crate::model_family::ModelFamily; use crate::model_provider_info::ModelProviderInfo; use crate::model_provider_info::WireApi; use crate::openai_model_info::get_model_info; -use crate::protocol::CreditsSnapshot; -use crate::protocol::RateLimitSnapshot; -use crate::protocol::RateLimitWindow; -use crate::protocol::TokenUsage; -use crate::token_data::PlanType; +use crate::tools::spec::create_tools_json_for_chat_completions_api; use crate::tools::spec::create_tools_json_for_responses_api; -use crate::util::backoff; - -#[derive(Debug, Deserialize)] -struct ErrorResponse { - error: Error, -} - -#[derive(Debug, Deserialize)] -struct Error { - r#type: Option, - code: Option, - message: Option, - - // Optional fields available on "usage_limit_reached" and "usage_not_included" errors - plan_type: Option, - resets_at: Option, -} - -#[derive(Debug, Serialize)] -struct CompactHistoryRequest<'a> { - model: &'a str, - input: &'a [ResponseItem], - instructions: &'a str, -} - -#[derive(Debug, Deserialize)] -struct CompactHistoryResponse { - output: Vec, -} #[derive(Debug, Clone)] pub struct ModelClient { config: Arc, auth_manager: Option>, otel_event_manager: OtelEventManager, - client: CodexHttpClient, provider: ModelProviderInfo, conversation_id: ConversationId, effort: Option, @@ -117,13 +77,10 @@ impl ModelClient { conversation_id: ConversationId, session_source: SessionSource, ) -> Self { - let client = create_client(); - Self { config, auth_manager, otel_event_manager, - client, provider, conversation_id, effort, @@ -154,65 +111,102 @@ impl ModelClient { &self.provider } + /// Streams a single model turn using either the Responses or Chat + /// Completions wire API, depending on the configured provider. + /// + /// For Chat providers, the underlying stream is optionally aggregated + /// based on the `show_raw_agent_reasoning` flag in the config. pub async fn stream(&self, prompt: &Prompt) -> Result { match self.provider.wire_api { - WireApi::Responses => self.stream_responses(prompt).await, + WireApi::Responses => self.stream_responses_api(prompt).await, WireApi::Chat => { - // Create the raw streaming connection first. - let response_stream = stream_chat_completions( - prompt, - &self.config.model_family, - &self.client, - &self.provider, - &self.otel_event_manager, - &self.session_source, - ) - .await?; + let api_stream = self.stream_chat_completions(prompt).await?; - // Wrap it with the aggregation adapter so callers see *only* - // the final assistant message per turn (matching the - // behaviour of the Responses API). - let mut aggregated = if self.config.show_raw_agent_reasoning { - crate::chat_completions::AggregatedChatStream::streaming_mode(response_stream) + if self.config.show_raw_agent_reasoning { + Ok(map_response_stream( + api_stream.streaming_mode(), + self.otel_event_manager.clone(), + )) } else { - response_stream.aggregate() - }; - - // Bridge the aggregated stream back into a standard - // `ResponseStream` by forwarding events through a channel. - let (tx, rx) = mpsc::channel::>(16); - - tokio::spawn(async move { - use futures::StreamExt; - while let Some(ev) = aggregated.next().await { - // Exit early if receiver hung up. - if tx.send(ev).await.is_err() { - break; - } - } - }); - - Ok(ResponseStream { rx_event: rx }) + Ok(map_response_stream( + api_stream.aggregate(), + self.otel_event_manager.clone(), + )) + } } } } - /// Implementation for the OpenAI *Responses* experimental API. - async fn stream_responses(&self, prompt: &Prompt) -> Result { - if let Some(path) = &*CODEX_RS_SSE_FIXTURE { - // short circuit for tests - warn!(path, "Streaming from fixture"); - return stream_from_fixture( - path, - self.provider.clone(), - self.otel_event_manager.clone(), - ) - .await; + /// Streams a turn via the OpenAI Chat Completions API. + /// + /// This path is only used when the provider is configured with + /// `WireApi::Chat`; it does not support `output_schema` today. + async fn stream_chat_completions(&self, prompt: &Prompt) -> Result { + if prompt.output_schema.is_some() { + return Err(CodexErr::UnsupportedOperation( + "output_schema is not supported for Chat Completions API".to_string(), + )); } let auth_manager = self.auth_manager.clone(); + let instructions = prompt + .get_full_instructions(&self.config.model_family) + .into_owned(); + let tools_json = create_tools_json_for_chat_completions_api(&prompt.tools)?; + let api_prompt = build_api_prompt(prompt, instructions, tools_json); + let conversation_id = self.conversation_id.to_string(); + let session_source = self.session_source.clone(); - let full_instructions = prompt.get_full_instructions(&self.config.model_family); + let mut refreshed = false; + loop { + let auth = auth_manager.as_ref().and_then(|m| m.auth()); + let api_provider = self + .provider + .to_api_provider(auth.as_ref().map(|a| a.mode))?; + let api_auth = auth_provider_from_auth(auth.clone(), &self.provider).await?; + let transport = ReqwestTransport::new(build_reqwest_client()); + let (request_telemetry, sse_telemetry) = self.build_streaming_telemetry(); + let client = ApiChatClient::new(transport, api_provider, api_auth) + .with_telemetry(Some(request_telemetry), Some(sse_telemetry)); + + let stream_result = client + .stream_prompt( + &self.config.model, + &api_prompt, + Some(conversation_id.clone()), + Some(session_source.clone()), + ) + .await; + + match stream_result { + Ok(stream) => return Ok(stream), + Err(ApiError::Transport(TransportError::Http { status, .. })) + if status == StatusCode::UNAUTHORIZED => + { + handle_unauthorized(status, &mut refreshed, &auth_manager, &auth).await?; + continue; + } + Err(err) => return Err(map_api_error(err)), + } + } + } + + /// Streams a turn via the OpenAI Responses API. + /// + /// Handles SSE fixtures, reasoning summaries, verbosity, and the + /// `text` controls used for output schemas. + async fn stream_responses_api(&self, prompt: &Prompt) -> Result { + if let Some(path) = &*CODEX_RS_SSE_FIXTURE { + warn!(path, "Streaming from fixture"); + let stream = codex_api::stream_from_fixture(path, self.provider.stream_idle_timeout()) + .map_err(map_api_error)?; + return Ok(map_response_stream(stream, self.otel_event_manager.clone())); + } + + let auth_manager = self.auth_manager.clone(); + let instructions = prompt + .get_full_instructions(&self.config.model_family) + .into_owned(); let tools_json: Vec = create_tools_json_for_responses_api(&prompt.tools)?; let reasoning = if self.config.model_family.supports_reasoning_summaries { @@ -232,8 +226,6 @@ impl ModelClient { vec![] }; - let input_with_instructions = prompt.get_formatted_input(); - let verbosity = if self.config.model_family.support_verbosity { self.config .model_verbosity @@ -248,241 +240,49 @@ impl ModelClient { None }; - // Only include `text.verbosity` for GPT-5 family models let text = create_text_param_for_request(verbosity, &prompt.output_schema); + let api_prompt = build_api_prompt(prompt, instructions.clone(), tools_json); + let conversation_id = self.conversation_id.to_string(); + let session_source = self.session_source.clone(); - // In general, we want to explicitly send `store: false` when using the Responses API, - // but in practice, the Azure Responses API rejects `store: false`: - // - // - If store = false and id is sent an error is thrown that ID is not found - // - If store = false and id is not sent an error is thrown that ID is required - // - // For Azure, we send `store: true` and preserve reasoning item IDs. - let azure_workaround = self.provider.is_azure_responses_endpoint(); + let mut refreshed = false; + loop { + let auth = auth_manager.as_ref().and_then(|m| m.auth()); + let api_provider = self + .provider + .to_api_provider(auth.as_ref().map(|a| a.mode))?; + let api_auth = auth_provider_from_auth(auth.clone(), &self.provider).await?; + let transport = ReqwestTransport::new(build_reqwest_client()); + let (request_telemetry, sse_telemetry) = self.build_streaming_telemetry(); + let client = ApiResponsesClient::new(transport, api_provider, api_auth) + .with_telemetry(Some(request_telemetry), Some(sse_telemetry)); - let payload = ResponsesApiRequest { - model: &self.config.model, - instructions: &full_instructions, - input: &input_with_instructions, - tools: &tools_json, - tool_choice: "auto", - parallel_tool_calls: prompt.parallel_tool_calls, - reasoning, - store: azure_workaround, - stream: true, - include, - prompt_cache_key: Some(self.conversation_id.to_string()), - text, - }; - - let mut payload_json = serde_json::to_value(&payload)?; - if azure_workaround { - attach_item_ids(&mut payload_json, &input_with_instructions); - } - - let max_attempts = self.provider.request_max_retries(); - for attempt in 0..=max_attempts { - match self - .attempt_stream_responses(attempt, &payload_json, &auth_manager) - .await - { - Ok(stream) => { - return Ok(stream); - } - Err(StreamAttemptError::Fatal(e)) => { - return Err(e); - } - Err(retryable_attempt_error) => { - if attempt == max_attempts { - return Err(retryable_attempt_error.into_error()); - } - - tokio::time::sleep(retryable_attempt_error.delay(attempt)).await; - } - } - } - - unreachable!("stream_responses_attempt should always return"); - } - - /// Single attempt to start a streaming Responses API call. - async fn attempt_stream_responses( - &self, - attempt: u64, - payload_json: &Value, - auth_manager: &Option>, - ) -> std::result::Result { - // Always fetch the latest auth in case a prior attempt refreshed the token. - let auth = auth_manager.as_ref().and_then(|m| m.auth()); - - trace!( - "POST to {}: {}", - self.provider.get_full_url(&auth), - payload_json.to_string() - ); - - let mut req_builder = self - .provider - .create_request_builder(&self.client, &auth) - .await - .map_err(StreamAttemptError::Fatal)?; - - // Include subagent header only for subagent sessions. - if let SessionSource::SubAgent(sub) = &self.session_source { - let subagent = if let crate::protocol::SubAgentSource::Other(label) = sub { - label.clone() - } else { - serde_json::to_value(sub) - .ok() - .and_then(|v| v.as_str().map(std::string::ToString::to_string)) - .unwrap_or_else(|| "other".to_string()) + let options = ApiResponsesOptions { + reasoning: reasoning.clone(), + include: include.clone(), + prompt_cache_key: Some(conversation_id.clone()), + text: text.clone(), + store_override: None, + conversation_id: Some(conversation_id.clone()), + session_source: Some(session_source.clone()), }; - req_builder = req_builder.header("x-openai-subagent", subagent); - } - req_builder = req_builder - // Send session_id for compatibility. - .header("conversation_id", self.conversation_id.to_string()) - .header("session_id", self.conversation_id.to_string()) - .header(reqwest::header::ACCEPT, "text/event-stream") - .json(payload_json); + let stream_result = client + .stream_prompt(&self.config.model, &api_prompt, options) + .await; - if let Some(auth) = auth.as_ref() - && auth.mode == AuthMode::ChatGPT - && let Some(account_id) = auth.get_account_id() - { - req_builder = req_builder.header("chatgpt-account-id", account_id); - } - - let res = self - .otel_event_manager - .log_request(attempt, || req_builder.send()) - .await; - - let mut request_id = None; - if let Ok(resp) = &res { - request_id = resp - .headers() - .get("cf-ray") - .map(|v| v.to_str().unwrap_or_default().to_string()); - } - - match res { - Ok(resp) if resp.status().is_success() => { - let (tx_event, rx_event) = mpsc::channel::>(1600); - - if let Some(snapshot) = parse_rate_limit_snapshot(resp.headers()) - && tx_event - .send(Ok(ResponseEvent::RateLimits(snapshot))) - .await - .is_err() - { - debug!("receiver dropped rate limit snapshot event"); + match stream_result { + Ok(stream) => { + return Ok(map_response_stream(stream, self.otel_event_manager.clone())); } - - // spawn task to process SSE - let stream = resp.bytes_stream().map_err(move |e| { - CodexErr::ResponseStreamFailed(ResponseStreamFailed { - source: e, - request_id: request_id.clone(), - }) - }); - tokio::spawn(process_sse( - stream, - tx_event, - self.provider.stream_idle_timeout(), - self.otel_event_manager.clone(), - )); - - Ok(ResponseStream { rx_event }) + Err(ApiError::Transport(TransportError::Http { status, .. })) + if status == StatusCode::UNAUTHORIZED => + { + handle_unauthorized(status, &mut refreshed, &auth_manager, &auth).await?; + continue; + } + Err(err) => return Err(map_api_error(err)), } - Ok(res) => { - let status = res.status(); - - // Pull out Retry‑After header if present. - let retry_after_secs = res - .headers() - .get(reqwest::header::RETRY_AFTER) - .and_then(|v| v.to_str().ok()) - .and_then(|s| s.parse::().ok()); - let retry_after = retry_after_secs.map(|s| Duration::from_millis(s * 1_000)); - - if status == StatusCode::UNAUTHORIZED - && let Some(manager) = auth_manager.as_ref() - && let Some(auth) = auth.as_ref() - && auth.mode == AuthMode::ChatGPT - && let Err(err) = manager.refresh_token().await - { - let stream_error = match err { - RefreshTokenError::Permanent(failed) => { - StreamAttemptError::Fatal(CodexErr::RefreshTokenFailed(failed)) - } - RefreshTokenError::Transient(other) => { - StreamAttemptError::RetryableTransportError(CodexErr::Io(other)) - } - }; - return Err(stream_error); - } - - // The OpenAI Responses endpoint returns structured JSON bodies even for 4xx/5xx - // errors. When we bubble early with only the HTTP status the caller sees an opaque - // "unexpected status 400 Bad Request" which makes debugging nearly impossible. - // Instead, read (and include) the response text so higher layers and users see the - // exact error message (e.g. "Unknown parameter: 'input[0].metadata'"). The body is - // small and this branch only runs on error paths so the extra allocation is - // negligible. - if !(status == StatusCode::TOO_MANY_REQUESTS - || status == StatusCode::UNAUTHORIZED - || status.is_server_error()) - { - // Surface the error body to callers. Use `unwrap_or_default` per Clippy. - let body = res.text().await.unwrap_or_default(); - return Err(StreamAttemptError::Fatal(CodexErr::UnexpectedStatus( - UnexpectedResponseError { - status, - body, - request_id: None, - }, - ))); - } - - if status == StatusCode::TOO_MANY_REQUESTS { - let rate_limit_snapshot = parse_rate_limit_snapshot(res.headers()); - let body = res.json::().await.ok(); - if let Some(ErrorResponse { error }) = body { - if error.r#type.as_deref() == Some("usage_limit_reached") { - // Prefer the plan_type provided in the error message if present - // because it's more up to date than the one encoded in the auth - // token. - let plan_type = error - .plan_type - .or_else(|| auth.as_ref().and_then(CodexAuth::get_plan_type)); - let resets_at = error - .resets_at - .and_then(|seconds| DateTime::::from_timestamp(seconds, 0)); - let codex_err = CodexErr::UsageLimitReached(UsageLimitReachedError { - plan_type, - resets_at, - rate_limits: rate_limit_snapshot, - }); - return Err(StreamAttemptError::Fatal(codex_err)); - } else if error.r#type.as_deref() == Some("usage_not_included") { - return Err(StreamAttemptError::Fatal(CodexErr::UsageNotIncluded)); - } else if is_quota_exceeded_error(&error) { - return Err(StreamAttemptError::Fatal(CodexErr::QuotaExceeded)); - } - } - } - - Err(StreamAttemptError::RetryableHttpError { - status, - retry_after, - request_id, - }) - } - Err(e) => Err(StreamAttemptError::RetryableTransportError( - CodexErr::ConnectionFailed(ConnectionFailedError { source: e }), - )), } } @@ -522,16 +322,35 @@ impl ModelClient { self.auth_manager.clone() } + /// Compacts the current conversation history using the Compact endpoint. + /// + /// This is a unary call (no streaming) that returns a new list of + /// `ResponseItem`s representing the compacted transcript. pub async fn compact_conversation_history(&self, prompt: &Prompt) -> Result> { if prompt.input.is_empty() { return Ok(Vec::new()); } let auth_manager = self.auth_manager.clone(); let auth = auth_manager.as_ref().and_then(|m| m.auth()); - let mut req_builder = self + let api_provider = self .provider - .create_compact_request_builder(&self.client, &auth) - .await?; + .to_api_provider(auth.as_ref().map(|a| a.mode))?; + let api_auth = auth_provider_from_auth(auth.clone(), &self.provider).await?; + let transport = ReqwestTransport::new(build_reqwest_client()); + let request_telemetry = self.build_request_telemetry(); + let client = ApiCompactClient::new(transport, api_provider, api_auth) + .with_telemetry(Some(request_telemetry)); + + let instructions = prompt + .get_full_instructions(&self.config.model_family) + .into_owned(); + let payload = ApiCompactionInput { + model: &self.config.model, + input: &prompt.input, + instructions: &instructions, + }; + + let mut extra_headers = ApiHeaderMap::new(); if let SessionSource::SubAgent(sub) = &self.session_source { let subagent = if let crate::protocol::SubAgentSource::Other(label) = sub { label.clone() @@ -541,1109 +360,183 @@ impl ModelClient { .and_then(|v| v.as_str().map(std::string::ToString::to_string)) .unwrap_or_else(|| "other".to_string()) }; - req_builder = req_builder.header("x-openai-subagent", subagent); - } - if let Some(auth) = auth.as_ref() - && auth.mode == AuthMode::ChatGPT - && let Some(account_id) = auth.get_account_id() - { - req_builder = req_builder.header("chatgpt-account-id", account_id); - } - let payload = CompactHistoryRequest { - model: &self.config.model, - input: &prompt.input, - instructions: &prompt.get_full_instructions(&self.config.model_family), - }; - - if enabled!(tracing::Level::TRACE) { - trace!( - "POST to {}: {}", - self.provider - .get_compact_url(&auth) - .unwrap_or("".to_string()), - serde_json::to_value(&payload).unwrap_or_default() - ); + if let Ok(val) = HeaderValue::from_str(&subagent) { + extra_headers.insert("x-openai-subagent", val); + } } - let response = req_builder - .json(&payload) - .send() + client + .compact_input(&payload, extra_headers) .await - .map_err(|source| CodexErr::ConnectionFailed(ConnectionFailedError { source }))?; - let status = response.status(); - let body = response - .text() - .await - .map_err(|source| CodexErr::ConnectionFailed(ConnectionFailedError { source }))?; - if !status.is_success() { - return Err(CodexErr::UnexpectedStatus(UnexpectedResponseError { - status, - body, - request_id: None, - })); - } - let CompactHistoryResponse { output } = serde_json::from_str(&body)?; - Ok(output) + .map_err(map_api_error) } } -enum StreamAttemptError { - RetryableHttpError { - status: StatusCode, - retry_after: Option, - request_id: Option, - }, - RetryableTransportError(CodexErr), - Fatal(CodexErr), -} - -impl StreamAttemptError { - /// attempt is 0-based. - fn delay(&self, attempt: u64) -> Duration { - // backoff() uses 1-based attempts. - let backoff_attempt = attempt + 1; - match self { - Self::RetryableHttpError { retry_after, .. } => { - retry_after.unwrap_or_else(|| backoff(backoff_attempt)) - } - Self::RetryableTransportError { .. } => backoff(backoff_attempt), - Self::Fatal(_) => { - // Should not be called on Fatal errors. - Duration::from_secs(0) - } - } +impl ModelClient { + /// Builds request and SSE telemetry for streaming API calls (Chat/Responses). + fn build_streaming_telemetry(&self) -> (Arc, Arc) { + let telemetry = Arc::new(ApiTelemetry::new(self.otel_event_manager.clone())); + let request_telemetry: Arc = telemetry.clone(); + let sse_telemetry: Arc = telemetry; + (request_telemetry, sse_telemetry) } - fn into_error(self) -> CodexErr { - match self { - Self::RetryableHttpError { - status, request_id, .. - } => { - if status == StatusCode::INTERNAL_SERVER_ERROR { - CodexErr::InternalServerError - } else { - CodexErr::RetryLimit(RetryLimitReachedError { status, request_id }) - } - } - Self::RetryableTransportError(error) => error, - Self::Fatal(error) => error, - } + /// Builds request telemetry for unary API calls (e.g., Compact endpoint). + fn build_request_telemetry(&self) -> Arc { + let telemetry = Arc::new(ApiTelemetry::new(self.otel_event_manager.clone())); + let request_telemetry: Arc = telemetry; + request_telemetry } } -#[derive(Debug, Deserialize, Serialize)] -struct SseEvent { - #[serde(rename = "type")] - kind: String, - response: Option, - item: Option, - delta: Option, - summary_index: Option, - content_index: Option, -} - -#[derive(Debug, Deserialize)] -struct ResponseCompleted { - id: String, - usage: Option, -} - -#[derive(Debug, Deserialize)] -struct ResponseCompletedUsage { - input_tokens: i64, - input_tokens_details: Option, - output_tokens: i64, - output_tokens_details: Option, - total_tokens: i64, -} - -impl From for TokenUsage { - fn from(val: ResponseCompletedUsage) -> Self { - TokenUsage { - input_tokens: val.input_tokens, - cached_input_tokens: val - .input_tokens_details - .map(|d| d.cached_tokens) - .unwrap_or(0), - output_tokens: val.output_tokens, - reasoning_output_tokens: val - .output_tokens_details - .map(|d| d.reasoning_tokens) - .unwrap_or(0), - total_tokens: val.total_tokens, - } +/// Adapts the core `Prompt` type into the `codex-api` payload shape. +fn build_api_prompt(prompt: &Prompt, instructions: String, tools_json: Vec) -> ApiPrompt { + ApiPrompt { + instructions, + input: prompt.get_formatted_input(), + tools: tools_json, + parallel_tool_calls: prompt.parallel_tool_calls, + output_schema: prompt.output_schema.clone(), } } -#[derive(Debug, Deserialize)] -struct ResponseCompletedInputTokensDetails { - cached_tokens: i64, -} - -#[derive(Debug, Deserialize)] -struct ResponseCompletedOutputTokensDetails { - reasoning_tokens: i64, -} - -fn attach_item_ids(payload_json: &mut Value, original_items: &[ResponseItem]) { - let Some(input_value) = payload_json.get_mut("input") else { - return; - }; - let serde_json::Value::Array(items) = input_value else { - return; - }; - - for (value, item) in items.iter_mut().zip(original_items.iter()) { - if let ResponseItem::Reasoning { id, .. } - | ResponseItem::Message { id: Some(id), .. } - | ResponseItem::WebSearchCall { id: Some(id), .. } - | ResponseItem::FunctionCall { id: Some(id), .. } - | ResponseItem::LocalShellCall { id: Some(id), .. } - | ResponseItem::CustomToolCall { id: Some(id), .. } = item - { - if id.is_empty() { - continue; - } - - if let Some(obj) = value.as_object_mut() { - obj.insert("id".to_string(), Value::String(id.clone())); - } - } - } -} - -fn parse_rate_limit_snapshot(headers: &HeaderMap) -> Option { - let primary = parse_rate_limit_window( - headers, - "x-codex-primary-used-percent", - "x-codex-primary-window-minutes", - "x-codex-primary-reset-at", - ); - - let secondary = parse_rate_limit_window( - headers, - "x-codex-secondary-used-percent", - "x-codex-secondary-window-minutes", - "x-codex-secondary-reset-at", - ); - - let credits = parse_credits_snapshot(headers); - - Some(RateLimitSnapshot { - primary, - secondary, - credits, - }) -} - -fn parse_rate_limit_window( - headers: &HeaderMap, - used_percent_header: &str, - window_minutes_header: &str, - resets_at_header: &str, -) -> Option { - let used_percent: Option = parse_header_f64(headers, used_percent_header); - - used_percent.and_then(|used_percent| { - let window_minutes = parse_header_i64(headers, window_minutes_header); - let resets_at = parse_header_i64(headers, resets_at_header); - - let has_data = used_percent != 0.0 - || window_minutes.is_some_and(|minutes| minutes != 0) - || resets_at.is_some(); - - has_data.then_some(RateLimitWindow { - used_percent, - window_minutes, - resets_at, - }) - }) -} - -fn parse_credits_snapshot(headers: &HeaderMap) -> Option { - let has_credits = parse_header_bool(headers, "x-codex-credits-has-credits")?; - let unlimited = parse_header_bool(headers, "x-codex-credits-unlimited")?; - let balance = parse_header_str(headers, "x-codex-credits-balance") - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(std::string::ToString::to_string); - Some(CreditsSnapshot { - has_credits, - unlimited, - balance, - }) -} - -fn parse_header_f64(headers: &HeaderMap, name: &str) -> Option { - parse_header_str(headers, name)? - .parse::() - .ok() - .filter(|v| v.is_finite()) -} - -fn parse_header_i64(headers: &HeaderMap, name: &str) -> Option { - parse_header_str(headers, name)?.parse::().ok() -} - -fn parse_header_bool(headers: &HeaderMap, name: &str) -> Option { - let raw = parse_header_str(headers, name)?; - if raw.eq_ignore_ascii_case("true") || raw == "1" { - Some(true) - } else if raw.eq_ignore_ascii_case("false") || raw == "0" { - Some(false) - } else { - None - } -} - -fn parse_header_str<'a>(headers: &'a HeaderMap, name: &str) -> Option<&'a str> { - headers.get(name)?.to_str().ok() -} - -async fn process_sse( - stream: S, - tx_event: mpsc::Sender>, - idle_timeout: Duration, - otel_event_manager: OtelEventManager, -) where - S: Stream> + Unpin, +fn map_response_stream(api_stream: S, otel_event_manager: OtelEventManager) -> ResponseStream +where + S: futures::Stream> + + Unpin + + Send + + 'static, { - let mut stream = stream.eventsource(); - - // If the stream stays completely silent for an extended period treat it as disconnected. - // The response id returned from the "complete" message. - let mut response_completed: Option = None; - let mut response_error: Option = None; - - loop { - let start = std::time::Instant::now(); - let response = timeout(idle_timeout, stream.next()).await; - let duration = start.elapsed(); - otel_event_manager.log_sse_event(&response, duration); - - let sse = match response { - Ok(Some(Ok(sse))) => sse, - Ok(Some(Err(e))) => { - debug!("SSE Error: {e:#}"); - let event = CodexErr::Stream(e.to_string(), None); - let _ = tx_event.send(Err(event)).await; - return; - } - Ok(None) => { - match response_completed { - Some(ResponseCompleted { - id: response_id, - usage, - }) => { - if let Some(token_usage) = &usage { - otel_event_manager.sse_event_completed( - token_usage.input_tokens, - token_usage.output_tokens, - token_usage - .input_tokens_details - .as_ref() - .map(|d| d.cached_tokens), - token_usage - .output_tokens_details - .as_ref() - .map(|d| d.reasoning_tokens), - token_usage.total_tokens, - ); - } - let event = ResponseEvent::Completed { - response_id, - token_usage: usage.map(Into::into), - }; - let _ = tx_event.send(Ok(event)).await; - } - None => { - let error = response_error.unwrap_or(CodexErr::Stream( - "stream closed before response.completed".into(), - None, - )); - otel_event_manager.see_event_completed_failed(&error); - - let _ = tx_event.send(Err(error)).await; - } - } - return; - } - Err(_) => { - let _ = tx_event - .send(Err(CodexErr::Stream( - "idle timeout waiting for SSE".into(), - None, - ))) - .await; - return; - } - }; - - let raw = sse.data.clone(); - trace!("SSE event: {}", raw); - - let event: SseEvent = match serde_json::from_str(&sse.data) { - Ok(event) => event, - Err(e) => { - debug!("Failed to parse SSE event: {e}, data: {}", &sse.data); - continue; - } - }; - - match event.kind.as_str() { - // Individual output item finalised. Forward immediately so the - // rest of the agent can stream assistant text/functions *live* - // instead of waiting for the final `response.completed` envelope. - // - // IMPORTANT: We used to ignore these events and forward the - // duplicated `output` array embedded in the `response.completed` - // payload. That produced two concrete issues: - // 1. No real‑time streaming – the user only saw output after the - // entire turn had finished, which broke the "typing" UX and - // made long‑running turns look stalled. - // 2. Duplicate `function_call_output` items – both the - // individual *and* the completed array were forwarded, which - // confused the backend and triggered 400 - // "previous_response_not_found" errors because the duplicated - // IDs did not match the incremental turn chain. - // - // The fix is to forward the incremental events *as they come* and - // drop the duplicated list inside `response.completed`. - "response.output_item.done" => { - let Some(item_val) = event.item else { continue }; - let Ok(item) = serde_json::from_value::(item_val) else { - debug!("failed to parse ResponseItem from output_item.done"); - continue; - }; - - let event = ResponseEvent::OutputItemDone(item); - if tx_event.send(Ok(event)).await.is_err() { - return; - } - } - "response.output_text.delta" => { - if let Some(delta) = event.delta { - let event = ResponseEvent::OutputTextDelta(delta); - if tx_event.send(Ok(event)).await.is_err() { - return; - } - } - } - "response.reasoning_summary_text.delta" => { - if let (Some(delta), Some(summary_index)) = (event.delta, event.summary_index) { - let event = ResponseEvent::ReasoningSummaryDelta { - delta, - summary_index, - }; - if tx_event.send(Ok(event)).await.is_err() { - return; - } - } - } - "response.reasoning_text.delta" => { - if let (Some(delta), Some(content_index)) = (event.delta, event.content_index) { - let event = ResponseEvent::ReasoningContentDelta { - delta, - content_index, - }; - if tx_event.send(Ok(event)).await.is_err() { - return; - } - } - } - "response.created" => { - if event.response.is_some() { - let _ = tx_event.send(Ok(ResponseEvent::Created {})).await; - } - } - "response.failed" => { - if let Some(resp_val) = event.response { - response_error = Some(CodexErr::Stream( - "response.failed event received".to_string(), - None, - )); - - let error = resp_val.get("error"); - - if let Some(error) = error { - match serde_json::from_value::(error.clone()) { - Ok(error) => { - if is_context_window_error(&error) { - response_error = Some(CodexErr::ContextWindowExceeded); - } else if is_quota_exceeded_error(&error) { - response_error = Some(CodexErr::QuotaExceeded); - } else { - let delay = try_parse_retry_after(&error); - let message = error.message.clone().unwrap_or_default(); - response_error = Some(CodexErr::Stream(message, delay)); - } - } - Err(e) => { - let error = format!("failed to parse ErrorResponse: {e}"); - debug!(error); - response_error = Some(CodexErr::Stream(error, None)) - } - } - } - } - } - // Final response completed – includes array of output items & id - "response.completed" => { - if let Some(resp_val) = event.response { - match serde_json::from_value::(resp_val) { - Ok(r) => { - response_completed = Some(r); - } - Err(e) => { - let error = format!("failed to parse ResponseCompleted: {e}"); - debug!(error); - response_error = Some(CodexErr::Stream(error, None)); - continue; - } - }; - }; - } - "response.content_part.done" - | "response.function_call_arguments.delta" - | "response.custom_tool_call_input.delta" - | "response.custom_tool_call_input.done" // also emitted as response.output_item.done - | "response.in_progress" - | "response.output_text.done" => {} - "response.output_item.added" => { - let Some(item_val) = event.item else { continue }; - let Ok(item) = serde_json::from_value::(item_val) else { - debug!("failed to parse ResponseItem from output_item.done"); - continue; - }; - - let event = ResponseEvent::OutputItemAdded(item); - if tx_event.send(Ok(event)).await.is_err() { - return; - } - } - "response.reasoning_summary_part.added" => { - if let Some(summary_index) = event.summary_index { - // Boundary between reasoning summary sections (e.g., titles). - let event = ResponseEvent::ReasoningSummaryPartAdded { summary_index }; - if tx_event.send(Ok(event)).await.is_err() { - return; - } - } - } - "response.reasoning_summary_text.done" => {} - _ => {} - } - } -} - -/// used in tests to stream from a text SSE file -async fn stream_from_fixture( - path: impl AsRef, - provider: ModelProviderInfo, - otel_event_manager: OtelEventManager, -) -> Result { let (tx_event, rx_event) = mpsc::channel::>(1600); - let f = std::fs::File::open(path.as_ref())?; - let lines = std::io::BufReader::new(f).lines(); - - // insert \n\n after each line for proper SSE parsing - let mut content = String::new(); - for line in lines { - content.push_str(&line?); - content.push_str("\n\n"); - } - - let rdr = std::io::Cursor::new(content); - let stream = ReaderStream::new(rdr).map_err(CodexErr::Io); - tokio::spawn(process_sse( - stream, - tx_event, - provider.stream_idle_timeout(), - otel_event_manager, - )); - Ok(ResponseStream { rx_event }) -} - -fn rate_limit_regex() -> &'static Regex { - static RE: OnceLock = OnceLock::new(); - - // Match both OpenAI-style messages like "Please try again in 1.898s" - // and Azure OpenAI-style messages like "Try again in 35 seconds". - #[expect(clippy::unwrap_used)] - RE.get_or_init(|| Regex::new(r"(?i)try again in\s*(\d+(?:\.\d+)?)\s*(s|ms|seconds?)").unwrap()) -} - -fn try_parse_retry_after(err: &Error) -> Option { - if err.code != Some("rate_limit_exceeded".to_string()) { - return None; - } - - // parse retry hints like "try again in 1.898s" or - // "Try again in 35 seconds" using regex - let re = rate_limit_regex(); - if let Some(message) = &err.message - && let Some(captures) = re.captures(message) - { - let seconds = captures.get(1); - let unit = captures.get(2); - - if let (Some(value), Some(unit)) = (seconds, unit) { - let value = value.as_str().parse::().ok()?; - let unit = unit.as_str().to_ascii_lowercase(); - - if unit == "s" || unit.starts_with("second") { - return Some(Duration::from_secs_f64(value)); - } else if unit == "ms" { - return Some(Duration::from_millis(value as u64)); - } - } - } - None -} - -fn is_context_window_error(error: &Error) -> bool { - error.code.as_deref() == Some("context_length_exceeded") -} - -fn is_quota_exceeded_error(error: &Error) -> bool { - error.code.as_deref() == Some("insufficient_quota") -} - -#[cfg(test)] -mod tests { - use super::*; - use assert_matches::assert_matches; - use serde_json::json; - use tokio::sync::mpsc; - use tokio_test::io::Builder as IoBuilder; - use tokio_util::io::ReaderStream; - - // ──────────────────────────── - // Helpers - // ──────────────────────────── - - /// Runs the SSE parser on pre-chunked byte slices and returns every event - /// (including any final `Err` from a stream-closure check). - async fn collect_events( - chunks: &[&[u8]], - provider: ModelProviderInfo, - otel_event_manager: OtelEventManager, - ) -> Vec> { - let mut builder = IoBuilder::new(); - for chunk in chunks { - builder.read(chunk); - } - - let reader = builder.build(); - let stream = ReaderStream::new(reader).map_err(CodexErr::Io); - let (tx, mut rx) = mpsc::channel::>(16); - tokio::spawn(process_sse( - stream, - tx, - provider.stream_idle_timeout(), - otel_event_manager, - )); - - let mut events = Vec::new(); - while let Some(ev) = rx.recv().await { - events.push(ev); - } - events - } - - /// Builds an in-memory SSE stream from JSON fixtures and returns only the - /// successfully parsed events (panics on internal channel errors). - async fn run_sse( - events: Vec, - provider: ModelProviderInfo, - otel_event_manager: OtelEventManager, - ) -> Vec { - let mut body = String::new(); - for e in events { - let kind = e - .get("type") - .and_then(|v| v.as_str()) - .expect("fixture event missing type"); - if e.as_object().map(|o| o.len() == 1).unwrap_or(false) { - body.push_str(&format!("event: {kind}\n\n")); - } else { - body.push_str(&format!("event: {kind}\ndata: {e}\n\n")); - } - } - - let (tx, mut rx) = mpsc::channel::>(8); - let stream = ReaderStream::new(std::io::Cursor::new(body)).map_err(CodexErr::Io); - tokio::spawn(process_sse( - stream, - tx, - provider.stream_idle_timeout(), - otel_event_manager, - )); - - let mut out = Vec::new(); - while let Some(ev) = rx.recv().await { - out.push(ev.expect("channel closed")); - } - out - } - - fn otel_event_manager() -> OtelEventManager { - OtelEventManager::new( - ConversationId::new(), - "test", - "test", - None, - Some("test@test.com".to_string()), - Some(AuthMode::ChatGPT), - false, - "test".to_string(), - ) - } - - // ──────────────────────────── - // Tests from `implement-test-for-responses-api-sse-parser` - // ──────────────────────────── - - #[tokio::test] - async fn parses_items_and_completed() { - let item1 = json!({ - "type": "response.output_item.done", - "item": { - "type": "message", - "role": "assistant", - "content": [{"type": "output_text", "text": "Hello"}] - } - }) - .to_string(); - - let item2 = json!({ - "type": "response.output_item.done", - "item": { - "type": "message", - "role": "assistant", - "content": [{"type": "output_text", "text": "World"}] - } - }) - .to_string(); - - let completed = json!({ - "type": "response.completed", - "response": { "id": "resp1" } - }) - .to_string(); - - let sse1 = format!("event: response.output_item.done\ndata: {item1}\n\n"); - let sse2 = format!("event: response.output_item.done\ndata: {item2}\n\n"); - let sse3 = format!("event: response.completed\ndata: {completed}\n\n"); - - let provider = ModelProviderInfo { - name: "test".to_string(), - base_url: Some("https://test.com".to_string()), - env_key: Some("TEST_API_KEY".to_string()), - env_key_instructions: None, - experimental_bearer_token: None, - wire_api: WireApi::Responses, - query_params: None, - http_headers: None, - env_http_headers: None, - request_max_retries: Some(0), - stream_max_retries: Some(0), - stream_idle_timeout_ms: Some(1000), - requires_openai_auth: false, - }; - - let otel_event_manager = otel_event_manager(); - - let events = collect_events( - &[sse1.as_bytes(), sse2.as_bytes(), sse3.as_bytes()], - provider, - otel_event_manager, - ) - .await; - - assert_eq!(events.len(), 3); - - matches!( - &events[0], - Ok(ResponseEvent::OutputItemDone(ResponseItem::Message { role, .. })) - if role == "assistant" - ); - - matches!( - &events[1], - Ok(ResponseEvent::OutputItemDone(ResponseItem::Message { role, .. })) - if role == "assistant" - ); - - match &events[2] { - Ok(ResponseEvent::Completed { - response_id, - token_usage, - }) => { - assert_eq!(response_id, "resp1"); - assert!(token_usage.is_none()); - } - other => panic!("unexpected third event: {other:?}"), - } - } - - #[tokio::test] - async fn error_when_missing_completed() { - let item1 = json!({ - "type": "response.output_item.done", - "item": { - "type": "message", - "role": "assistant", - "content": [{"type": "output_text", "text": "Hello"}] - } - }) - .to_string(); - - let sse1 = format!("event: response.output_item.done\ndata: {item1}\n\n"); - let provider = ModelProviderInfo { - name: "test".to_string(), - base_url: Some("https://test.com".to_string()), - env_key: Some("TEST_API_KEY".to_string()), - env_key_instructions: None, - experimental_bearer_token: None, - wire_api: WireApi::Responses, - query_params: None, - http_headers: None, - env_http_headers: None, - request_max_retries: Some(0), - stream_max_retries: Some(0), - stream_idle_timeout_ms: Some(1000), - requires_openai_auth: false, - }; - - let otel_event_manager = otel_event_manager(); - - let events = collect_events(&[sse1.as_bytes()], provider, otel_event_manager).await; - - assert_eq!(events.len(), 2); - - matches!(events[0], Ok(ResponseEvent::OutputItemDone(_))); - - match &events[1] { - Err(CodexErr::Stream(msg, _)) => { - assert_eq!(msg, "stream closed before response.completed") - } - other => panic!("unexpected second event: {other:?}"), - } - } - - #[tokio::test] - async fn error_when_error_event() { - let raw_error = r#"{"type":"response.failed","sequence_number":3,"response":{"id":"resp_689bcf18d7f08194bf3440ba62fe05d803fee0cdac429894","object":"response","created_at":1755041560,"status":"failed","background":false,"error":{"code":"rate_limit_exceeded","message":"Rate limit reached for gpt-5.1 in organization org-AAA on tokens per min (TPM): Limit 30000, Used 22999, Requested 12528. Please try again in 11.054s. Visit https://platform.openai.com/account/rate-limits to learn more."}, "usage":null,"user":null,"metadata":{}}}"#; - - let sse1 = format!("event: response.failed\ndata: {raw_error}\n\n"); - let provider = ModelProviderInfo { - name: "test".to_string(), - base_url: Some("https://test.com".to_string()), - env_key: Some("TEST_API_KEY".to_string()), - env_key_instructions: None, - experimental_bearer_token: None, - wire_api: WireApi::Responses, - query_params: None, - http_headers: None, - env_http_headers: None, - request_max_retries: Some(0), - stream_max_retries: Some(0), - stream_idle_timeout_ms: Some(1000), - requires_openai_auth: false, - }; - - let otel_event_manager = otel_event_manager(); - - let events = collect_events(&[sse1.as_bytes()], provider, otel_event_manager).await; - - assert_eq!(events.len(), 1); - - match &events[0] { - Err(CodexErr::Stream(msg, delay)) => { - assert_eq!( - msg, - "Rate limit reached for gpt-5.1 in organization org-AAA on tokens per min (TPM): Limit 30000, Used 22999, Requested 12528. Please try again in 11.054s. Visit https://platform.openai.com/account/rate-limits to learn more." - ); - assert_eq!(*delay, Some(Duration::from_secs_f64(11.054))); - } - other => panic!("unexpected second event: {other:?}"), - } - } - - #[tokio::test] - async fn context_window_error_is_fatal() { - let raw_error = r#"{"type":"response.failed","sequence_number":3,"response":{"id":"resp_5c66275b97b9baef1ed95550adb3b7ec13b17aafd1d2f11b","object":"response","created_at":1759510079,"status":"failed","background":false,"error":{"code":"context_length_exceeded","message":"Your input exceeds the context window of this model. Please adjust your input and try again."},"usage":null,"user":null,"metadata":{}}}"#; - - let sse1 = format!("event: response.failed\ndata: {raw_error}\n\n"); - let provider = ModelProviderInfo { - name: "test".to_string(), - base_url: Some("https://test.com".to_string()), - env_key: Some("TEST_API_KEY".to_string()), - env_key_instructions: None, - experimental_bearer_token: None, - wire_api: WireApi::Responses, - query_params: None, - http_headers: None, - env_http_headers: None, - request_max_retries: Some(0), - stream_max_retries: Some(0), - stream_idle_timeout_ms: Some(1000), - requires_openai_auth: false, - }; - - let otel_event_manager = otel_event_manager(); - - let events = collect_events(&[sse1.as_bytes()], provider, otel_event_manager).await; - - assert_eq!(events.len(), 1); - - match &events[0] { - Err(err @ CodexErr::ContextWindowExceeded) => { - assert_eq!(err.to_string(), CodexErr::ContextWindowExceeded.to_string()); - } - other => panic!("unexpected context window event: {other:?}"), - } - } - - #[tokio::test] - async fn context_window_error_with_newline_is_fatal() { - let raw_error = r#"{"type":"response.failed","sequence_number":4,"response":{"id":"resp_fatal_newline","object":"response","created_at":1759510080,"status":"failed","background":false,"error":{"code":"context_length_exceeded","message":"Your input exceeds the context window of this model. Please adjust your input and try\nagain."},"usage":null,"user":null,"metadata":{}}}"#; - - let sse1 = format!("event: response.failed\ndata: {raw_error}\n\n"); - let provider = ModelProviderInfo { - name: "test".to_string(), - base_url: Some("https://test.com".to_string()), - env_key: Some("TEST_API_KEY".to_string()), - env_key_instructions: None, - experimental_bearer_token: None, - wire_api: WireApi::Responses, - query_params: None, - http_headers: None, - env_http_headers: None, - request_max_retries: Some(0), - stream_max_retries: Some(0), - stream_idle_timeout_ms: Some(1000), - requires_openai_auth: false, - }; - - let otel_event_manager = otel_event_manager(); - - let events = collect_events(&[sse1.as_bytes()], provider, otel_event_manager).await; - - assert_eq!(events.len(), 1); - - match &events[0] { - Err(err @ CodexErr::ContextWindowExceeded) => { - assert_eq!(err.to_string(), CodexErr::ContextWindowExceeded.to_string()); - } - other => panic!("unexpected context window event: {other:?}"), - } - } - - #[tokio::test] - async fn quota_exceeded_error_is_fatal() { - let raw_error = r#"{"type":"response.failed","sequence_number":3,"response":{"id":"resp_fatal_quota","object":"response","created_at":1759771626,"status":"failed","background":false,"error":{"code":"insufficient_quota","message":"You exceeded your current quota, please check your plan and billing details. For more information on this error, read the docs: https://platform.openai.com/docs/guides/error-codes/api-errors."},"incomplete_details":null}}"#; - - let sse1 = format!("event: response.failed\ndata: {raw_error}\n\n"); - let provider = ModelProviderInfo { - name: "test".to_string(), - base_url: Some("https://test.com".to_string()), - env_key: Some("TEST_API_KEY".to_string()), - env_key_instructions: None, - experimental_bearer_token: None, - wire_api: WireApi::Responses, - query_params: None, - http_headers: None, - env_http_headers: None, - request_max_retries: Some(0), - stream_max_retries: Some(0), - stream_idle_timeout_ms: Some(1000), - requires_openai_auth: false, - }; - - let otel_event_manager = otel_event_manager(); - - let events = collect_events(&[sse1.as_bytes()], provider, otel_event_manager).await; - - assert_eq!(events.len(), 1); - - match &events[0] { - Err(err @ CodexErr::QuotaExceeded) => { - assert_eq!(err.to_string(), CodexErr::QuotaExceeded.to_string()); - } - other => panic!("unexpected quota exceeded event: {other:?}"), - } - } - - // ──────────────────────────── - // Table-driven test from `main` - // ──────────────────────────── - - /// Verifies that the adapter produces the right `ResponseEvent` for a - /// variety of incoming `type` values. - #[tokio::test] - async fn table_driven_event_kinds() { - struct TestCase { - name: &'static str, - event: serde_json::Value, - expect_first: fn(&ResponseEvent) -> bool, - expected_len: usize, - } - - fn is_created(ev: &ResponseEvent) -> bool { - matches!(ev, ResponseEvent::Created) - } - fn is_output(ev: &ResponseEvent) -> bool { - matches!(ev, ResponseEvent::OutputItemDone(_)) - } - fn is_completed(ev: &ResponseEvent) -> bool { - matches!(ev, ResponseEvent::Completed { .. }) - } - - let completed = json!({ - "type": "response.completed", - "response": { - "id": "c", - "usage": { - "input_tokens": 0, - "input_tokens_details": null, - "output_tokens": 0, - "output_tokens_details": null, - "total_tokens": 0 - }, - "output": [] - } - }); - - let cases = vec![ - TestCase { - name: "created", - event: json!({"type": "response.created", "response": {}}), - expect_first: is_created, - expected_len: 2, - }, - TestCase { - name: "output_item.done", - event: json!({ - "type": "response.output_item.done", - "item": { - "type": "message", - "role": "assistant", - "content": [ - {"type": "output_text", "text": "hi"} - ] + let manager = otel_event_manager; + + tokio::spawn(async move { + let mut logged_error = false; + let mut api_stream = api_stream; + while let Some(event) = api_stream.next().await { + match event { + Ok(ResponseEvent::Completed { + response_id, + token_usage, + }) => { + if let Some(usage) = &token_usage { + manager.sse_event_completed( + usage.input_tokens, + usage.output_tokens, + Some(usage.cached_input_tokens), + Some(usage.reasoning_output_tokens), + usage.total_tokens, + ); } - }), - expect_first: is_output, - expected_len: 2, - }, - TestCase { - name: "unknown", - event: json!({"type": "response.new_tool_event"}), - expect_first: is_completed, - expected_len: 1, - }, - ]; - - for case in cases { - let mut evs = vec![case.event]; - evs.push(completed.clone()); - - let provider = ModelProviderInfo { - name: "test".to_string(), - base_url: Some("https://test.com".to_string()), - env_key: Some("TEST_API_KEY".to_string()), - env_key_instructions: None, - experimental_bearer_token: None, - wire_api: WireApi::Responses, - query_params: None, - http_headers: None, - env_http_headers: None, - request_max_retries: Some(0), - stream_max_retries: Some(0), - stream_idle_timeout_ms: Some(1000), - requires_openai_auth: false, - }; - - let otel_event_manager = otel_event_manager(); - - let out = run_sse(evs, provider, otel_event_manager).await; - assert_eq!(out.len(), case.expected_len, "case {}", case.name); - assert!( - (case.expect_first)(&out[0]), - "first event mismatch in case {}", - case.name - ); + if tx_event + .send(Ok(ResponseEvent::Completed { + response_id, + token_usage, + })) + .await + .is_err() + { + return; + } + } + Ok(event) => { + if tx_event.send(Ok(event)).await.is_err() { + return; + } + } + Err(err) => { + let mapped = map_api_error(err); + if !logged_error { + manager.see_event_completed_failed(&mapped); + logged_error = true; + } + if tx_event.send(Err(mapped)).await.is_err() { + return; + } + } + } } + }); + + ResponseStream { rx_event } +} + +/// Handles a 401 response by optionally refreshing ChatGPT tokens once. +/// +/// When refresh succeeds, the caller should retry the API call; otherwise +/// the mapped `CodexErr` is returned to the caller. +async fn handle_unauthorized( + status: StatusCode, + refreshed: &mut bool, + auth_manager: &Option>, + auth: &Option, +) -> Result<()> { + if *refreshed { + return Err(map_unauthorized_status(status)); } - #[test] - fn test_try_parse_retry_after() { - let err = Error { - r#type: None, - message: Some("Rate limit reached for gpt-5.1 in organization org- on tokens per min (TPM): Limit 1, Used 1, Requested 19304. Please try again in 28ms. Visit https://platform.openai.com/account/rate-limits to learn more.".to_string()), - code: Some("rate_limit_exceeded".to_string()), - plan_type: None, - resets_at: None - }; - - let delay = try_parse_retry_after(&err); - assert_eq!(delay, Some(Duration::from_millis(28))); - } - - #[test] - fn test_try_parse_retry_after_no_delay() { - let err = Error { - r#type: None, - message: Some("Rate limit reached for gpt-5.1 in organization on tokens per min (TPM): Limit 30000, Used 6899, Requested 24050. Please try again in 1.898s. Visit https://platform.openai.com/account/rate-limits to learn more.".to_string()), - code: Some("rate_limit_exceeded".to_string()), - plan_type: None, - resets_at: None - }; - let delay = try_parse_retry_after(&err); - assert_eq!(delay, Some(Duration::from_secs_f64(1.898))); - } - - #[test] - fn test_try_parse_retry_after_azure() { - let err = Error { - r#type: None, - message: Some("Rate limit exceeded. Try again in 35 seconds.".to_string()), - code: Some("rate_limit_exceeded".to_string()), - plan_type: None, - resets_at: None, - }; - let delay = try_parse_retry_after(&err); - assert_eq!(delay, Some(Duration::from_secs(35))); - } - - #[test] - fn error_response_deserializes_schema_known_plan_type_and_serializes_back() { - use crate::token_data::KnownPlan; - use crate::token_data::PlanType; - - let json = - r#"{"error":{"type":"usage_limit_reached","plan_type":"pro","resets_at":1704067200}}"#; - let resp: ErrorResponse = serde_json::from_str(json).expect("should deserialize schema"); - - assert_matches!(resp.error.plan_type, Some(PlanType::Known(KnownPlan::Pro))); - - let plan_json = serde_json::to_string(&resp.error.plan_type).expect("serialize plan_type"); - assert_eq!(plan_json, "\"pro\""); - } - - #[test] - fn error_response_deserializes_schema_unknown_plan_type_and_serializes_back() { - use crate::token_data::PlanType; - - let json = - r#"{"error":{"type":"usage_limit_reached","plan_type":"vip","resets_at":1704067260}}"#; - let resp: ErrorResponse = serde_json::from_str(json).expect("should deserialize schema"); - - assert_matches!(resp.error.plan_type, Some(PlanType::Unknown(ref s)) if s == "vip"); - - let plan_json = serde_json::to_string(&resp.error.plan_type).expect("serialize plan_type"); - assert_eq!(plan_json, "\"vip\""); + if let Some(manager) = auth_manager.as_ref() + && let Some(auth) = auth.as_ref() + && auth.mode == AuthMode::ChatGPT + { + match manager.refresh_token().await { + Ok(_) => { + *refreshed = true; + Ok(()) + } + Err(RefreshTokenError::Permanent(failed)) => Err(CodexErr::RefreshTokenFailed(failed)), + Err(RefreshTokenError::Transient(other)) => Err(CodexErr::Io(other)), + } + } else { + Err(map_unauthorized_status(status)) + } +} + +fn map_unauthorized_status(status: StatusCode) -> CodexErr { + map_api_error(ApiError::Transport(TransportError::Http { + status, + headers: None, + body: None, + })) +} + +struct ApiTelemetry { + otel_event_manager: OtelEventManager, +} + +impl ApiTelemetry { + fn new(otel_event_manager: OtelEventManager) -> Self { + Self { otel_event_manager } + } +} + +impl RequestTelemetry for ApiTelemetry { + fn on_request( + &self, + attempt: u64, + status: Option, + error: Option<&TransportError>, + duration: Duration, + ) { + let error_message = error.map(std::string::ToString::to_string); + self.otel_event_manager.record_api_request( + attempt, + status.map(|s| s.as_u16()), + error_message.as_deref(), + duration, + ); + } +} + +impl SseTelemetry for ApiTelemetry { + fn on_sse_poll( + &self, + result: &std::result::Result< + Option>>, + tokio::time::error::Elapsed, + >, + duration: Duration, + ) { + self.otel_event_manager.log_sse_event(result, duration); } } diff --git a/codex-rs/core/src/client_common.rs b/codex-rs/core/src/client_common.rs index 21f6fc6576..a249ca6fcc 100644 --- a/codex-rs/core/src/client_common.rs +++ b/codex-rs/core/src/client_common.rs @@ -1,16 +1,11 @@ use crate::client_common::tools::ToolSpec; use crate::error::Result; use crate::model_family::ModelFamily; -use crate::protocol::RateLimitSnapshot; -use crate::protocol::TokenUsage; +pub use codex_api::common::ResponseEvent; use codex_apply_patch::APPLY_PATCH_TOOL_INSTRUCTIONS; -use codex_protocol::config_types::ReasoningEffort as ReasoningEffortConfig; -use codex_protocol::config_types::ReasoningSummary as ReasoningSummaryConfig; -use codex_protocol::config_types::Verbosity as VerbosityConfig; use codex_protocol::models::ResponseItem; use futures::Stream; use serde::Deserialize; -use serde::Serialize; use serde_json::Value; use std::borrow::Cow; use std::collections::HashSet; @@ -184,104 +179,6 @@ fn strip_total_output_header(output: &str) -> Option<(&str, u32)> { Some((remainder, total_lines)) } -#[derive(Debug)] -pub enum ResponseEvent { - Created, - OutputItemDone(ResponseItem), - OutputItemAdded(ResponseItem), - Completed { - response_id: String, - token_usage: Option, - }, - OutputTextDelta(String), - ReasoningSummaryDelta { - delta: String, - summary_index: i64, - }, - ReasoningContentDelta { - delta: String, - content_index: i64, - }, - ReasoningSummaryPartAdded { - summary_index: i64, - }, - RateLimits(RateLimitSnapshot), -} - -#[derive(Debug, Serialize)] -pub(crate) struct Reasoning { - #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) effort: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) summary: Option, -} - -#[derive(Debug, Serialize, Default, Clone)] -#[serde(rename_all = "snake_case")] -pub(crate) enum TextFormatType { - #[default] - JsonSchema, -} - -#[derive(Debug, Serialize, Default, Clone)] -pub(crate) struct TextFormat { - pub(crate) r#type: TextFormatType, - pub(crate) strict: bool, - pub(crate) schema: Value, - pub(crate) name: String, -} - -/// Controls under the `text` field in the Responses API for GPT-5. -#[derive(Debug, Serialize, Default, Clone)] -pub(crate) struct TextControls { - #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) verbosity: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) format: Option, -} - -#[derive(Debug, Serialize, Default, Clone)] -#[serde(rename_all = "lowercase")] -pub(crate) enum OpenAiVerbosity { - Low, - #[default] - Medium, - High, -} - -impl From for OpenAiVerbosity { - fn from(v: VerbosityConfig) -> Self { - match v { - VerbosityConfig::Low => OpenAiVerbosity::Low, - VerbosityConfig::Medium => OpenAiVerbosity::Medium, - VerbosityConfig::High => OpenAiVerbosity::High, - } - } -} - -/// Request object that is serialized as JSON and POST'ed when using the -/// Responses API. -#[derive(Debug, Serialize)] -pub(crate) struct ResponsesApiRequest<'a> { - pub(crate) model: &'a str, - pub(crate) instructions: &'a str, - // TODO(mbolin): ResponseItem::Other should not be serialized. Currently, - // we code defensively to avoid this case, but perhaps we should use a - // separate enum for serialization. - pub(crate) input: &'a Vec, - pub(crate) tools: &'a [serde_json::Value], - pub(crate) tool_choice: &'static str, - pub(crate) parallel_tool_calls: bool, - pub(crate) reasoning: Option, - pub(crate) store: bool, - pub(crate) stream: bool, - pub(crate) include: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) prompt_cache_key: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub(crate) text: Option, -} - pub(crate) mod tools { use crate::tools::spec::JsonSchema; use serde::Deserialize; @@ -341,25 +238,6 @@ pub(crate) mod tools { } } -pub(crate) fn create_text_param_for_request( - verbosity: Option, - output_schema: &Option, -) -> Option { - if verbosity.is_none() && output_schema.is_none() { - return None; - } - - Some(TextControls { - verbosity: verbosity.map(std::convert::Into::into), - format: output_schema.as_ref().map(|schema| TextFormat { - r#type: TextFormatType::JsonSchema, - strict: true, - schema: schema.clone(), - name: "codex_output_schema".to_string(), - }), - }) -} - pub struct ResponseStream { pub(crate) rx_event: mpsc::Receiver>, } @@ -375,6 +253,10 @@ impl Stream for ResponseStream { #[cfg(test)] mod tests { use crate::model_family::find_family_for_model; + use codex_api::ResponsesApiRequest; + use codex_api::common::OpenAiVerbosity; + use codex_api::common::TextControls; + use codex_api::create_text_param_for_request; use pretty_assertions::assert_eq; use super::*; diff --git a/codex-rs/core/src/default_client.rs b/codex-rs/core/src/default_client.rs index 8e4635460c..29986c401d 100644 --- a/codex-rs/core/src/default_client.rs +++ b/codex-rs/core/src/default_client.rs @@ -258,6 +258,11 @@ fn sanitize_user_agent(candidate: String, fallback: &str) -> String { /// Create an HTTP client with default `originator` and `User-Agent` headers set. pub fn create_client() -> CodexHttpClient { + let inner = build_reqwest_client(); + CodexHttpClient::new(inner) +} + +pub fn build_reqwest_client() -> reqwest::Client { use reqwest::header::HeaderMap; let mut headers = HeaderMap::new(); @@ -272,8 +277,7 @@ pub fn create_client() -> CodexHttpClient { builder = builder.no_proxy(); } - let inner = builder.build().unwrap_or_else(|_| reqwest::Client::new()); - CodexHttpClient::new(inner) + builder.build().unwrap_or_else(|_| reqwest::Client::new()) } fn is_sandboxed() -> bool { diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index 805943a2e7..7a9440eb28 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -5,10 +5,10 @@ // the TUI or the tracing stack). #![deny(clippy::print_stdout, clippy::print_stderr)] +pub mod api_bridge; mod apply_patch; pub mod auth; pub mod bash; -mod chat_completions; mod client; mod client_common; pub mod codex; diff --git a/codex-rs/core/src/model_provider_info.rs b/codex-rs/core/src/model_provider_info.rs index 3ab341eea5..4912a64694 100644 --- a/codex-rs/core/src/model_provider_info.rs +++ b/codex-rs/core/src/model_provider_info.rs @@ -5,11 +5,13 @@ //! 2. User-defined entries inside `~/.codex/config.toml` under the `model_providers` //! key. These override or extend the defaults at runtime. -use crate::CodexAuth; -use crate::default_client::CodexHttpClient; -use crate::default_client::CodexRequestBuilder; -use crate::error::CodexErr; +use codex_api::Provider as ApiProvider; +use codex_api::WireApi as ApiWireApi; +use codex_api::provider::RetryConfig as ApiRetryConfig; use codex_app_server_protocol::AuthMode; +use http::HeaderMap; +use http::header::HeaderName; +use http::header::HeaderValue; use serde::Deserialize; use serde::Serialize; use std::collections::HashMap; @@ -97,148 +99,14 @@ pub struct ModelProviderInfo { } impl ModelProviderInfo { - /// Construct a `POST` RequestBuilder for the given URL using the provided - /// [`CodexHttpClient`] applying: - /// • provider-specific headers (static + env based) - /// • Bearer auth header when an API key is available. - /// • Auth token for OAuth. - /// - /// If the provider declares an `env_key` but the variable is missing/empty, returns an [`Err`] identical to the - /// one produced by [`ModelProviderInfo::api_key`]. - pub async fn create_request_builder<'a>( - &'a self, - client: &'a CodexHttpClient, - auth: &Option, - ) -> crate::error::Result { - let effective_auth = self.effective_auth(auth)?; - - let url = self.get_full_url(&effective_auth); - - let mut builder = client.post(url); - - if let Some(auth) = effective_auth.as_ref() { - builder = builder.bearer_auth(auth.get_token().await?); - } - - Ok(self.apply_http_headers(builder)) - } - - pub async fn create_compact_request_builder<'a>( - &'a self, - client: &'a CodexHttpClient, - auth: &Option, - ) -> crate::error::Result { - if self.wire_api != WireApi::Responses { - return Err(CodexErr::UnsupportedOperation( - "Compaction endpoint requires Responses API providers".to_string(), - )); - } - let effective_auth = self.effective_auth(auth)?; - - let url = self.get_compact_url(&effective_auth).ok_or_else(|| { - CodexErr::UnsupportedOperation( - "Compaction endpoint requires Responses API providers".to_string(), - ) - })?; - - let mut builder = client.post(url); - - if let Some(auth) = effective_auth.as_ref() { - builder = builder.bearer_auth(auth.get_token().await?); - } - - Ok(self.apply_http_headers(builder)) - } - - fn effective_auth(&self, auth: &Option) -> crate::error::Result> { - if let Some(secret_key) = &self.experimental_bearer_token { - return Ok(Some(CodexAuth::from_api_key(secret_key))); - } - - match self.api_key() { - Ok(Some(key)) => Ok(Some(CodexAuth::from_api_key(&key))), - Ok(None) => Ok(auth.clone()), - Err(err) => { - if auth.is_some() { - Ok(auth.clone()) - } else { - Err(err) - } - } - } - } - - fn get_query_string(&self) -> String { - self.query_params - .as_ref() - .map_or_else(String::new, |params| { - let full_params = params - .iter() - .map(|(k, v)| format!("{k}={v}")) - .collect::>() - .join("&"); - format!("?{full_params}") - }) - } - - pub(crate) fn get_full_url(&self, auth: &Option) -> String { - let default_base_url = if matches!( - auth, - Some(CodexAuth { - mode: AuthMode::ChatGPT, - .. - }) - ) { - "https://chatgpt.com/backend-api/codex" - } else { - "https://api.openai.com/v1" - }; - let query_string = self.get_query_string(); - let base_url = self - .base_url - .clone() - .unwrap_or(default_base_url.to_string()); - - match self.wire_api { - WireApi::Responses => format!("{base_url}/responses{query_string}"), - WireApi::Chat => format!("{base_url}/chat/completions{query_string}"), - } - } - - pub(crate) fn get_compact_url(&self, auth: &Option) -> Option { - if self.wire_api != WireApi::Responses { - return None; - } - let full = self.get_full_url(auth); - if let Some((path, query)) = full.split_once('?') { - Some(format!("{path}/compact?{query}")) - } else { - Some(format!("{full}/compact")) - } - } - - pub(crate) fn is_azure_responses_endpoint(&self) -> bool { - if self.wire_api != WireApi::Responses { - return false; - } - - if self.name.eq_ignore_ascii_case("azure") { - return true; - } - - self.base_url - .as_ref() - .map(|base| matches_azure_responses_base_url(base)) - .unwrap_or(false) - } - - /// Apply provider-specific HTTP headers (both static and environment-based) - /// onto an existing [`CodexRequestBuilder`] and return the updated - /// builder. - fn apply_http_headers(&self, mut builder: CodexRequestBuilder) -> CodexRequestBuilder { + #[allow(dead_code)] + fn build_header_map(&self) -> crate::error::Result { + let mut headers = HeaderMap::new(); if let Some(extra) = &self.http_headers { for (k, v) in extra { - builder = builder.header(k, v); + if let (Ok(name), Ok(value)) = (HeaderName::try_from(k), HeaderValue::try_from(v)) { + headers.insert(name, value); + } } } @@ -246,12 +114,52 @@ impl ModelProviderInfo { for (header, env_var) in env_headers { if let Ok(val) = std::env::var(env_var) && !val.trim().is_empty() + && let (Ok(name), Ok(value)) = + (HeaderName::try_from(header), HeaderValue::try_from(val)) { - builder = builder.header(header, val); + headers.insert(name, value); } } } - builder + + Ok(headers) + } + + pub(crate) fn to_api_provider( + &self, + auth_mode: Option, + ) -> crate::error::Result { + let default_base_url = if matches!(auth_mode, Some(AuthMode::ChatGPT)) { + "https://chatgpt.com/backend-api/codex" + } else { + "https://api.openai.com/v1" + }; + let base_url = self + .base_url + .clone() + .unwrap_or_else(|| default_base_url.to_string()); + + let headers = self.build_header_map()?; + let retry = ApiRetryConfig { + max_attempts: self.request_max_retries(), + base_delay: Duration::from_millis(200), + retry_429: false, + retry_5xx: true, + retry_transport: true, + }; + + Ok(ApiProvider { + name: self.name.clone(), + base_url, + query_params: self.query_params.clone(), + wire: match self.wire_api { + WireApi::Responses => ApiWireApi::Responses, + WireApi::Chat => ApiWireApi::Chat, + }, + headers, + retry, + stream_idle_timeout: self.stream_idle_timeout(), + }) } /// If `env_key` is Some, returns the API key for this provider if present @@ -409,18 +317,6 @@ pub fn create_oss_provider_with_base_url(base_url: &str, wire_api: WireApi) -> M } } -fn matches_azure_responses_base_url(base_url: &str) -> bool { - let base = base_url.to_ascii_lowercase(); - const AZURE_MARKERS: [&str; 5] = [ - "openai.azure.", - "cognitiveservices.azure.", - "aoai.azure.", - "azure-api.", - "azurefd.", - ]; - AZURE_MARKERS.iter().any(|marker| base.contains(marker)) -} - #[cfg(test)] mod tests { use super::*; @@ -517,8 +413,16 @@ env_http_headers = { "X-Example-Env-Header" = "EXAMPLE_ENV_VAR" } #[test] fn detects_azure_responses_base_urls() { - fn provider_for(base_url: &str) -> ModelProviderInfo { - ModelProviderInfo { + let positive_cases = [ + "https://foo.openai.azure.com/openai", + "https://foo.openai.azure.us/openai/deployments/bar", + "https://foo.cognitiveservices.azure.cn/openai", + "https://foo.aoai.azure.com/openai", + "https://foo.openai.azure-api.net/openai", + "https://foo.z01.azurefd.net/", + ]; + for base_url in positive_cases { + let provider = ModelProviderInfo { name: "test".into(), base_url: Some(base_url.into()), env_key: None, @@ -532,21 +436,10 @@ env_http_headers = { "X-Example-Env-Header" = "EXAMPLE_ENV_VAR" } stream_max_retries: None, stream_idle_timeout_ms: None, requires_openai_auth: false, - } - } - - let positive_cases = [ - "https://foo.openai.azure.com/openai", - "https://foo.openai.azure.us/openai/deployments/bar", - "https://foo.cognitiveservices.azure.cn/openai", - "https://foo.aoai.azure.com/openai", - "https://foo.openai.azure-api.net/openai", - "https://foo.z01.azurefd.net/", - ]; - for base_url in positive_cases { - let provider = provider_for(base_url); + }; + let api = provider.to_api_provider(None).expect("api provider"); assert!( - provider.is_azure_responses_endpoint(), + api.is_azure_responses_endpoint(), "expected {base_url} to be detected as Azure" ); } @@ -566,7 +459,8 @@ env_http_headers = { "X-Example-Env-Header" = "EXAMPLE_ENV_VAR" } stream_idle_timeout_ms: None, requires_openai_auth: false, }; - assert!(named_provider.is_azure_responses_endpoint()); + let named_api = named_provider.to_api_provider(None).expect("api provider"); + assert!(named_api.is_azure_responses_endpoint()); let negative_cases = [ "https://api.openai.com/v1", @@ -574,9 +468,24 @@ env_http_headers = { "X-Example-Env-Header" = "EXAMPLE_ENV_VAR" } "https://myproxy.azurewebsites.net/openai", ]; for base_url in negative_cases { - let provider = provider_for(base_url); + let provider = ModelProviderInfo { + name: "test".into(), + base_url: Some(base_url.into()), + env_key: None, + env_key_instructions: None, + experimental_bearer_token: None, + wire_api: WireApi::Responses, + query_params: None, + http_headers: None, + env_http_headers: None, + request_max_retries: None, + stream_max_retries: None, + stream_idle_timeout_ms: None, + requires_openai_auth: false, + }; + let api = provider.to_api_provider(None).expect("api provider"); assert!( - !provider.is_azure_responses_endpoint(), + !api.is_azure_responses_endpoint(), "expected {base_url} not to be detected as Azure" ); } diff --git a/codex-rs/otel/src/otel_event_manager.rs b/codex-rs/otel/src/otel_event_manager.rs index fde351cd69..b6bc07e79f 100644 --- a/codex-rs/otel/src/otel_event_manager.rs +++ b/codex-rs/otel/src/otel_event_manager.rs @@ -131,7 +131,18 @@ impl OtelEventManager { Ok(response) => (Some(response.status().as_u16()), None), Err(error) => (error.status().map(|s| s.as_u16()), Some(error.to_string())), }; + self.record_api_request(attempt, status, error.as_deref(), duration); + response + } + + pub fn record_api_request( + &self, + attempt: u64, + status: Option, + error: Option<&str>, + duration: Duration, + ) { tracing::event!( tracing::Level::INFO, event.name = "codex.api_request", @@ -149,8 +160,6 @@ impl OtelEventManager { error.message = error, attempt = attempt, ); - - response } pub fn log_sse_event( From 865e225bdea6d19836a79e6d64010d94b62e7379 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Tue, 25 Nov 2025 18:15:19 +0000 Subject: [PATCH 35/61] tmp: drop flaky ubuntu (#7300) --- .github/workflows/shell-tool-mcp.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/shell-tool-mcp.yml b/.github/workflows/shell-tool-mcp.yml index 6a0f68ff0e..63ce308d10 100644 --- a/.github/workflows/shell-tool-mcp.yml +++ b/.github/workflows/shell-tool-mcp.yml @@ -138,10 +138,6 @@ jobs: target: x86_64-unknown-linux-musl variant: ubuntu-22.04 image: ubuntu:22.04 - - runner: ubuntu-24.04 - target: x86_64-unknown-linux-musl - variant: ubuntu-20.04 - image: ubuntu:20.04 - runner: ubuntu-24.04 target: x86_64-unknown-linux-musl variant: debian-12 From 401f94ca313af00719e7815a5277c45269abc963 Mon Sep 17 00:00:00 2001 From: Celia Chen Date: Tue, 25 Nov 2025 11:56:04 -0800 Subject: [PATCH 36/61] [app-server] add `thread/tokenUsage/updated` v2 event (#7268) the TokenEvent event message becomes `thread/tokenUsage/updated` in v2. before & after: ``` < { < "method": "codex/event/token_count", < "params": { < "conversationId": "019ab891-4c55-7790-9670-6c3b48c33281", < "id": "1", < "msg": { < "info": { < "last_token_usage": { < "cached_input_tokens": 3072, < "input_tokens": 5152, < "output_tokens": 16, < "reasoning_output_tokens": 0, < "total_tokens": 5168 < }, < "model_context_window": 258400, < "total_token_usage": { < "cached_input_tokens": 3072, < "input_tokens": 5152, < "output_tokens": 16, < "reasoning_output_tokens": 0, < "total_tokens": 5168 < } < }, < "rate_limits": { < "credits": null, < "primary": null, < "secondary": null < }, < "type": "token_count" < } < } < } < { < "method": "thread/tokenUsage/updated", < "params": { < "threadId": "019ab891-4c55-7790-9670-6c3b48c33281", < "tokenUsage": { < "last": { < "cachedInputTokens": 3072, < "inputTokens": 5152, < "outputTokens": 16, < "reasoningOutputTokens": 0, < "totalTokens": 5168 < }, < "modelContextWindow": 258400, < "total": { < "cachedInputTokens": 3072, < "inputTokens": 5152, < "outputTokens": 16, < "reasoningOutputTokens": 0, < "totalTokens": 5168 < } < }, < "turnId": "1" < } < } ``` --- .../src/protocol/common.rs | 1 + .../app-server-protocol/src/protocol/v2.rs | 59 ++++ codex-rs/app-server/README.md | 2 +- .../app-server/src/bespoke_event_handling.rs | 289 ++++++++++++++---- 4 files changed, 287 insertions(+), 64 deletions(-) diff --git a/codex-rs/app-server-protocol/src/protocol/common.rs b/codex-rs/app-server-protocol/src/protocol/common.rs index 885cfae26c..bc7141f008 100644 --- a/codex-rs/app-server-protocol/src/protocol/common.rs +++ b/codex-rs/app-server-protocol/src/protocol/common.rs @@ -502,6 +502,7 @@ server_notification_definitions! { /// NEW NOTIFICATIONS Error => "error" (v2::ErrorNotification), ThreadStarted => "thread/started" (v2::ThreadStartedNotification), + ThreadTokenUsageUpdated => "thread/tokenUsage/updated" (v2::ThreadTokenUsageUpdatedNotification), TurnStarted => "turn/started" (v2::TurnStartedNotification), TurnCompleted => "turn/completed" (v2::TurnCompletedNotification), TurnDiffUpdated => "turn/diff/updated" (v2::TurnDiffUpdatedNotification), diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index 8c8e71a2ca..3ee502ebd5 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -16,6 +16,8 @@ use codex_protocol::protocol::CreditsSnapshot as CoreCreditsSnapshot; use codex_protocol::protocol::RateLimitSnapshot as CoreRateLimitSnapshot; use codex_protocol::protocol::RateLimitWindow as CoreRateLimitWindow; use codex_protocol::protocol::SessionSource as CoreSessionSource; +use codex_protocol::protocol::TokenUsage as CoreTokenUsage; +use codex_protocol::protocol::TokenUsageInfo as CoreTokenUsageInfo; use codex_protocol::user_input::UserInput as CoreUserInput; use mcp_types::ContentBlock as McpContentBlock; use schemars::JsonSchema; @@ -780,6 +782,63 @@ pub struct AccountUpdatedNotification { pub auth_mode: Option, } +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadTokenUsageUpdatedNotification { + pub thread_id: String, + pub turn_id: String, + pub token_usage: ThreadTokenUsage, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadTokenUsage { + pub total: TokenUsageBreakdown, + pub last: TokenUsageBreakdown, + #[ts(type = "number | null")] + pub model_context_window: Option, +} + +impl From for ThreadTokenUsage { + fn from(value: CoreTokenUsageInfo) -> Self { + Self { + total: value.total_token_usage.into(), + last: value.last_token_usage.into(), + model_context_window: value.model_context_window, + } + } +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct TokenUsageBreakdown { + #[ts(type = "number")] + pub total_tokens: i64, + #[ts(type = "number")] + pub input_tokens: i64, + #[ts(type = "number")] + pub cached_input_tokens: i64, + #[ts(type = "number")] + pub output_tokens: i64, + #[ts(type = "number")] + pub reasoning_output_tokens: i64, +} + +impl From for TokenUsageBreakdown { + fn from(value: CoreTokenUsage) -> Self { + Self { + total_tokens: value.total_tokens, + input_tokens: value.input_tokens, + cached_input_tokens: value.cached_input_tokens, + output_tokens: value.output_tokens, + reasoning_output_tokens: value.reasoning_output_tokens, + } + } +} + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index fa0a4baaa0..176cd27339 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -240,7 +240,7 @@ Event notifications are the server-initiated event stream for thread lifecycles, ### Turn events -The app-server streams JSON-RPC notifications while a turn is running. Each turn starts with `turn/started` (initial `turn`) and ends with `turn/completed` (final `turn` plus token `usage`), and clients subscribe to the events they care about, rendering each item incrementally as updates arrive. The per-item lifecycle is always: `item/started` → zero or more item-specific deltas → `item/completed`. +The app-server streams JSON-RPC notifications while a turn is running. Each turn starts with `turn/started` (initial `turn`) and ends with `turn/completed` (final `turn` status). Token usage events stream separately via `thread/tokenUsage/updated`. Clients subscribe to the events they care about, rendering each item incrementally as updates arrive. The per-item lifecycle is always: `item/started` → zero or more item-specific deltas → `item/completed`. - `turn/started` — `{ turn }` with the turn id, empty `items`, and `status: "inProgress"`. - `turn/completed` — `{ turn }` where `turn.status` is `completed`, `interrupted`, or `failed`; failures carry `{ error: { message, codexErrorInfo? } }`. diff --git a/codex-rs/app-server/src/bespoke_event_handling.rs b/codex-rs/app-server/src/bespoke_event_handling.rs index d5037358b3..4b4e40d0c6 100644 --- a/codex-rs/app-server/src/bespoke_event_handling.rs +++ b/codex-rs/app-server/src/bespoke_event_handling.rs @@ -36,6 +36,8 @@ use codex_app_server_protocol::SandboxCommandAssessment as V2SandboxCommandAsses use codex_app_server_protocol::ServerNotification; use codex_app_server_protocol::ServerRequestPayload; use codex_app_server_protocol::ThreadItem; +use codex_app_server_protocol::ThreadTokenUsage; +use codex_app_server_protocol::ThreadTokenUsageUpdatedNotification; use codex_app_server_protocol::Turn; use codex_app_server_protocol::TurnCompletedNotification; use codex_app_server_protocol::TurnDiffUpdatedNotification; @@ -54,6 +56,7 @@ use codex_core::protocol::McpToolCallBeginEvent; use codex_core::protocol::McpToolCallEndEvent; use codex_core::protocol::Op; use codex_core::protocol::ReviewDecision; +use codex_core::protocol::TokenCountEvent; use codex_core::protocol::TurnDiffEvent; use codex_core::review_format::format_review_findings_block; use codex_protocol::ConversationId; @@ -76,10 +79,19 @@ pub(crate) async fn apply_bespoke_event_handling( turn_summary_store: TurnSummaryStore, api_version: ApiVersion, ) { - let Event { id: event_id, msg } = event; + let Event { + id: event_turn_id, + msg, + } = event; match msg { EventMsg::TaskComplete(_ev) => { - handle_turn_complete(conversation_id, event_id, &outgoing, &turn_summary_store).await; + handle_turn_complete( + conversation_id, + event_turn_id, + &outgoing, + &turn_summary_store, + ) + .await; } EventMsg::ApplyPatchApprovalRequest(ApplyPatchApprovalRequestEvent { call_id, @@ -100,7 +112,7 @@ pub(crate) async fn apply_bespoke_event_handling( .send_request(ServerRequestPayload::ApplyPatchApproval(params)) .await; tokio::spawn(async move { - on_patch_approval_response(event_id, rx, conversation).await; + on_patch_approval_response(event_turn_id, rx, conversation).await; }); } ApiVersion::V2 => { @@ -122,7 +134,7 @@ pub(crate) async fn apply_bespoke_event_handling( }; let notification = ItemStartedNotification { thread_id: conversation_id.to_string(), - turn_id: event_id.clone(), + turn_id: event_turn_id.clone(), item, }; outgoing @@ -142,7 +154,7 @@ pub(crate) async fn apply_bespoke_event_handling( .await; tokio::spawn(async move { on_file_change_request_approval_response( - event_id, + event_turn_id, conversation_id, item_id, patch_changes, @@ -178,7 +190,7 @@ pub(crate) async fn apply_bespoke_event_handling( .send_request(ServerRequestPayload::ExecCommandApproval(params)) .await; tokio::spawn(async move { - on_exec_approval_response(event_id, rx, conversation).await; + on_exec_approval_response(event_turn_id, rx, conversation).await; }); } ApiVersion::V2 => { @@ -206,7 +218,7 @@ pub(crate) async fn apply_bespoke_event_handling( .await; tokio::spawn(async move { on_command_execution_request_approval_response( - event_id, + event_turn_id, conversation_id, item_id, command_string, @@ -225,7 +237,7 @@ pub(crate) async fn apply_bespoke_event_handling( let notification = construct_mcp_tool_call_notification( begin_event, conversation_id.to_string(), - event_id.clone(), + event_turn_id.clone(), ) .await; outgoing @@ -236,7 +248,7 @@ pub(crate) async fn apply_bespoke_event_handling( let notification = construct_mcp_tool_call_end_notification( end_event, conversation_id.to_string(), - event_id.clone(), + event_turn_id.clone(), ) .await; outgoing @@ -255,7 +267,7 @@ pub(crate) async fn apply_bespoke_event_handling( EventMsg::ContextCompacted(..) => { let notification = ContextCompactedNotification { thread_id: conversation_id.to_string(), - turn_id: event_id.clone(), + turn_id: event_turn_id.clone(), }; outgoing .send_server_notification(ServerNotification::ContextCompacted(notification)) @@ -295,15 +307,8 @@ pub(crate) async fn apply_bespoke_event_handling( .await; } EventMsg::TokenCount(token_count_event) => { - if let Some(rate_limits) = token_count_event.rate_limits { - outgoing - .send_server_notification(ServerNotification::AccountRateLimitsUpdated( - AccountRateLimitsUpdatedNotification { - rate_limits: rate_limits.into(), - }, - )) - .await; - } + handle_token_count_event(conversation_id, event_turn_id, token_count_event, &outgoing) + .await; } EventMsg::Error(ev) => { let turn_error = TurnError { @@ -315,7 +320,7 @@ pub(crate) async fn apply_bespoke_event_handling( .send_server_notification(ServerNotification::Error(ErrorNotification { error: turn_error, thread_id: conversation_id.to_string(), - turn_id: event_id.clone(), + turn_id: event_turn_id.clone(), })) .await; } @@ -330,16 +335,16 @@ pub(crate) async fn apply_bespoke_event_handling( .send_server_notification(ServerNotification::Error(ErrorNotification { error: turn_error, thread_id: conversation_id.to_string(), - turn_id: event_id.clone(), + turn_id: event_turn_id.clone(), })) .await; } EventMsg::EnteredReviewMode(review_request) => { let notification = ItemStartedNotification { thread_id: conversation_id.to_string(), - turn_id: event_id.clone(), + turn_id: event_turn_id.clone(), item: ThreadItem::CodeReview { - id: event_id.clone(), + id: event_turn_id.clone(), review: review_request.user_facing_hint, }, }; @@ -351,7 +356,7 @@ pub(crate) async fn apply_bespoke_event_handling( let item: ThreadItem = item_started_event.item.clone().into(); let notification = ItemStartedNotification { thread_id: conversation_id.to_string(), - turn_id: event_id.clone(), + turn_id: event_turn_id.clone(), item, }; outgoing @@ -362,7 +367,7 @@ pub(crate) async fn apply_bespoke_event_handling( let item: ThreadItem = item_completed_event.item.clone().into(); let notification = ItemCompletedNotification { thread_id: conversation_id.to_string(), - turn_id: event_id.clone(), + turn_id: event_turn_id.clone(), item, }; outgoing @@ -374,10 +379,10 @@ pub(crate) async fn apply_bespoke_event_handling( Some(output) => render_review_output_text(&output), None => REVIEW_FALLBACK_MESSAGE.to_string(), }; - let review_item_id = event_id.clone(); + let review_item_id = event_turn_id.clone(); let notification = ItemCompletedNotification { thread_id: conversation_id.to_string(), - turn_id: event_id.clone(), + turn_id: event_turn_id.clone(), item: ThreadItem::CodeReview { id: review_item_id, review: review_text, @@ -405,7 +410,7 @@ pub(crate) async fn apply_bespoke_event_handling( }; let notification = ItemStartedNotification { thread_id: conversation_id.to_string(), - turn_id: event_id.clone(), + turn_id: event_turn_id.clone(), item, }; outgoing @@ -429,7 +434,7 @@ pub(crate) async fn apply_bespoke_event_handling( item_id, changes, status, - event_id.clone(), + event_turn_id.clone(), outgoing.as_ref(), &turn_summary_store, ) @@ -457,7 +462,7 @@ pub(crate) async fn apply_bespoke_event_handling( }; let notification = ItemStartedNotification { thread_id: conversation_id.to_string(), - turn_id: event_id.clone(), + turn_id: event_turn_id.clone(), item, }; outgoing @@ -518,7 +523,7 @@ pub(crate) async fn apply_bespoke_event_handling( let notification = ItemCompletedNotification { thread_id: conversation_id.to_string(), - turn_id: event_id.clone(), + turn_id: event_turn_id.clone(), item, }; outgoing @@ -548,11 +553,22 @@ pub(crate) async fn apply_bespoke_event_handling( } } - handle_turn_interrupted(conversation_id, event_id, &outgoing, &turn_summary_store) - .await; + handle_turn_interrupted( + conversation_id, + event_turn_id, + &outgoing, + &turn_summary_store, + ) + .await; } EventMsg::TurnDiff(turn_diff_event) => { - handle_turn_diff(&event_id, turn_diff_event, api_version, outgoing.as_ref()).await; + handle_turn_diff( + &event_turn_id, + turn_diff_event, + api_version, + outgoing.as_ref(), + ) + .await; } _ => {} @@ -560,14 +576,14 @@ pub(crate) async fn apply_bespoke_event_handling( } async fn handle_turn_diff( - event_id: &str, + event_turn_id: &str, turn_diff_event: TurnDiffEvent, api_version: ApiVersion, outgoing: &OutgoingMessageSender, ) { if let ApiVersion::V2 = api_version { let notification = TurnDiffUpdatedNotification { - turn_id: event_id.to_string(), + turn_id: event_turn_id.to_string(), diff: turn_diff_event.unified_diff, }; outgoing @@ -578,14 +594,14 @@ async fn handle_turn_diff( async fn emit_turn_completed_with_status( conversation_id: ConversationId, - event_id: String, + event_turn_id: String, status: TurnStatus, outgoing: &OutgoingMessageSender, ) { let notification = TurnCompletedNotification { thread_id: conversation_id.to_string(), turn: Turn { - id: event_id, + id: event_turn_id, items: vec![], status, }, @@ -667,7 +683,7 @@ async fn find_and_remove_turn_summary( async fn handle_turn_complete( conversation_id: ConversationId, - event_id: String, + event_turn_id: String, outgoing: &OutgoingMessageSender, turn_summary_store: &TurnSummaryStore, ) { @@ -679,19 +695,52 @@ async fn handle_turn_complete( TurnStatus::Completed }; - emit_turn_completed_with_status(conversation_id, event_id, status, outgoing).await; + emit_turn_completed_with_status(conversation_id, event_turn_id, status, outgoing).await; } async fn handle_turn_interrupted( conversation_id: ConversationId, - event_id: String, + event_turn_id: String, outgoing: &OutgoingMessageSender, turn_summary_store: &TurnSummaryStore, ) { find_and_remove_turn_summary(conversation_id, turn_summary_store).await; - emit_turn_completed_with_status(conversation_id, event_id, TurnStatus::Interrupted, outgoing) - .await; + emit_turn_completed_with_status( + conversation_id, + event_turn_id, + TurnStatus::Interrupted, + outgoing, + ) + .await; +} + +async fn handle_token_count_event( + conversation_id: ConversationId, + turn_id: String, + token_count_event: TokenCountEvent, + outgoing: &OutgoingMessageSender, +) { + let TokenCountEvent { info, rate_limits } = token_count_event; + if let Some(token_usage) = info.map(ThreadTokenUsage::from) { + let notification = ThreadTokenUsageUpdatedNotification { + thread_id: conversation_id.to_string(), + turn_id, + token_usage, + }; + outgoing + .send_server_notification(ServerNotification::ThreadTokenUsageUpdated(notification)) + .await; + } + if let Some(rate_limits) = rate_limits { + outgoing + .send_server_notification(ServerNotification::AccountRateLimitsUpdated( + AccountRateLimitsUpdatedNotification { + rate_limits: rate_limits.into(), + }, + )) + .await; + } } async fn handle_error( @@ -704,7 +753,7 @@ async fn handle_error( } async fn on_patch_approval_response( - event_id: String, + event_turn_id: String, receiver: oneshot::Receiver, codex: Arc, ) { @@ -715,7 +764,7 @@ async fn on_patch_approval_response( error!("request failed: {err:?}"); if let Err(submit_err) = codex .submit(Op::PatchApproval { - id: event_id.clone(), + id: event_turn_id.clone(), decision: ReviewDecision::Denied, }) .await @@ -736,7 +785,7 @@ async fn on_patch_approval_response( if let Err(err) = codex .submit(Op::PatchApproval { - id: event_id, + id: event_turn_id, decision: response.decision, }) .await @@ -746,7 +795,7 @@ async fn on_patch_approval_response( } async fn on_exec_approval_response( - event_id: String, + event_turn_id: String, receiver: oneshot::Receiver, conversation: Arc, ) { @@ -772,7 +821,7 @@ async fn on_exec_approval_response( if let Err(err) = conversation .submit(Op::ExecApproval { - id: event_id, + id: event_turn_id, decision: response.decision, }) .await @@ -845,7 +894,7 @@ fn format_file_change_diff(change: &CoreFileChange) -> String { #[allow(clippy::too_many_arguments)] async fn on_file_change_request_approval_response( - event_id: String, + event_turn_id: String, conversation_id: ConversationId, item_id: String, changes: Vec, @@ -890,7 +939,7 @@ async fn on_file_change_request_approval_response( item_id, changes, status, - event_id.clone(), + event_turn_id.clone(), outgoing.as_ref(), &turn_summary_store, ) @@ -899,7 +948,7 @@ async fn on_file_change_request_approval_response( if let Err(err) = codex .submit(Op::PatchApproval { - id: event_id, + id: event_turn_id, decision, }) .await @@ -910,7 +959,7 @@ async fn on_file_change_request_approval_response( #[allow(clippy::too_many_arguments)] async fn on_command_execution_request_approval_response( - event_id: String, + event_turn_id: String, conversation_id: ConversationId, item_id: String, command: String, @@ -962,7 +1011,7 @@ async fn on_command_execution_request_approval_response( if let Some(status) = completion_status { complete_command_execution_item( conversation_id, - event_id.clone(), + event_turn_id.clone(), item_id.clone(), command.clone(), cwd.clone(), @@ -975,7 +1024,7 @@ async fn on_command_execution_request_approval_response( if let Err(err) = conversation .submit(Op::ExecApproval { - id: event_id, + id: event_turn_id, decision, }) .await @@ -1059,7 +1108,12 @@ mod tests { use anyhow::Result; use anyhow::anyhow; use anyhow::bail; + use codex_core::protocol::CreditsSnapshot; use codex_core::protocol::McpInvocation; + use codex_core::protocol::RateLimitSnapshot; + use codex_core::protocol::RateLimitWindow; + use codex_core::protocol::TokenUsage; + use codex_core::protocol::TokenUsageInfo; use mcp_types::CallToolResult; use mcp_types::ContentBlock; use mcp_types::TextContent; @@ -1103,14 +1157,14 @@ mod tests { #[tokio::test] async fn test_handle_turn_complete_emits_completed_without_error() -> Result<()> { let conversation_id = ConversationId::new(); - let event_id = "complete1".to_string(); + let event_turn_id = "complete1".to_string(); let (tx, mut rx) = mpsc::channel(CHANNEL_CAPACITY); let outgoing = Arc::new(OutgoingMessageSender::new(tx)); let turn_summary_store = new_turn_summary_store(); handle_turn_complete( conversation_id, - event_id.clone(), + event_turn_id.clone(), &outgoing, &turn_summary_store, ) @@ -1122,7 +1176,7 @@ mod tests { .ok_or_else(|| anyhow!("should send one notification"))?; match msg { OutgoingMessage::AppServerNotification(ServerNotification::TurnCompleted(n)) => { - assert_eq!(n.turn.id, event_id); + assert_eq!(n.turn.id, event_turn_id); assert_eq!(n.turn.status, TurnStatus::Completed); } other => bail!("unexpected message: {other:?}"), @@ -1134,7 +1188,7 @@ mod tests { #[tokio::test] async fn test_handle_turn_interrupted_emits_interrupted_with_error() -> Result<()> { let conversation_id = ConversationId::new(); - let event_id = "interrupt1".to_string(); + let event_turn_id = "interrupt1".to_string(); let turn_summary_store = new_turn_summary_store(); handle_error( conversation_id, @@ -1150,7 +1204,7 @@ mod tests { handle_turn_interrupted( conversation_id, - event_id.clone(), + event_turn_id.clone(), &outgoing, &turn_summary_store, ) @@ -1162,7 +1216,7 @@ mod tests { .ok_or_else(|| anyhow!("should send one notification"))?; match msg { OutgoingMessage::AppServerNotification(ServerNotification::TurnCompleted(n)) => { - assert_eq!(n.turn.id, event_id); + assert_eq!(n.turn.id, event_turn_id); assert_eq!(n.turn.status, TurnStatus::Interrupted); } other => bail!("unexpected message: {other:?}"), @@ -1174,7 +1228,7 @@ mod tests { #[tokio::test] async fn test_handle_turn_complete_emits_failed_with_error() -> Result<()> { let conversation_id = ConversationId::new(); - let event_id = "complete_err1".to_string(); + let event_turn_id = "complete_err1".to_string(); let turn_summary_store = new_turn_summary_store(); handle_error( conversation_id, @@ -1190,7 +1244,7 @@ mod tests { handle_turn_complete( conversation_id, - event_id.clone(), + event_turn_id.clone(), &outgoing, &turn_summary_store, ) @@ -1202,7 +1256,7 @@ mod tests { .ok_or_else(|| anyhow!("should send one notification"))?; match msg { OutgoingMessage::AppServerNotification(ServerNotification::TurnCompleted(n)) => { - assert_eq!(n.turn.id, event_id); + assert_eq!(n.turn.id, event_turn_id); assert_eq!( n.turn.status, TurnStatus::Failed { @@ -1219,6 +1273,115 @@ mod tests { Ok(()) } + #[tokio::test] + async fn test_handle_token_count_event_emits_usage_and_rate_limits() -> Result<()> { + let conversation_id = ConversationId::new(); + let turn_id = "turn-123".to_string(); + let (tx, mut rx) = mpsc::channel(CHANNEL_CAPACITY); + let outgoing = Arc::new(OutgoingMessageSender::new(tx)); + + let info = TokenUsageInfo { + total_token_usage: TokenUsage { + input_tokens: 100, + cached_input_tokens: 25, + output_tokens: 50, + reasoning_output_tokens: 9, + total_tokens: 200, + }, + last_token_usage: TokenUsage { + input_tokens: 10, + cached_input_tokens: 5, + output_tokens: 7, + reasoning_output_tokens: 1, + total_tokens: 23, + }, + model_context_window: Some(4096), + }; + let rate_limits = RateLimitSnapshot { + primary: Some(RateLimitWindow { + used_percent: 42.5, + window_minutes: Some(15), + resets_at: Some(1700000000), + }), + secondary: None, + credits: Some(CreditsSnapshot { + has_credits: true, + unlimited: false, + balance: Some("5".to_string()), + }), + }; + + handle_token_count_event( + conversation_id, + turn_id.clone(), + TokenCountEvent { + info: Some(info), + rate_limits: Some(rate_limits), + }, + &outgoing, + ) + .await; + + let first = rx + .recv() + .await + .ok_or_else(|| anyhow!("expected usage notification"))?; + match first { + OutgoingMessage::AppServerNotification( + ServerNotification::ThreadTokenUsageUpdated(payload), + ) => { + assert_eq!(payload.thread_id, conversation_id.to_string()); + assert_eq!(payload.turn_id, turn_id); + let usage = payload.token_usage; + assert_eq!(usage.total.total_tokens, 200); + assert_eq!(usage.total.cached_input_tokens, 25); + assert_eq!(usage.last.output_tokens, 7); + assert_eq!(usage.model_context_window, Some(4096)); + } + other => bail!("unexpected notification: {other:?}"), + } + + let second = rx + .recv() + .await + .ok_or_else(|| anyhow!("expected rate limit notification"))?; + match second { + OutgoingMessage::AppServerNotification( + ServerNotification::AccountRateLimitsUpdated(payload), + ) => { + assert!(payload.rate_limits.primary.is_some()); + assert!(payload.rate_limits.credits.is_some()); + } + other => bail!("unexpected notification: {other:?}"), + } + Ok(()) + } + + #[tokio::test] + async fn test_handle_token_count_event_without_usage_info() -> Result<()> { + let conversation_id = ConversationId::new(); + let turn_id = "turn-456".to_string(); + let (tx, mut rx) = mpsc::channel(CHANNEL_CAPACITY); + let outgoing = Arc::new(OutgoingMessageSender::new(tx)); + + handle_token_count_event( + conversation_id, + turn_id.clone(), + TokenCountEvent { + info: None, + rate_limits: None, + }, + &outgoing, + ) + .await; + + assert!( + rx.try_recv().is_err(), + "no notifications should be emitted when token usage info is absent" + ); + Ok(()) + } + #[tokio::test] async fn test_construct_mcp_tool_call_begin_notification_with_args() { let begin_event = McpToolCallBeginEvent { From 981e2f742d6102b714ac7bd24505e1fb1d65fff9 Mon Sep 17 00:00:00 2001 From: iceweasel-oai Date: Tue, 25 Nov 2025 12:41:00 -0800 Subject: [PATCH 37/61] correctly recognize WorkspaceWrite policy on /approvals (#7301) the `/approvals` popup fails to recognize that the CLI is in WorkspaceWrite mode if that policy has extra bits, like `writable_roots` etc. This change matches the policy, ignoring additional config aspects. --- codex-rs/tui/src/chatwidget.rs | 24 +++++++++++++++++++++++- codex-rs/tui/src/chatwidget/tests.rs | 23 +++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 53c77f82be..fbabbc300e 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -2332,7 +2332,7 @@ impl ChatWidget { let presets: Vec = builtin_approval_presets(); for preset in presets.into_iter() { let is_current = - current_approval == preset.approval && current_sandbox == preset.sandbox; + Self::preset_matches_current(current_approval, ¤t_sandbox, &preset); let name = preset.label.to_string(); let description_text = preset.description; let description = Some(description_text.to_string()); @@ -2420,6 +2420,28 @@ impl ChatWidget { })] } + fn preset_matches_current( + current_approval: AskForApproval, + current_sandbox: &SandboxPolicy, + preset: &ApprovalPreset, + ) -> bool { + if current_approval != preset.approval { + return false; + } + matches!( + (&preset.sandbox, current_sandbox), + (SandboxPolicy::ReadOnly, SandboxPolicy::ReadOnly) + | ( + SandboxPolicy::DangerFullAccess, + SandboxPolicy::DangerFullAccess + ) + | ( + SandboxPolicy::WorkspaceWrite { .. }, + SandboxPolicy::WorkspaceWrite { .. } + ) + ) + } + #[cfg(target_os = "windows")] pub(crate) fn world_writable_warning_details(&self) -> Option<(Vec, usize, bool)> { if self diff --git a/codex-rs/tui/src/chatwidget/tests.rs b/codex-rs/tui/src/chatwidget/tests.rs index 7b044e8dfb..fde52a0646 100644 --- a/codex-rs/tui/src/chatwidget/tests.rs +++ b/codex-rs/tui/src/chatwidget/tests.rs @@ -1567,6 +1567,29 @@ fn approvals_selection_popup_snapshot() { assert_snapshot!("approvals_selection_popup", popup); } +#[test] +fn preset_matching_ignores_extra_writable_roots() { + let preset = builtin_approval_presets() + .into_iter() + .find(|p| p.id == "auto") + .expect("auto preset exists"); + let current_sandbox = SandboxPolicy::WorkspaceWrite { + writable_roots: vec![PathBuf::from("C:\\extra")], + network_access: false, + exclude_tmpdir_env_var: false, + exclude_slash_tmp: false, + }; + + assert!( + ChatWidget::preset_matches_current(AskForApproval::OnRequest, ¤t_sandbox, &preset), + "WorkspaceWrite with extra roots should still match the Agent preset" + ); + assert!( + !ChatWidget::preset_matches_current(AskForApproval::Never, ¤t_sandbox, &preset), + "approval mismatch should prevent matching the preset" + ); +} + #[test] fn full_access_confirmation_popup_snapshot() { let (mut chat, _rx, _op_rx) = make_chatwidget_manual(); From 28ff364c3adf938c60fd73c5446e836762111186 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Tue, 25 Nov 2025 22:21:05 +0000 Subject: [PATCH 38/61] feat: update process ID for event handling (#7261) --- codex-rs/Cargo.lock | 1 + .../app-server-protocol/src/protocol/v2.rs | 2 + .../app-server/src/bespoke_event_handling.rs | 7 + codex-rs/app-server/tests/common/lib.rs | 1 + codex-rs/app-server/tests/common/responses.rs | 39 ++++ .../app-server/tests/suite/v2/turn_start.rs | 129 +++++++++++ codex-rs/core/Cargo.toml | 4 + codex-rs/core/src/tasks/user_shell.rs | 4 + codex-rs/core/src/tools/events.rs | 14 +- .../core/src/tools/handlers/unified_exec.rs | 11 +- codex-rs/core/src/unified_exec/errors.rs | 5 +- codex-rs/core/src/unified_exec/mod.rs | 89 +++++--- .../core/src/unified_exec/session_manager.rs | 213 +++++++++++------- codex-rs/core/tests/suite/unified_exec.rs | 99 ++++---- .../tests/event_processor_with_json_output.rs | 5 + codex-rs/protocol/src/protocol.rs | 8 + codex-rs/tui/src/chatwidget/tests.rs | 5 + 17 files changed, 473 insertions(+), 163 deletions(-) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 34ea8c8773..4c87b8dc05 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1144,6 +1144,7 @@ dependencies = [ "codex-apply-patch", "codex-arg0", "codex-async-utils", + "codex-core", "codex-execpolicy", "codex-file-search", "codex-git", diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index 3ee502ebd5..222f3e4efc 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -1020,6 +1020,8 @@ pub enum ThreadItem { command: String, /// The command's working directory. cwd: PathBuf, + /// Identifier for the underlying PTY process (when available). + process_id: Option, status: CommandExecutionStatus, /// A best-effort parsing of the command to understand the action(s) it will perform. /// This returns a list of CommandAction objects because a single shell command may diff --git a/codex-rs/app-server/src/bespoke_event_handling.rs b/codex-rs/app-server/src/bespoke_event_handling.rs index 4b4e40d0c6..24c74ebe43 100644 --- a/codex-rs/app-server/src/bespoke_event_handling.rs +++ b/codex-rs/app-server/src/bespoke_event_handling.rs @@ -449,11 +449,13 @@ pub(crate) async fn apply_bespoke_event_handling( .collect::>(); let command = shlex_join(&exec_command_begin_event.command); let cwd = exec_command_begin_event.cwd; + let process_id = exec_command_begin_event.process_id; let item = ThreadItem::CommandExecution { id: item_id, command, cwd, + process_id, status: CommandExecutionStatus::InProgress, command_actions, aggregated_output: None, @@ -486,6 +488,7 @@ pub(crate) async fn apply_bespoke_event_handling( command, cwd, parsed_cmd, + process_id, aggregated_output, exit_code, duration, @@ -514,6 +517,7 @@ pub(crate) async fn apply_bespoke_event_handling( id: call_id, command: shlex_join(&command), cwd, + process_id, status, command_actions, aggregated_output, @@ -649,6 +653,7 @@ async fn complete_command_execution_item( item_id: String, command: String, cwd: PathBuf, + process_id: Option, command_actions: Vec, status: CommandExecutionStatus, outgoing: &OutgoingMessageSender, @@ -657,6 +662,7 @@ async fn complete_command_execution_item( id: item_id, command, cwd, + process_id, status, command_actions, aggregated_output: None, @@ -1015,6 +1021,7 @@ async fn on_command_execution_request_approval_response( item_id.clone(), command.clone(), cwd.clone(), + None, command_actions.clone(), status, outgoing.as_ref(), diff --git a/codex-rs/app-server/tests/common/lib.rs b/codex-rs/app-server/tests/common/lib.rs index 7d3eb53886..6fd54a66dc 100644 --- a/codex-rs/app-server/tests/common/lib.rs +++ b/codex-rs/app-server/tests/common/lib.rs @@ -15,6 +15,7 @@ pub use mcp_process::McpProcess; pub use mock_model_server::create_mock_chat_completions_server; pub use mock_model_server::create_mock_chat_completions_server_unchecked; pub use responses::create_apply_patch_sse_response; +pub use responses::create_exec_command_sse_response; pub use responses::create_final_assistant_message_sse_response; pub use responses::create_shell_command_sse_response; pub use rollout::create_fake_rollout; diff --git a/codex-rs/app-server/tests/common/responses.rs b/codex-rs/app-server/tests/common/responses.rs index 0a9183c043..d3d1f40cdc 100644 --- a/codex-rs/app-server/tests/common/responses.rs +++ b/codex-rs/app-server/tests/common/responses.rs @@ -94,3 +94,42 @@ pub fn create_apply_patch_sse_response( ); Ok(sse) } + +pub fn create_exec_command_sse_response(call_id: &str) -> anyhow::Result { + let (cmd, args) = if cfg!(windows) { + ("cmd.exe", vec!["/d", "/c", "echo hi"]) + } else { + ("/bin/sh", vec!["-c", "echo hi"]) + }; + let command = std::iter::once(cmd.to_string()) + .chain(args.into_iter().map(str::to_string)) + .collect::>(); + let tool_call_arguments = serde_json::to_string(&json!({ + "cmd": command.join(" "), + "yield_time_ms": 500 + }))?; + let tool_call = json!({ + "choices": [ + { + "delta": { + "tool_calls": [ + { + "id": call_id, + "function": { + "name": "exec_command", + "arguments": tool_call_arguments + } + } + ] + }, + "finish_reason": "tool_calls" + } + ] + }); + + let sse = format!( + "data: {}\n\ndata: DONE\n\n", + serde_json::to_string(&tool_call)? + ); + Ok(sse) +} diff --git a/codex-rs/app-server/tests/suite/v2/turn_start.rs b/codex-rs/app-server/tests/suite/v2/turn_start.rs index 2c23fedd94..cd35e31adc 100644 --- a/codex-rs/app-server/tests/suite/v2/turn_start.rs +++ b/codex-rs/app-server/tests/suite/v2/turn_start.rs @@ -1,6 +1,7 @@ use anyhow::Result; use app_test_support::McpProcess; use app_test_support::create_apply_patch_sse_response; +use app_test_support::create_exec_command_sse_response; use app_test_support::create_final_assistant_message_sse_response; use app_test_support::create_mock_chat_completions_server; use app_test_support::create_mock_chat_completions_server_unchecked; @@ -907,6 +908,134 @@ async fn turn_start_file_change_approval_decline_v2() -> Result<()> { Ok(()) } +#[tokio::test] +#[cfg_attr(windows, ignore = "process id reporting differs on Windows")] +async fn command_execution_notifications_include_process_id() -> Result<()> { + skip_if_no_network!(Ok(())); + + let responses = vec![ + create_exec_command_sse_response("uexec-1")?, + create_final_assistant_message_sse_response("done")?, + ]; + let server = create_mock_chat_completions_server(responses).await; + let codex_home = TempDir::new()?; + create_config_toml(codex_home.path(), &server.uri(), "never")?; + let config_toml = codex_home.path().join("config.toml"); + let mut config_contents = std::fs::read_to_string(&config_toml)?; + config_contents.push_str( + r#" +[features] +unified_exec = true +"#, + ); + std::fs::write(&config_toml, config_contents)?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let start_id = mcp + .send_thread_start_request(ThreadStartParams { + model: Some("mock-model".to_string()), + ..Default::default() + }) + .await?; + let start_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(start_id)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response::(start_resp)?; + + let turn_id = mcp + .send_turn_start_request(TurnStartParams { + thread_id: thread.id.clone(), + input: vec![V2UserInput::Text { + text: "run a command".to_string(), + }], + ..Default::default() + }) + .await?; + let turn_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(turn_id)), + ) + .await??; + let TurnStartResponse { turn: _turn } = to_response::(turn_resp)?; + + let started_command = timeout(DEFAULT_READ_TIMEOUT, async { + loop { + let notif = mcp + .read_stream_until_notification_message("item/started") + .await?; + let started: ItemStartedNotification = serde_json::from_value( + notif + .params + .clone() + .expect("item/started should include params"), + )?; + if let ThreadItem::CommandExecution { .. } = started.item { + return Ok::(started.item); + } + } + }) + .await??; + let ThreadItem::CommandExecution { + id, + process_id: started_process_id, + status, + .. + } = started_command + else { + unreachable!("loop ensures we break on command execution items"); + }; + assert_eq!(id, "uexec-1"); + assert_eq!(status, CommandExecutionStatus::InProgress); + let started_process_id = started_process_id.expect("process id should be present"); + + let completed_command = timeout(DEFAULT_READ_TIMEOUT, async { + loop { + let notif = mcp + .read_stream_until_notification_message("item/completed") + .await?; + let completed: ItemCompletedNotification = serde_json::from_value( + notif + .params + .clone() + .expect("item/completed should include params"), + )?; + if let ThreadItem::CommandExecution { .. } = completed.item { + return Ok::(completed.item); + } + } + }) + .await??; + let ThreadItem::CommandExecution { + id: completed_id, + process_id: completed_process_id, + status: completed_status, + exit_code, + .. + } = completed_command + else { + unreachable!("loop ensures we break on command execution items"); + }; + assert_eq!(completed_id, "uexec-1"); + assert_eq!(completed_status, CommandExecutionStatus::Completed); + assert_eq!(exit_code, Some(0)); + assert_eq!( + completed_process_id.as_deref(), + Some(started_process_id.as_str()) + ); + + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("turn/completed"), + ) + .await??; + + Ok(()) +} + // Helper to create a config.toml pointing at the mock model server. fn create_config_toml( codex_home: &Path, diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index 705bab57df..7bd2d652e1 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -87,6 +87,9 @@ uuid = { workspace = true, features = ["serde", "v4", "v5"] } which = { workspace = true } wildmatch = { workspace = true } +[features] +deterministic_process_ids = [] + [target.'cfg(target_os = "linux")'.dependencies] landlock = { workspace = true } @@ -115,6 +118,7 @@ keyring = { workspace = true, features = ["sync-secret-service"] } assert_cmd = { workspace = true } assert_matches = { workspace = true } codex-arg0 = { workspace = true } +codex-core = { path = ".", features = ["deterministic_process_ids"] } core_test_support = { workspace = true } ctor = { workspace = true } escargot = { workspace = true } diff --git a/codex-rs/core/src/tasks/user_shell.rs b/codex-rs/core/src/tasks/user_shell.rs index 32e8a25963..ca5243241a 100644 --- a/codex-rs/core/src/tasks/user_shell.rs +++ b/codex-rs/core/src/tasks/user_shell.rs @@ -81,6 +81,7 @@ impl SessionTask for UserShellCommandTask { turn_context.as_ref(), EventMsg::ExecCommandBegin(ExecCommandBeginEvent { call_id: call_id.clone(), + process_id: None, turn_id: turn_context.sub_id.clone(), command: command.clone(), cwd: cwd.clone(), @@ -139,6 +140,7 @@ impl SessionTask for UserShellCommandTask { turn_context.as_ref(), EventMsg::ExecCommandEnd(ExecCommandEndEvent { call_id, + process_id: None, turn_id: turn_context.sub_id.clone(), command: command.clone(), cwd: cwd.clone(), @@ -161,6 +163,7 @@ impl SessionTask for UserShellCommandTask { turn_context.as_ref(), EventMsg::ExecCommandEnd(ExecCommandEndEvent { call_id: call_id.clone(), + process_id: None, turn_id: turn_context.sub_id.clone(), command: command.clone(), cwd: cwd.clone(), @@ -205,6 +208,7 @@ impl SessionTask for UserShellCommandTask { turn_context.as_ref(), EventMsg::ExecCommandEnd(ExecCommandEndEvent { call_id, + process_id: None, turn_id: turn_context.sub_id.clone(), command, cwd, diff --git a/codex-rs/core/src/tools/events.rs b/codex-rs/core/src/tools/events.rs index 37df12d4b0..93bce60489 100644 --- a/codex-rs/core/src/tools/events.rs +++ b/codex-rs/core/src/tools/events.rs @@ -65,12 +65,14 @@ pub(crate) async fn emit_exec_command_begin( parsed_cmd: &[ParsedCommand], source: ExecCommandSource, interaction_input: Option, + process_id: Option<&str>, ) { ctx.session .send_event( ctx.turn, EventMsg::ExecCommandBegin(ExecCommandBeginEvent { call_id: ctx.call_id.to_string(), + process_id: process_id.map(str::to_owned), turn_id: ctx.turn.sub_id.clone(), command: command.to_vec(), cwd: cwd.to_path_buf(), @@ -100,6 +102,7 @@ pub(crate) enum ToolEmitter { source: ExecCommandSource, interaction_input: Option, parsed_cmd: Vec, + process_id: Option, }, } @@ -132,6 +135,7 @@ impl ToolEmitter { cwd: PathBuf, source: ExecCommandSource, interaction_input: Option, + process_id: Option, ) -> Self { let parsed_cmd = parse_command(command); Self::UnifiedExec { @@ -140,6 +144,7 @@ impl ToolEmitter { source, interaction_input, parsed_cmd, + process_id, } } @@ -157,7 +162,7 @@ impl ToolEmitter { ) => { emit_exec_stage( ctx, - ExecCommandInput::new(command, cwd.as_path(), parsed_cmd, *source, None), + ExecCommandInput::new(command, cwd.as_path(), parsed_cmd, *source, None, None), stage, ) .await; @@ -229,6 +234,7 @@ impl ToolEmitter { source, interaction_input, parsed_cmd, + process_id, }, stage, ) => { @@ -240,6 +246,7 @@ impl ToolEmitter { parsed_cmd, *source, interaction_input.as_deref(), + process_id.as_deref(), ), stage, ) @@ -319,6 +326,7 @@ struct ExecCommandInput<'a> { parsed_cmd: &'a [ParsedCommand], source: ExecCommandSource, interaction_input: Option<&'a str>, + process_id: Option<&'a str>, } impl<'a> ExecCommandInput<'a> { @@ -328,6 +336,7 @@ impl<'a> ExecCommandInput<'a> { parsed_cmd: &'a [ParsedCommand], source: ExecCommandSource, interaction_input: Option<&'a str>, + process_id: Option<&'a str>, ) -> Self { Self { command, @@ -335,6 +344,7 @@ impl<'a> ExecCommandInput<'a> { parsed_cmd, source, interaction_input, + process_id, } } } @@ -362,6 +372,7 @@ async fn emit_exec_stage( exec_input.parsed_cmd, exec_input.source, exec_input.interaction_input.map(str::to_owned), + exec_input.process_id, ) .await; } @@ -402,6 +413,7 @@ async fn emit_exec_end( ctx.turn, EventMsg::ExecCommandEnd(ExecCommandEndEvent { call_id: ctx.call_id.to_string(), + process_id: exec_input.process_id.map(str::to_owned), turn_id: ctx.turn.sub_id.clone(), command: exec_input.command.to_vec(), cwd: exec_input.cwd.to_path_buf(), diff --git a/codex-rs/core/src/tools/handlers/unified_exec.rs b/codex-rs/core/src/tools/handlers/unified_exec.rs index 6790b73599..0c4f0a031b 100644 --- a/codex-rs/core/src/tools/handlers/unified_exec.rs +++ b/codex-rs/core/src/tools/handlers/unified_exec.rs @@ -46,6 +46,7 @@ struct ExecCommandArgs { #[derive(Debug, Deserialize)] struct WriteStdinArgs { + // The model is trained on `session_id`. session_id: i32, #[serde(default)] chars: String, @@ -128,6 +129,7 @@ impl ToolHandler for UnifiedExecHandler { "failed to parse exec_command arguments: {err:?}" )) })?; + let process_id = manager.allocate_process_id().await; let command = get_command(&args); let ExecCommandArgs { @@ -168,6 +170,7 @@ impl ToolHandler for UnifiedExecHandler { cwd.clone(), ExecCommandSource::UnifiedExecStartup, None, + Some(process_id.clone()), ); emitter.emit(event_ctx, ToolEventStage::Begin).await; @@ -175,6 +178,7 @@ impl ToolHandler for UnifiedExecHandler { .exec_command( ExecCommandRequest { command, + process_id, yield_time_ms, max_output_tokens, workdir, @@ -197,7 +201,7 @@ impl ToolHandler for UnifiedExecHandler { manager .write_stdin(WriteStdinRequest { call_id: &call_id, - session_id: args.session_id, + process_id: &args.session_id.to_string(), input: &args.chars, yield_time_ms: args.yield_time_ms, max_output_tokens: args.max_output_tokens, @@ -255,8 +259,9 @@ fn format_response(response: &UnifiedExecResponse) -> String { sections.push(format!("Process exited with code {exit_code}")); } - if let Some(session_id) = response.session_id { - sections.push(format!("Process running with session ID {session_id}")); + if let Some(process_id) = &response.process_id { + // Training still uses "session ID". + sections.push(format!("Process running with session ID {process_id}")); } if let Some(original_token_count) = response.original_token_count { diff --git a/codex-rs/core/src/unified_exec/errors.rs b/codex-rs/core/src/unified_exec/errors.rs index bfbef93572..02031f22fa 100644 --- a/codex-rs/core/src/unified_exec/errors.rs +++ b/codex-rs/core/src/unified_exec/errors.rs @@ -5,8 +5,9 @@ use thiserror::Error; pub(crate) enum UnifiedExecError { #[error("Failed to create unified exec session: {message}")] CreateSession { message: String }, - #[error("Unknown session id {session_id}")] - UnknownSessionId { session_id: i32 }, + // Called "session" in the model's training. + #[error("Unknown session id {process_id}")] + UnknownSessionId { process_id: String }, #[error("failed to write to stdin")] WriteToStdin, #[error("missing command line for unified exec request")] diff --git a/codex-rs/core/src/unified_exec/mod.rs b/codex-rs/core/src/unified_exec/mod.rs index 1c9194baa8..35df80255b 100644 --- a/codex-rs/core/src/unified_exec/mod.rs +++ b/codex-rs/core/src/unified_exec/mod.rs @@ -22,9 +22,9 @@ //! - `session_manager.rs`: orchestration (approvals, sandboxing, reuse) and request handling. use std::collections::HashMap; +use std::collections::HashSet; use std::path::PathBuf; use std::sync::Arc; -use std::sync::atomic::AtomicI32; use std::time::Duration; use rand::Rng; @@ -67,6 +67,7 @@ impl UnifiedExecContext { #[derive(Debug)] pub(crate) struct ExecCommandRequest { pub command: Vec, + pub process_id: String, pub yield_time_ms: u64, pub max_output_tokens: Option, pub workdir: Option, @@ -77,7 +78,7 @@ pub(crate) struct ExecCommandRequest { #[derive(Debug)] pub(crate) struct WriteStdinRequest<'a> { pub call_id: &'a str, - pub session_id: i32, + pub process_id: &'a str, pub input: &'a str, pub yield_time_ms: u64, pub max_output_tokens: Option, @@ -89,7 +90,7 @@ pub(crate) struct UnifiedExecResponse { pub chunk_id: String, pub wall_time: Duration, pub output: String, - pub session_id: Option, + pub process_id: Option, pub exit_code: Option, pub original_token_count: Option, pub session_command: Option>, @@ -97,15 +98,16 @@ pub(crate) struct UnifiedExecResponse { #[derive(Default)] pub(crate) struct UnifiedExecSessionManager { - next_session_id: AtomicI32, - sessions: Mutex>, + sessions: Mutex>, + used_session_ids: Mutex>, } struct SessionEntry { - session: session::UnifiedExecSession, + session: UnifiedExecSession, session_ref: Arc, turn_ref: Arc, call_id: String, + process_id: String, command: Vec, cwd: PathBuf, started_at: tokio::time::Instant, @@ -159,6 +161,11 @@ mod tests { ) -> Result { let context = UnifiedExecContext::new(Arc::clone(session), Arc::clone(turn), "call".to_string()); + let process_id = session + .services + .unified_exec_manager + .allocate_process_id() + .await; session .services @@ -166,6 +173,7 @@ mod tests { .exec_command( ExecCommandRequest { command: vec!["bash".to_string(), "-lc".to_string(), cmd.to_string()], + process_id, yield_time_ms, max_output_tokens: None, workdir: None, @@ -179,7 +187,7 @@ mod tests { async fn write_stdin( session: &Arc, - session_id: i32, + process_id: &str, input: &str, yield_time_ms: u64, ) -> Result { @@ -188,7 +196,7 @@ mod tests { .unified_exec_manager .write_stdin(WriteStdinRequest { call_id: "write-stdin", - session_id, + process_id, input, yield_time_ms, max_output_tokens: None, @@ -221,11 +229,15 @@ mod tests { let (session, turn) = test_session_and_turn(); let open_shell = exec_command(&session, &turn, "bash -i", 2_500).await?; - let session_id = open_shell.session_id.expect("expected session_id"); + let process_id = open_shell + .process_id + .as_ref() + .expect("expected process_id") + .as_str(); write_stdin( &session, - session_id, + process_id, "export CODEX_INTERACTIVE_SHELL_VAR=codex\n", 2_500, ) @@ -233,7 +245,7 @@ mod tests { let out_2 = write_stdin( &session, - session_id, + process_id, "echo $CODEX_INTERACTIVE_SHELL_VAR\n", 2_500, ) @@ -253,11 +265,15 @@ mod tests { let (session, turn) = test_session_and_turn(); let shell_a = exec_command(&session, &turn, "bash -i", 2_500).await?; - let session_a = shell_a.session_id.expect("expected session id"); + let session_a = shell_a + .process_id + .as_ref() + .expect("expected process id") + .clone(); write_stdin( &session, - session_a, + session_a.as_str(), "export CODEX_INTERACTIVE_SHELL_VAR=codex\n", 2_500, ) @@ -265,9 +281,10 @@ mod tests { let out_2 = exec_command(&session, &turn, "echo $CODEX_INTERACTIVE_SHELL_VAR", 2_500).await?; + tokio::time::sleep(Duration::from_secs(2)).await; assert!( - out_2.session_id.is_none(), - "short command should not retain a session" + out_2.process_id.is_none(), + "short command should not report a process id if it exits quickly" ); assert!( !out_2.output.contains("codex"), @@ -276,7 +293,11 @@ mod tests { let out_3 = write_stdin( &session, - session_a, + shell_a + .process_id + .as_ref() + .expect("expected process id") + .as_str(), "echo $CODEX_INTERACTIVE_SHELL_VAR\n", 2_500, ) @@ -296,11 +317,15 @@ mod tests { let (session, turn) = test_session_and_turn(); let open_shell = exec_command(&session, &turn, "bash -i", 2_500).await?; - let session_id = open_shell.session_id.expect("expected session id"); + let process_id = open_shell + .process_id + .as_ref() + .expect("expected process id") + .as_str(); write_stdin( &session, - session_id, + process_id, "export CODEX_INTERACTIVE_SHELL_VAR=codex\n", 2_500, ) @@ -308,7 +333,7 @@ mod tests { let out_2 = write_stdin( &session, - session_id, + process_id, "sleep 5 && echo $CODEX_INTERACTIVE_SHELL_VAR\n", 10, ) @@ -320,7 +345,7 @@ mod tests { tokio::time::sleep(Duration::from_secs(7)).await; - let out_3 = write_stdin(&session, session_id, "", 100).await?; + let out_3 = write_stdin(&session, process_id, "", 100).await?; assert!( out_3.output.contains("codex"), @@ -337,7 +362,7 @@ mod tests { let result = exec_command(&session, &turn, "echo codex", 120_000).await?; - assert!(result.session_id.is_none()); + assert!(result.process_id.is_some()); assert!(result.output.contains("codex")); Ok(()) @@ -350,8 +375,8 @@ mod tests { let result = exec_command(&session, &turn, "echo codex", 2_500).await?; assert!( - result.session_id.is_none(), - "completed command should not retain session" + result.process_id.is_some(), + "completed command should report a process id" ); assert!(result.output.contains("codex")); @@ -375,31 +400,35 @@ mod tests { let (session, turn) = test_session_and_turn(); let open_shell = exec_command(&session, &turn, "bash -i", 2_500).await?; - let session_id = open_shell.session_id.expect("expected session id"); + let process_id = open_shell + .process_id + .as_ref() + .expect("expected process id") + .as_str(); - write_stdin(&session, session_id, "exit\n", 2_500).await?; + write_stdin(&session, process_id, "exit\n", 2_500).await?; tokio::time::sleep(Duration::from_millis(200)).await; - let err = write_stdin(&session, session_id, "", 100) + let err = write_stdin(&session, process_id, "", 100) .await .expect_err("expected unknown session error"); match err { - UnifiedExecError::UnknownSessionId { session_id: err_id } => { - assert_eq!(err_id, session_id); + UnifiedExecError::UnknownSessionId { process_id: err_id } => { + assert_eq!(err_id, process_id, "process id should match request"); } other => panic!("expected UnknownSessionId, got {other:?}"), } assert!( - !session + session .services .unified_exec_manager .sessions .lock() .await - .contains_key(&session_id) + .is_empty() ); Ok(()) diff --git a/codex-rs/core/src/unified_exec/session_manager.rs b/codex-rs/core/src/unified_exec/session_manager.rs index 515bb607f6..df8f543212 100644 --- a/codex-rs/core/src/unified_exec/session_manager.rs +++ b/codex-rs/core/src/unified_exec/session_manager.rs @@ -1,9 +1,9 @@ +use rand::Rng; use std::cmp::Reverse; use std::collections::HashMap; use std::collections::HashSet; use std::path::PathBuf; use std::sync::Arc; - use tokio::sync::Notify; use tokio::sync::mpsc; use tokio::time::Duration; @@ -75,9 +75,38 @@ struct PreparedSessionHandles { turn_ref: Arc, command: Vec, cwd: PathBuf, + process_id: String, } impl UnifiedExecSessionManager { + pub(crate) async fn allocate_process_id(&self) -> String { + loop { + let mut store = self.used_session_ids.lock().await; + + let process_id = if !cfg!(test) && !cfg!(feature = "deterministic_process_ids") { + // production mode → random + rand::rng().random_range(1_000..100_000).to_string() + } else { + // test or deterministic mode + let next = store + .iter() + .filter_map(|s| s.parse::().ok()) + .max() + .map(|m| std::cmp::max(m, 999) + 1) + .unwrap_or(1000); + + next.to_string() + }; + + if store.contains(&process_id) { + continue; + } + + store.insert(process_id.clone()); + return process_id; + } + } + pub(crate) async fn exec_command( &self, request: ExecCommandRequest, @@ -122,14 +151,20 @@ impl UnifiedExecSessionManager { let has_exited = session.has_exited(); let exit_code = session.exit_code(); let chunk_id = generate_chunk_id(); - let session_id = if has_exited { + let process_id = if has_exited { None } else { // Only store session if not exited. - let stored_id = self - .store_session(session, context, &request.command, cwd.clone(), start) - .await; - Some(stored_id) + self.store_session( + session, + context, + &request.command, + cwd.clone(), + start, + request.process_id.clone(), + ) + .await; + Some(request.process_id.clone()) }; let original_token_count = approx_token_count(&text); @@ -138,18 +173,18 @@ impl UnifiedExecSessionManager { chunk_id, wall_time, output, - session_id, + process_id: process_id.clone(), exit_code, original_token_count: Some(original_token_count), session_command: Some(request.command.clone()), }; - if response.session_id.is_some() { + if !has_exited { Self::emit_waiting_status(&context.session, &context.turn, &request.command).await; } // If the command completed during this call, emit an ExecCommandEnd via the emitter. - if response.session_id.is_none() { + if has_exited { let exit = response.exit_code.unwrap_or(-1); Self::emit_exec_end_from_context( context, @@ -158,6 +193,9 @@ impl UnifiedExecSessionManager { response.output.clone(), exit, response.wall_time, + // We always emit the process ID in order to keep consistency between the Begin + // event and the End event. + Some(request.process_id), ) .await; } @@ -169,7 +207,7 @@ impl UnifiedExecSessionManager { &self, request: WriteStdinRequest<'_>, ) -> Result { - let session_id = request.session_id; + let process_id = request.process_id.to_string(); let PreparedSessionHandles { writer_tx, @@ -180,13 +218,15 @@ impl UnifiedExecSessionManager { turn_ref, command: session_command, cwd: session_cwd, - } = self.prepare_session_handles(session_id).await?; + process_id, + } = self.prepare_session_handles(process_id.as_str()).await?; let interaction_emitter = ToolEmitter::unified_exec( &session_command, session_cwd.clone(), ExecCommandSource::UnifiedExecInteraction, (!request.input.is_empty()).then(|| request.input.to_string()), + Some(process_id.clone()), ); let make_event_ctx = || { ToolEventCtx::new( @@ -233,17 +273,21 @@ impl UnifiedExecSessionManager { let original_token_count = approx_token_count(&text); let chunk_id = generate_chunk_id(); - let status = self.refresh_session_state(session_id).await; - let (session_id, exit_code, completion_entry, event_call_id) = match status { - SessionStatus::Alive { exit_code, call_id } => { - (Some(session_id), exit_code, None, call_id) - } + let status = self.refresh_session_state(process_id.as_str()).await; + let (process_id, exit_code, completion_entry, event_call_id) = match status { + SessionStatus::Alive { + exit_code, + call_id, + process_id, + } => (Some(process_id), exit_code, None, call_id), SessionStatus::Exited { exit_code, entry } => { let call_id = entry.call_id.clone(); (None, exit_code, Some(*entry), call_id) } SessionStatus::Unknown => { - return Err(UnifiedExecError::UnknownSessionId { session_id }); + return Err(UnifiedExecError::UnknownSessionId { + process_id: request.process_id.to_string(), + }); } }; @@ -252,7 +296,7 @@ impl UnifiedExecSessionManager { chunk_id, wall_time, output, - session_id, + process_id, exit_code, original_token_count: Some(original_token_count), session_command: Some(session_command.clone()), @@ -273,7 +317,7 @@ impl UnifiedExecSessionManager { ) .await; - if response.session_id.is_some() { + if response.process_id.is_some() { Self::emit_waiting_status(&session_ref, &turn_ref, &session_command).await; } @@ -286,16 +330,17 @@ impl UnifiedExecSessionManager { Ok(response) } - async fn refresh_session_state(&self, session_id: i32) -> SessionStatus { + async fn refresh_session_state(&self, process_id: &str) -> SessionStatus { let mut sessions = self.sessions.lock().await; - let Some(entry) = sessions.get(&session_id) else { + let Some(entry) = sessions.get(process_id) else { return SessionStatus::Unknown; }; let exit_code = entry.session.exit_code(); + let process_id = entry.process_id.clone(); if entry.session.has_exited() { - let Some(entry) = sessions.remove(&session_id) else { + let Some(entry) = sessions.remove(&process_id) else { return SessionStatus::Unknown; }; SessionStatus::Exited { @@ -306,18 +351,21 @@ impl UnifiedExecSessionManager { SessionStatus::Alive { exit_code, call_id: entry.call_id.clone(), + process_id, } } } async fn prepare_session_handles( &self, - session_id: i32, + process_id: &str, ) -> Result { let mut sessions = self.sessions.lock().await; let entry = sessions - .get_mut(&session_id) - .ok_or(UnifiedExecError::UnknownSessionId { session_id })?; + .get_mut(process_id) + .ok_or(UnifiedExecError::UnknownSessionId { + process_id: process_id.to_string(), + })?; entry.last_used = Instant::now(); let OutputHandles { output_buffer, @@ -334,6 +382,7 @@ impl UnifiedExecSessionManager { turn_ref: Arc::clone(&entry.turn_ref), command: entry.command.clone(), cwd: entry.cwd.clone(), + process_id: entry.process_id.clone(), }) } @@ -347,6 +396,7 @@ impl UnifiedExecSessionManager { .map_err(|_| UnifiedExecError::WriteToStdin) } + #[allow(clippy::too_many_arguments)] async fn store_session( &self, session: UnifiedExecSession, @@ -354,15 +404,14 @@ impl UnifiedExecSessionManager { command: &[String], cwd: PathBuf, started_at: Instant, - ) -> i32 { - let session_id = self - .next_session_id - .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + process_id: String, + ) { let entry = SessionEntry { session, session_ref: Arc::clone(&context.session), turn_ref: Arc::clone(&context.turn), call_id: context.call_id.clone(), + process_id: process_id.clone(), command: command.to_vec(), cwd, started_at, @@ -370,8 +419,7 @@ impl UnifiedExecSessionManager { }; let mut sessions = self.sessions.lock().await; Self::prune_sessions_if_needed(&mut sessions); - sessions.insert(session_id, entry); - session_id + sessions.insert(process_id, entry); } async fn emit_exec_end_from_entry( @@ -399,6 +447,7 @@ impl UnifiedExecSessionManager { entry.cwd, ExecCommandSource::UnifiedExecStartup, None, + Some(entry.process_id.clone()), ); emitter .emit(event_ctx, ToolEventStage::Success(output)) @@ -412,6 +461,7 @@ impl UnifiedExecSessionManager { aggregated_output: String, exit_code: i32, duration: Duration, + process_id: Option, ) { let output = ExecToolCallOutput { exit_code, @@ -427,8 +477,13 @@ impl UnifiedExecSessionManager { &context.call_id, None, ); - let emitter = - ToolEmitter::unified_exec(command, cwd, ExecCommandSource::UnifiedExecStartup, None); + let emitter = ToolEmitter::unified_exec( + command, + cwd, + ExecCommandSource::UnifiedExecStartup, + None, + process_id, + ); emitter .emit(event_ctx, ToolEventStage::Success(output)) .await; @@ -574,14 +629,14 @@ impl UnifiedExecSessionManager { collected } - fn prune_sessions_if_needed(sessions: &mut HashMap) { + fn prune_sessions_if_needed(sessions: &mut HashMap) { if sessions.len() < MAX_UNIFIED_EXEC_SESSIONS { return; } - let meta: Vec<(i32, Instant, bool)> = sessions + let meta: Vec<(String, Instant, bool)> = sessions .iter() - .map(|(id, entry)| (*id, entry.last_used, entry.session.has_exited())) + .map(|(id, entry)| (id.clone(), entry.last_used, entry.session.has_exited())) .collect(); if let Some(session_id) = Self::session_id_to_prune_from_meta(&meta) { @@ -590,32 +645,32 @@ impl UnifiedExecSessionManager { } // Centralized pruning policy so we can easily swap strategies later. - fn session_id_to_prune_from_meta(meta: &[(i32, Instant, bool)]) -> Option { + fn session_id_to_prune_from_meta(meta: &[(String, Instant, bool)]) -> Option { if meta.is_empty() { return None; } let mut by_recency = meta.to_vec(); by_recency.sort_by_key(|(_, last_used, _)| Reverse(*last_used)); - let protected: HashSet = by_recency + let protected: HashSet = by_recency .iter() .take(8) - .map(|(session_id, _, _)| *session_id) + .map(|(process_id, _, _)| process_id.clone()) .collect(); let mut lru = meta.to_vec(); lru.sort_by_key(|(_, last_used, _)| *last_used); - if let Some((session_id, _, _)) = lru + if let Some((process_id, _, _)) = lru .iter() - .find(|(session_id, _, exited)| !protected.contains(session_id) && *exited) + .find(|(process_id, _, exited)| !protected.contains(process_id) && *exited) { - return Some(*session_id); + return Some(process_id.clone()); } lru.into_iter() - .find(|(session_id, _, _)| !protected.contains(session_id)) - .map(|(session_id, _, _)| session_id) + .find(|(process_id, _, _)| !protected.contains(process_id)) + .map(|(process_id, _, _)| process_id) } pub(crate) async fn terminate_all_sessions(&self) { @@ -628,6 +683,7 @@ enum SessionStatus { Alive { exit_code: Option, call_id: String, + process_id: String, }, Exited { exit_code: Option, @@ -675,64 +731,67 @@ mod tests { #[test] fn pruning_prefers_exited_sessions_outside_recently_used() { let now = Instant::now(); + let id = |n: i32| n.to_string(); let meta = vec![ - (1, now - Duration::from_secs(40), false), - (2, now - Duration::from_secs(30), true), - (3, now - Duration::from_secs(20), false), - (4, now - Duration::from_secs(19), false), - (5, now - Duration::from_secs(18), false), - (6, now - Duration::from_secs(17), false), - (7, now - Duration::from_secs(16), false), - (8, now - Duration::from_secs(15), false), - (9, now - Duration::from_secs(14), false), - (10, now - Duration::from_secs(13), false), + (id(1), now - Duration::from_secs(40), false), + (id(2), now - Duration::from_secs(30), true), + (id(3), now - Duration::from_secs(20), false), + (id(4), now - Duration::from_secs(19), false), + (id(5), now - Duration::from_secs(18), false), + (id(6), now - Duration::from_secs(17), false), + (id(7), now - Duration::from_secs(16), false), + (id(8), now - Duration::from_secs(15), false), + (id(9), now - Duration::from_secs(14), false), + (id(10), now - Duration::from_secs(13), false), ]; let candidate = UnifiedExecSessionManager::session_id_to_prune_from_meta(&meta); - assert_eq!(candidate, Some(2)); + assert_eq!(candidate, Some(id(2))); } #[test] fn pruning_falls_back_to_lru_when_no_exited() { let now = Instant::now(); + let id = |n: i32| n.to_string(); let meta = vec![ - (1, now - Duration::from_secs(40), false), - (2, now - Duration::from_secs(30), false), - (3, now - Duration::from_secs(20), false), - (4, now - Duration::from_secs(19), false), - (5, now - Duration::from_secs(18), false), - (6, now - Duration::from_secs(17), false), - (7, now - Duration::from_secs(16), false), - (8, now - Duration::from_secs(15), false), - (9, now - Duration::from_secs(14), false), - (10, now - Duration::from_secs(13), false), + (id(1), now - Duration::from_secs(40), false), + (id(2), now - Duration::from_secs(30), false), + (id(3), now - Duration::from_secs(20), false), + (id(4), now - Duration::from_secs(19), false), + (id(5), now - Duration::from_secs(18), false), + (id(6), now - Duration::from_secs(17), false), + (id(7), now - Duration::from_secs(16), false), + (id(8), now - Duration::from_secs(15), false), + (id(9), now - Duration::from_secs(14), false), + (id(10), now - Duration::from_secs(13), false), ]; let candidate = UnifiedExecSessionManager::session_id_to_prune_from_meta(&meta); - assert_eq!(candidate, Some(1)); + assert_eq!(candidate, Some(id(1))); } #[test] fn pruning_protects_recent_sessions_even_if_exited() { let now = Instant::now(); + let id = |n: i32| n.to_string(); let meta = vec![ - (1, now - Duration::from_secs(40), false), - (2, now - Duration::from_secs(30), false), - (3, now - Duration::from_secs(20), true), - (4, now - Duration::from_secs(19), false), - (5, now - Duration::from_secs(18), false), - (6, now - Duration::from_secs(17), false), - (7, now - Duration::from_secs(16), false), - (8, now - Duration::from_secs(15), false), - (9, now - Duration::from_secs(14), false), - (10, now - Duration::from_secs(13), true), + (id(1), now - Duration::from_secs(40), false), + (id(2), now - Duration::from_secs(30), false), + (id(3), now - Duration::from_secs(20), true), + (id(4), now - Duration::from_secs(19), false), + (id(5), now - Duration::from_secs(18), false), + (id(6), now - Duration::from_secs(17), false), + (id(7), now - Duration::from_secs(16), false), + (id(8), now - Duration::from_secs(15), false), + (id(9), now - Duration::from_secs(14), false), + (id(10), now - Duration::from_secs(13), true), ]; let candidate = UnifiedExecSessionManager::session_id_to_prune_from_meta(&meta); // (10) is exited but among the last 8; we should drop the LRU outside that set. - assert_eq!(candidate, Some(1)); + assert_eq!(candidate, Some(id(1))); } } diff --git a/codex-rs/core/tests/suite/unified_exec.rs b/codex-rs/core/tests/suite/unified_exec.rs index 404801658c..63964b2415 100644 --- a/codex-rs/core/tests/suite/unified_exec.rs +++ b/codex-rs/core/tests/suite/unified_exec.rs @@ -44,7 +44,7 @@ fn extract_output_text(item: &Value) -> Option<&str> { struct ParsedUnifiedExecOutput { chunk_id: Option, wall_time_seconds: f64, - session_id: Option, + process_id: Option, exit_code: Option, original_token_count: Option, output: String, @@ -59,7 +59,7 @@ fn parse_unified_exec_output(raw: &str) -> Result { r#"(?:Chunk ID: (?P[^\n]+)\n)?"#, r#"Wall time: (?P-?\d+(?:\.\d+)?) seconds\n"#, r#"(?:Process exited with code (?P-?\d+)\n)?"#, - r#"(?:Process running with session ID (?P-?\d+)\n)?"#, + r#"(?:Process running with session ID (?P-?\d+)\n)?"#, r#"(?:Original token count: (?P\d+)\n)?"#, r#"Output:\n?(?P.*)$"#, )) @@ -92,15 +92,9 @@ fn parse_unified_exec_output(raw: &str) -> Result { }) .transpose()?; - let session_id = captures - .name("session_id") - .map(|value| { - value - .as_str() - .parse::() - .context("failed to parse session id from unified exec output") - }) - .transpose()?; + let process_id = captures + .name("process_id") + .map(|value| value.as_str().to_string()); let original_token_count = captures .name("original_token_count") @@ -121,7 +115,7 @@ fn parse_unified_exec_output(raw: &str) -> Result { Ok(ParsedUnifiedExecOutput { chunk_id, wall_time_seconds, - session_id, + process_id, exit_code, original_token_count, output, @@ -335,7 +329,7 @@ async fn unified_exec_emits_exec_command_end_event() -> Result<()> { let poll_call_id = "uexec-end-event-poll"; let poll_args = json!({ "chars": "", - "session_id": 0, + "session_id": 1000, "yield_time_ms": 250, }); @@ -493,7 +487,7 @@ async fn unified_exec_emits_output_delta_for_write_stdin() -> Result<()> { let stdin_call_id = "uexec-stdin-delta"; let stdin_args = json!({ "chars": "echo WSTDIN-MARK\\n", - "session_id": 0, + "session_id": 1000, "yield_time_ms": 800, }); @@ -592,7 +586,7 @@ async fn unified_exec_emits_begin_for_write_stdin() -> Result<()> { let stdin_call_id = "uexec-stdin-begin"; let stdin_args = json!({ "chars": "echo hello", - "session_id": 0, + "session_id": 1000, "yield_time_ms": 400, }); @@ -694,7 +688,7 @@ async fn unified_exec_emits_begin_event_for_write_stdin_requests() -> Result<()> let poll_call_id = "uexec-poll-empty"; let poll_args = json!({ "chars": "", - "session_id": 0, + "session_id": 1000, "yield_time_ms": 150, }); @@ -880,8 +874,8 @@ async fn exec_command_reports_chunk_and_exit_metadata() -> Result<()> { ); assert!( - metadata.session_id.is_none(), - "exec_command for a completed process should not include session_id" + metadata.process_id.is_none(), + "exec_command for a completed process should not include process_id" ); let exit_code = metadata.exit_code.expect("expected exit_code"); @@ -973,7 +967,7 @@ async fn unified_exec_respects_early_exit_notifications() -> Result<()> { .expect("missing early exit unified_exec output"); assert!( - output.session_id.is_none(), + output.process_id.is_none(), "short-lived process should not keep a session alive" ); assert_eq!( @@ -1023,12 +1017,12 @@ async fn write_stdin_returns_exit_metadata_and_clears_session() -> Result<()> { }); let send_args = serde_json::json!({ "chars": "hello unified exec\n", - "session_id": 0, + "session_id": 1000, "yield_time_ms": 500, }); let exit_args = serde_json::json!({ "chars": "\u{0004}", - "session_id": 0, + "session_id": 1000, "yield_time_ms": 500, }); @@ -1099,12 +1093,13 @@ async fn write_stdin_returns_exit_metadata_and_clears_session() -> Result<()> { let start_output = outputs .get(start_call_id) .expect("missing start output for exec_command"); - let session_id = start_output - .session_id - .expect("expected session id from exec_command"); + let process_id = start_output + .process_id + .clone() + .expect("expected process id from exec_command"); assert!( - session_id >= 0, - "session_id should be non-negative, got {session_id}" + process_id.len() > 3, + "process_id should be at least 4 digits, got {process_id}" ); assert!( start_output.exit_code.is_none(), @@ -1120,11 +1115,12 @@ async fn write_stdin_returns_exit_metadata_and_clears_session() -> Result<()> { "expected echoed output from cat, got {echoed:?}" ); let echoed_session = send_output - .session_id - .expect("write_stdin should return session id while process is running"); + .process_id + .clone() + .expect("write_stdin should return process id while process is running"); assert_eq!( - echoed_session, session_id, - "write_stdin should reuse existing session id" + echoed_session, process_id, + "write_stdin should reuse existing process id" ); assert!( send_output.exit_code.is_none(), @@ -1135,8 +1131,8 @@ async fn write_stdin_returns_exit_metadata_and_clears_session() -> Result<()> { .get(exit_call_id) .expect("missing exit metadata output"); assert!( - exit_output.session_id.is_none(), - "session_id should be omitted once the process exits" + exit_output.process_id.is_none(), + "process_id should be omitted once the process exits" ); let exit_code = exit_output .exit_code @@ -1182,14 +1178,14 @@ async fn unified_exec_emits_end_event_when_session_dies_via_stdin() -> Result<() let echo_call_id = "uexec-end-on-exit-echo"; let echo_args = serde_json::json!({ "chars": "bye-END\n", - "session_id": 0, + "session_id": 1000, "yield_time_ms": 300, }); let exit_call_id = "uexec-end-on-exit"; let exit_args = serde_json::json!({ "chars": "\u{0004}", - "session_id": 0, + "session_id": 1000, "yield_time_ms": 500, }); @@ -1285,7 +1281,7 @@ async fn unified_exec_reuses_session_via_stdin() -> Result<()> { let second_call_id = "uexec-stdin"; let second_args = serde_json::json!({ "chars": "hello unified exec\n", - "session_id": 0, + "session_id": 1000, "yield_time_ms": 500, }); @@ -1347,17 +1343,20 @@ async fn unified_exec_reuses_session_via_stdin() -> Result<()> { let start_output = outputs .get(first_call_id) .expect("missing first unified_exec output"); - let session_id = start_output.session_id.unwrap_or_default(); + let process_id = start_output.process_id.clone().unwrap_or_default(); assert!( - session_id >= 0, - "expected session id in first unified_exec response" + !process_id.is_empty(), + "expected process id in first unified_exec response" ); assert!(start_output.output.is_empty()); let reuse_output = outputs .get(second_call_id) .expect("missing reused unified_exec output"); - assert_eq!(reuse_output.session_id.unwrap_or_default(), session_id); + assert_eq!( + reuse_output.process_id.clone().unwrap_or_default(), + process_id + ); let echoed = reuse_output.output.as_str(); assert!( echoed.contains("hello unified exec"), @@ -1413,7 +1412,7 @@ PY let second_call_id = "uexec-lag-poll"; let second_args = serde_json::json!({ "chars": "", - "session_id": 0, + "session_id": 1000, "yield_time_ms": 2_000, }); @@ -1480,9 +1479,9 @@ PY let start_output = outputs .get(first_call_id) .expect("missing initial unified_exec output"); - let session_id = start_output.session_id.unwrap_or_default(); + let process_id = start_output.process_id.clone().unwrap_or_default(); assert!( - session_id >= 0, + !process_id.is_empty(), "expected session id from initial unified_exec response" ); @@ -1524,7 +1523,7 @@ async fn unified_exec_timeout_and_followup_poll() -> Result<()> { let second_call_id = "uexec-poll"; let second_args = serde_json::json!({ "chars": "", - "session_id": 0, + "session_id": 1000, "yield_time_ms": 800, }); @@ -1589,7 +1588,7 @@ async fn unified_exec_timeout_and_followup_poll() -> Result<()> { let outputs = collect_tool_outputs(&bodies)?; let first_output = outputs.get(first_call_id).expect("missing timeout output"); - assert_eq!(first_output.session_id, Some(0)); + assert!(first_output.process_id.is_some()); assert!(first_output.output.is_empty()); let poll_output = outputs.get(second_call_id).expect("missing poll output"); @@ -1824,7 +1823,7 @@ async fn unified_exec_prunes_exited_sessions_first() -> Result<()> { let keep_write_call_id = "uexec-prune-keep-write"; let keep_write_args = serde_json::json!({ "chars": "still alive\n", - "session_id": 0, + "session_id": 1000, "yield_time_ms": 500, }); events.push(ev_function_call( @@ -1836,7 +1835,7 @@ async fn unified_exec_prunes_exited_sessions_first() -> Result<()> { let probe_call_id = "uexec-prune-probe"; let probe_args = serde_json::json!({ "chars": "should fail\n", - "session_id": 1, + "session_id": 1001, "yield_time_ms": 500, }); events.push(ev_function_call( @@ -1885,7 +1884,7 @@ async fn unified_exec_prunes_exited_sessions_first() -> Result<()> { .find_map(|req| req.function_call_output_text(keep_call_id)) .expect("missing initial keep session output"); let keep_start_output = parse_unified_exec_output(&keep_start)?; - pretty_assertions::assert_eq!(keep_start_output.session_id, Some(0)); + assert!(keep_start_output.process_id.is_some()); assert!(keep_start_output.exit_code.is_none()); let prune_start = requests @@ -1893,7 +1892,7 @@ async fn unified_exec_prunes_exited_sessions_first() -> Result<()> { .find_map(|req| req.function_call_output_text(prune_call_id)) .expect("missing initial prune session output"); let prune_start_output = parse_unified_exec_output(&prune_start)?; - pretty_assertions::assert_eq!(prune_start_output.session_id, Some(1)); + assert!(prune_start_output.process_id.is_some()); assert!(prune_start_output.exit_code.is_none()); let keep_write = requests @@ -1901,7 +1900,7 @@ async fn unified_exec_prunes_exited_sessions_first() -> Result<()> { .find_map(|req| req.function_call_output_text(keep_write_call_id)) .expect("missing keep write output"); let keep_write_output = parse_unified_exec_output(&keep_write)?; - pretty_assertions::assert_eq!(keep_write_output.session_id, Some(0)); + assert!(keep_write_output.process_id.is_some()); assert!( keep_write_output.output.contains("still alive"), "expected cat session to echo input, got {:?}", @@ -1913,7 +1912,7 @@ async fn unified_exec_prunes_exited_sessions_first() -> Result<()> { .find_map(|req| req.function_call_output_text(probe_call_id)) .expect("missing probe output"); assert!( - pruned_probe.contains("UnknownSessionId") || pruned_probe.contains("Unknown session id"), + pruned_probe.contains("UnknownSessionId") || pruned_probe.contains("Unknown process id"), "expected probe to fail after pruning, got {pruned_probe:?}" ); diff --git a/codex-rs/exec/tests/event_processor_with_json_output.rs b/codex-rs/exec/tests/event_processor_with_json_output.rs index 06f99b96b1..7b2d3c18ba 100644 --- a/codex-rs/exec/tests/event_processor_with_json_output.rs +++ b/codex-rs/exec/tests/event_processor_with_json_output.rs @@ -637,6 +637,7 @@ fn exec_command_end_success_produces_completed_command_item() { "c1", EventMsg::ExecCommandBegin(ExecCommandBeginEvent { call_id: "1".to_string(), + process_id: None, turn_id: "turn-1".to_string(), command: command.clone(), cwd: cwd.clone(), @@ -666,6 +667,7 @@ fn exec_command_end_success_produces_completed_command_item() { "c2", EventMsg::ExecCommandEnd(ExecCommandEndEvent { call_id: "1".to_string(), + process_id: None, turn_id: "turn-1".to_string(), command, cwd, @@ -709,6 +711,7 @@ fn exec_command_end_failure_produces_failed_command_item() { "c1", EventMsg::ExecCommandBegin(ExecCommandBeginEvent { call_id: "2".to_string(), + process_id: None, turn_id: "turn-1".to_string(), command: command.clone(), cwd: cwd.clone(), @@ -737,6 +740,7 @@ fn exec_command_end_failure_produces_failed_command_item() { "c2", EventMsg::ExecCommandEnd(ExecCommandEndEvent { call_id: "2".to_string(), + process_id: None, turn_id: "turn-1".to_string(), command, cwd, @@ -777,6 +781,7 @@ fn exec_command_end_without_begin_is_ignored() { "c1", EventMsg::ExecCommandEnd(ExecCommandEndEvent { call_id: "no-begin".to_string(), + process_id: None, turn_id: "turn-1".to_string(), command: Vec::new(), cwd: PathBuf::from("."), diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index 2fc47a50c9..60c4dffb2a 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -1328,6 +1328,10 @@ impl Default for ExecCommandSource { pub struct ExecCommandBeginEvent { /// Identifier so this can be paired with the ExecCommandEnd event. pub call_id: String, + /// Identifier for the underlying PTY process (when available). + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub process_id: Option, /// Turn ID that this command belongs to. pub turn_id: String, /// The command to be executed. @@ -1348,6 +1352,10 @@ pub struct ExecCommandBeginEvent { pub struct ExecCommandEndEvent { /// Identifier for the ExecCommandBegin that finished. pub call_id: String, + /// Identifier for the underlying PTY process (when available). + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional)] + pub process_id: Option, /// Turn ID that this command belongs to. pub turn_id: String, /// The command that was executed. diff --git a/codex-rs/tui/src/chatwidget/tests.rs b/codex-rs/tui/src/chatwidget/tests.rs index fde52a0646..2eebdaf237 100644 --- a/codex-rs/tui/src/chatwidget/tests.rs +++ b/codex-rs/tui/src/chatwidget/tests.rs @@ -722,6 +722,7 @@ fn begin_exec_with_source( let interaction_input = None; let event = ExecCommandBeginEvent { call_id: call_id.to_string(), + process_id: None, turn_id: "turn-1".to_string(), command, cwd, @@ -760,11 +761,13 @@ fn end_exec( parsed_cmd, source, interaction_input, + process_id, } = begin_event; chat.handle_codex_event(Event { id: call_id.clone(), msg: EventMsg::ExecCommandEnd(ExecCommandEndEvent { call_id, + process_id, turn_id, command, cwd, @@ -2880,6 +2883,7 @@ fn chatwidget_exec_and_status_layout_vt100_snapshot() { id: "c1".into(), msg: EventMsg::ExecCommandBegin(ExecCommandBeginEvent { call_id: "c1".into(), + process_id: None, turn_id: "turn-1".into(), command: command.clone(), cwd: cwd.clone(), @@ -2892,6 +2896,7 @@ fn chatwidget_exec_and_status_layout_vt100_snapshot() { id: "c1".into(), msg: EventMsg::ExecCommandEnd(ExecCommandEndEvent { call_id: "c1".into(), + process_id: None, turn_id: "turn-1".into(), command, cwd, From e953092949e485f05f06b26723e0e495fcfefe64 Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Tue, 25 Nov 2025 18:15:13 -0600 Subject: [PATCH 39/61] Fixed regression in experimental "sandbox command assessment" feature (#7308) Recent model updates caused the experimental "sandbox tool assessment" to time out most of the time leaving the user without any risk assessment or tool summary. This change explicitly sets the reasoning effort to medium and bumps the timeout. This change has no effect if the user hasn't enabled the `experimental_sandbox_command_assessment` feature flag. --- codex-rs/core/src/sandboxing/assessment.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/codex-rs/core/src/sandboxing/assessment.rs b/codex-rs/core/src/sandboxing/assessment.rs index 31e76777bb..719e3be1f0 100644 --- a/codex-rs/core/src/sandboxing/assessment.rs +++ b/codex-rs/core/src/sandboxing/assessment.rs @@ -14,6 +14,7 @@ use crate::protocol::SandboxPolicy; use askama::Template; use codex_otel::otel_event_manager::OtelEventManager; use codex_protocol::ConversationId; +use codex_protocol::config_types::ReasoningEffort as ReasoningEffortConfig; use codex_protocol::models::ContentItem; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::SandboxCommandAssessment; @@ -23,7 +24,8 @@ use serde_json::json; use tokio::time::timeout; use tracing::warn; -const SANDBOX_ASSESSMENT_TIMEOUT: Duration = Duration::from_secs(5); +const SANDBOX_ASSESSMENT_TIMEOUT: Duration = Duration::from_secs(15); +const SANDBOX_ASSESSMENT_REASONING_EFFORT: ReasoningEffortConfig = ReasoningEffortConfig::Medium; #[derive(Template)] #[template(path = "sandboxing/assessment_prompt.md", escape = "none")] @@ -130,7 +132,7 @@ pub(crate) async fn assess_command( Some(auth_manager), child_otel, provider, - config.model_reasoning_effort, + Some(SANDBOX_ASSESSMENT_REASONING_EFFORT), config.model_reasoning_summary, conversation_id, session_source, From 963009737fc6e7d45ca5cb37d63107b8be368eda Mon Sep 17 00:00:00 2001 From: jif-oai Date: Wed, 26 Nov 2025 11:30:34 +0000 Subject: [PATCH 40/61] nit: drop file (#7314) --- codex-rs/client.md | 206 --------------------------------------------- 1 file changed, 206 deletions(-) delete mode 100644 codex-rs/client.md diff --git a/codex-rs/client.md b/codex-rs/client.md deleted file mode 100644 index 4c9027ccba..0000000000 --- a/codex-rs/client.md +++ /dev/null @@ -1,206 +0,0 @@ -# Client Extraction Plan - -## Goals -- Split the HTTP transport/client code out of `codex-core` into a reusable crate that is agnostic of Codex/OpenAI business logic and API schemas. -- Create a separate API library crate that houses typed requests/responses for well-known APIs (Responses, Chat Completions, Compact) and plugs into the transport crate via minimal traits. -- Preserve current behaviour (auth headers, retries, SSE handling, rate-limit parsing, compaction, fixtures) while making the APIs symmetric and avoiding code duplication. -- Keep existing consumers (`codex-core`, tests, and tools) stable by providing a small compatibility layer during the transition. - -## Snapshot of Today -- `core/src/client.rs (ModelClient)` owns config/auth/session state, chooses wire API, builds payloads, drives retries, parses SSE, compaction, and rate-limit headers. -- `core/src/chat_completions.rs` implements the Chat Completions call + SSE parser + aggregation helper. -- `core/src/client_common.rs` holds `Prompt`, tool specs, shared request structs (`ResponsesApiRequest`, `TextControls`), and `ResponseEvent`/`ResponseStream`. -- `core/src/default_client.rs` wraps `reqwest` with Codex UA/originator defaults. -- `core/src/model_provider_info.rs` models providers (base URL, headers, env keys, retry/timeout tuning) and builds `CodexRequestBuilder`s. - - Current retry logic is co-located with API handling; streaming SSE parsing is duplicated across Responses/Chat. - -## Target Crates (with interfaces) - -- `codex-client` (generic transport) - - Owns the generic HTTP machinery: a `CodexHttpClient`/`CodexRequestBuilder`-style wrapper, retry/backoff hooks, streaming connector (SSE framing + idle timeout), header injection, and optional telemetry callbacks. - - Does **not** know about OpenAI/Codex-specific paths, headers, or error codes; it only exposes HTTP-level concepts (status, headers, bodies, connection errors). - - Minimal surface: - ```rust - pub trait HttpTransport { - fn execute(&self, req: Request) -> Result; - fn stream(&self, req: Request) -> Result; - } - - pub struct Request { - pub method: Method, - pub url: String, - pub headers: HeaderMap, - pub body: Option, - pub timeout: Option, - } - ``` - - Generic client traits (request/response/chunk are abstract over the transport): - ```rust - #[async_trait::async_trait] - pub trait UnaryClient { - async fn run(&self, req: Req) -> Result; - } - - #[async_trait::async_trait] - pub trait StreamClient { - async fn run(&self, req: Req) -> Result, TransportError>; - } - - pub struct RetryPolicy { - pub max_attempts: u64, - pub base_delay: Duration, - pub retry_on: RetryOn, // e.g., transport errors + 429/5xx - } - ``` - - `RetryOn` lives in `codex-client` and captures HTTP status classes and transport failures that qualify for retry. - - Implementations in `codex-api` plug in their own request types, parsers, and retry policies while reusing the transport’s backoff and error types. - - Planned runtime helper: - ```rust - pub async fn run_with_retry( - policy: RetryPolicy, - make_req: impl Fn() -> Request, - op: F, - ) -> Result - where - F: Fn(Request) -> Fut, - Fut: Future>, - { - for attempt in 0..=policy.max_attempts { - let req = make_req(); - match op(req).await { - Ok(resp) => return Ok(resp), - Err(err) if policy.retry_on.should_retry(&err, attempt) => { - tokio::time::sleep(backoff(policy.base_delay, attempt + 1)).await; - } - Err(err) => return Err(err), - } - } - Err(TransportError::RetryLimit) - } - ``` - - Unary clients wrap `transport.execute` with this helper and then deserialize. - - Stream clients wrap the **initial** `transport.stream` call with this helper. Mid-stream disconnects are surfaced as `StreamError`s; automatic resume/reconnect can be added later on top of this primitive if we introduce cursor support. - - Common helpers: `retry::backoff(attempt)`, `errors::{TransportError, StreamError}`. - - Streaming utility (SSE framing only): - ```rust - pub fn sse_stream( - bytes: S, - idle_timeout: Duration, - tx: mpsc::Sender>, - telemetry: Option>, - ) - where - S: Stream> + Unpin + Send + 'static; - ``` - - `sse_stream` is responsible for timeouts, connection-level errors, and emitting raw `data:` chunks as UTF-8 strings; parsing those strings into structured events is done in `codex-api`. - -- `codex-api` (OpenAI/Codex API library) - - Owns typed models for Responses/Chat/Compact plus shared helpers (`Prompt`, tool specs, text controls, `ResponsesApiRequest`, etc.). - - Knows about OpenAI/Codex semantics: - - URL shapes (`/v1/responses`, `/v1/chat/completions`, `/responses/compact`). - - Provider configuration (`WireApi`, base URLs, query params, per-provider retry knobs). - - Rate-limit headers (`x-codex-*`) and their mapping into `RateLimitSnapshot` / `CreditsSnapshot`. - - Error body formats (`{ error: { type, code, message, plan_type, resets_at } }`) and how they become API errors (context window exceeded, quota/usage limit, etc.). - - SSE event names (`response.output_item.done`, `response.completed`, `response.failed`, etc.) and their mapping into high-level events. - - Provides a provider abstraction (conceptually similar to `ModelProviderInfo`): - ```rust - pub struct Provider { - pub name: String, - pub base_url: String, - pub wire: WireApi, // Responses | Chat - pub headers: HeaderMap, - pub retry: RetryConfig, - pub stream_idle_timeout: Duration, - } - - pub trait AuthProvider { - /// Returns a bearer token to use for this request (if any). - /// Implementations are expected to be cheap and to surface already-refreshed tokens; - /// higher layers (`codex-core`) remain responsible for token refresh flows. - fn bearer_token(&self) -> Option; - - /// Optional ChatGPT account id header for Chat mode. - fn account_id(&self) -> Option; - } - ``` - - Ready-made clients built on `HttpTransport`: - ```rust - pub struct ResponsesClient { /* ... */ } - impl ResponsesClient { - pub async fn stream(&self, prompt: &Prompt) -> ApiResult>; - pub async fn compact(&self, prompt: &Prompt) -> ApiResult>; - } - - pub struct ChatClient { /* ... */ } - impl ChatClient { - pub async fn stream(&self, prompt: &Prompt) -> ApiResult>; - } - - pub struct CompactClient { /* ... */ } - impl CompactClient { - pub async fn compact(&self, prompt: &Prompt) -> ApiResult>; - } - ``` - - Streaming events unified across wire APIs (this can closely mirror `ResponseEvent` today, and we may type-alias one to the other during migration): - ```rust - pub enum ApiEvent { - Created, - OutputItemAdded(ResponseItem), - OutputItemDone(ResponseItem), - OutputTextDelta(String), - ReasoningContentDelta { delta: String, content_index: i64 }, - ReasoningSummaryDelta { delta: String, summary_index: i64 }, - RateLimits(RateLimitSnapshot), - Completed { response_id: String, token_usage: Option }, - } - ``` - - Error layering: - - `codex-client`: defines `TransportError` / `StreamError` (status codes, IO, timeouts). - - `codex-api`: defines `ApiError` that wraps `TransportError` plus API-specific errors parsed from bodies and headers. - - `codex-core`: maps `ApiError` into existing `CodexErr` variants so downstream callers remain unchanged. - - Aggregation strategies (today’s `AggregateStreamExt`) live here as adapters (`Aggregated`, `Streaming`) that transform `ResponseStream` into the higher-level views used by `codex-core`. - -## Implementation Steps - -1. **Create crates**: add `codex-client` and `codex-api` (names keep the `codex-` prefix). Stub lib files with feature flags/tests wired into the workspace; wire them into `Cargo.toml`. -2. **Extract API-level SSE + rate limits into `codex-api`**: - - Move the Responses SSE parser (`process_sse`), rate-limit parsing, and related tests from `core/src/client.rs` into `codex-api`, keeping the behavior identical. - - Introduce `ApiEvent` (initially equivalent to `ResponseEvent`) and `ApiError`, and adjust the parser to emit those. - - Provide test-only helpers for fixture streams (replacement for `CODEX_RS_SSE_FIXTURE`) in `codex-api`. -3. **Lift transport layer into `codex-client`**: - - Move `CodexHttpClient`/`CodexRequestBuilder`, UA/originator plumbing, and backoff helpers from `core/src/default_client.rs` into `codex-client` (or a thin wrapper on top of it). - - Introduce `HttpTransport`, `Request`, `RetryPolicy`, `RetryOn`, and `run_with_retry` as described above. - - Keep sandbox/no-proxy toggles behind injected configuration so `codex-client` stays generic and does not depend on Codex-specific env vars. -4. **Model provider abstraction in `codex-api`**: - - Relocate `ModelProviderInfo` (base URL, env/header resolution, retry knobs, wire API enum) into `codex-api`, expressed in terms of `Provider` and `AuthProvider`. - - Ensure provider logic handles: - - URL building for Responses/Chat/Compact (including Azure special cases). - - Static and env-based headers. - - Per-provider retry and idle-timeout settings that map cleanly into `RetryPolicy`/`RetryOn`. -5. **API crate wiring**: - - Move `Prompt`, tool specs, `ResponsesApiRequest`, `TextControls`, and `ResponseEvent/ResponseStream` into `codex-api` under modules (`common`, `responses`, `chat`, `compact`), keeping public types stable or re-exported through `codex-core` as needed. - - Rebuild Responses and Chat clients on top of `HttpTransport` + `StreamClient`, reusing shared retry + SSE helpers; keep aggregation adapters as reusable strategies instead of `ModelClient`-local logic. - - Implement Compact on top of `UnaryClient` and the unary `execute` path with JSON deserialization, sharing the same retry policy. - - Keep request builders symmetric: each client prepares a `Request`, attaches headers/auth via `AuthProvider`, and plugs in its parser (streaming clients) or deserializer (unary) while sharing retry/backoff configuration derived from `Provider`. -6. **Core integration layer**: - - Replace `core::ModelClient` internals with thin adapters that construct `codex-api` clients using `Config`, `AuthManager`, and `OtelEventManager`. - - Keep the public `ModelClient` API and `ResponseEvent`/`ResponseStream` types stable by re-exporting `codex-api` types or providing type aliases. - - Preserve existing auth flows (including ChatGPT token refresh) inside `codex-core` or a thin adapter, using `AuthProvider` to surface bearer tokens to `codex-api` and handling 401/refresh semantics at this layer. -7. **Tests/migration**: - - Move unit tests for SSE parsing, retry/backoff decisions, and provider/header behavior into the new crates; keep integration tests in `core` using the compatibility layer. - - Update fixtures to be consumed via test-only adapters in `codex-api`. - - Run targeted `just fmt`, `just fix -p` for the touched crates, and scoped `cargo test -p codex-client`, `-p codex-api`, and existing `codex-core` suites. - -## Design Decisions - -- **UA construction** - - `codex-client` exposes an optional UA suffix/provider hook (tiny feature) and remains unaware of the CLI; `codex-core` / the CLI compute the full UA (including `terminal::user_agent()`) and pass the suffix or builder down. -- **Config vs provider** - - Most configuration stays in `codex-core`. `codex-api::Provider` only contains what is strictly required for HTTP (base URLs, query params, retry/timeout knobs, wire API), while higher-level knobs (reasoning defaults, verbosity flags, etc.) remain core concerns. -- **Auth flow ownership** - - Auth flows (including ChatGPT token refresh) remain in `codex-core`. `AuthProvider` simply exposes already-fresh tokens/account IDs; 401 handling and refresh retries stay in the existing auth layer. -- **Error enums** - - `codex-client` continues to define `TransportError` / `StreamError`. `codex-api` defines an `ApiError` (deriving `thiserror::Error`) that wraps `TransportError` and API-specific failures, and `codex-core` maps `ApiError` into existing `CodexErr` variants for callers. -- **Streaming reconnection semantics** - - For now, mid-stream SSE failures are surfaced as errors and only the initial connection is retried via `run_with_retry`. We will revisit mid-stream reconnect/resume once the underlying APIs support cursor/idempotent event semantics. - From 8b314e2d04fef9012ecc880999436e3042ab9897 Mon Sep 17 00:00:00 2001 From: lionel-oai Date: Wed, 26 Nov 2025 18:35:08 +0100 Subject: [PATCH 41/61] doc: fix relative links and add tips (#7319) This PR is a documentation only one which: - addresses the #7231 by adding a paragraph in `docs/getting-started.md` in the tips category to encourage users to load everything needed in their environment - corrects link referencing in `docs/platform-sandboxing.md` so that the page link opens at the right section - removes the explicit heading IDs like {#my-id} in `docs/advanced.md` which are not supported by GitHub and are **not** rendered in the UI: Screenshot 2025-11-26 at 16 25 31 This caused the following links in `README.md` to not work in `main` but to work in this branch (you can test by going to https://github.com/openai/codex/blob/docs/getting-started-enhancement/README.md) - the MCP link goes straight to the correct section now: ```markdown - [**Advanced**](./docs/advanced.md) - [Tracing / verbose logging](./docs/advanced.md#tracing--verbose-logging) - [Model Context Protocol (MCP)](./docs/advanced.md#model-context-protocol-mcp) ``` --------- Signed-off-by: lionel-oai Signed-off-by: lionelchg Co-authored-by: lionelchg --- docs/advanced.md | 12 ++++++------ docs/getting-started.md | 4 ++++ docs/platform-sandboxing.md | 2 +- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/advanced.md b/docs/advanced.md index 1d69e4d54f..26ffca8a92 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -2,11 +2,11 @@ If you already lean on Codex every day and just need a little more control, this page collects the knobs you are most likely to reach for: tweak defaults in [Config](./config.md), add extra tools through [Model Context Protocol support](#model-context-protocol), and script full runs with [`codex exec`](./exec.md). Jump to the section you need and keep building. -## Config quickstart {#config-quickstart} +## Config quickstart Most day-to-day tuning lives in `config.toml`: set approval + sandbox presets, pin model defaults, and add MCP server launchers. The [Config guide](./config.md) walks through every option and provides copy-paste examples for common setups. -## Tracing / verbose logging {#tracing-verbose-logging} +## Tracing / verbose logging Because Codex is written in Rust, it honors the `RUST_LOG` environment variable to configure its logging behavior. @@ -20,15 +20,15 @@ By comparison, the non-interactive mode (`codex exec`) defaults to `RUST_LOG=err See the Rust documentation on [`RUST_LOG`](https://docs.rs/env_logger/latest/env_logger/#enabling-logging) for more information on the configuration options. -## Model Context Protocol (MCP) {#model-context-protocol} +## Model Context Protocol (MCP) The Codex CLI and IDE extension is a MCP client which means that it can be configured to connect to MCP servers. For more information, refer to the [`config docs`](./config.md#mcp-integration). -## Using Codex as an MCP Server {#mcp-server} +## Using Codex as an MCP Server The Codex CLI can also be run as an MCP _server_ via `codex mcp-server`. For example, you can use `codex mcp-server` to make Codex available as a tool inside of a multi-agent framework like the OpenAI [Agents SDK](https://platform.openai.com/docs/guides/agents). Use `codex mcp` separately to add/list/get/remove MCP server launchers in your configuration. -### Codex MCP Server Quickstart {#mcp-server-quickstart} +### Codex MCP Server Quickstart You can launch a Codex MCP server with the [Model Context Protocol Inspector](https://modelcontextprotocol.io/legacy/tools/inspector): @@ -58,7 +58,7 @@ Send a `tools/list` request and you will see that there are two tools available: | **`prompt`** (required) | string | The next user prompt to continue the Codex conversation. | | **`conversationId`** (required) | string | The id of the conversation to continue. | -### Trying it Out {#mcp-server-trying-it-out} +### Trying it Out > [!TIP] > Codex often takes a few minutes to run. To accommodate this, adjust the MCP inspector's Request and Total timeouts to 600000ms (10 minutes) under ⛭ Configuration. diff --git a/docs/getting-started.md b/docs/getting-started.md index 83c66c5724..e06a43ad6b 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -113,3 +113,7 @@ Paste images directly into the composer (Ctrl+V / Cmd+V) to attach them to your codex -i screenshot.png "Explain this error" codex --image img1.png,img2.jpg "Summarize these diagrams" ``` + +#### Environment variables and executables + +Make sure your environment is already set up before launching Codex so it does not spend tokens probing what to activate. For example, source your Python virtualenv (or other language runtimes), start any required daemons, and export the env vars you expect to use ahead of time. diff --git a/docs/platform-sandboxing.md b/docs/platform-sandboxing.md index 39ffce67c3..e635520401 100644 --- a/docs/platform-sandboxing.md +++ b/docs/platform-sandboxing.md @@ -1,3 +1,3 @@ ## Platform sandboxing -This content now lives alongside the rest of the sandbox guidance. See [Sandbox mechanics by platform](./sandbox.md#platform-sandboxing-details) for up-to-date details. +This content now lives alongside the rest of the sandbox guidance. See [Sandbox mechanics by platform](./sandbox.md#sandbox-mechanics-by-platform) for up-to-date details. From fabdbfef9c0881d9a5eba1cc87ffd471a21330a6 Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Wed, 26 Nov 2025 11:52:13 -0600 Subject: [PATCH 42/61] Fixes two bugs in example-config.md documentation (#7324) This PR is a modified version of [a PR](https://github.com/openai/codex/pull/7316) submitted by @yydrowz3. * Removes a redundant `experimental_sandbox_command_assessment` flag * Moves `mcp_oauth_credentials_store` from the `[features]` table, where it doesn't belong --- docs/example-config.md | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/docs/example-config.md b/docs/example-config.md index 97ed22a2b7..bd220a41e6 100644 --- a/docs/example-config.md +++ b/docs/example-config.md @@ -180,6 +180,9 @@ chatgpt_base_url = "https://chatgpt.com/backend-api/" # Allowed values: chatgpt | api # forced_login_method = "chatgpt" +# Preferred store for MCP OAuth credentials: auto (default) | file | keyring +mcp_oauth_credentials_store = "auto" + ################################################################################ # Project Documentation Controls ################################################################################ @@ -232,16 +235,6 @@ experimental_use_rmcp_client = false # Include apply_patch via freeform editing path (affects default tool set). Default: false experimental_use_freeform_apply_patch = false -# Enable model-based sandbox command assessment. Default: false -experimental_sandbox_command_assessment = false - -################################################################################ -# MCP (Model Context Protocol) servers -################################################################################ - -# Preferred store for MCP OAuth credentials: auto (default) | file | keyring -mcp_oauth_credentials_store = "auto" - # Define MCP servers under this table. Leave empty to disable. [mcp_servers] From cbd7d0d54330443887852b21636c816f60f1bde8 Mon Sep 17 00:00:00 2001 From: Job Chong Date: Thu, 27 Nov 2025 16:20:33 +0800 Subject: [PATCH 43/61] chore: improve rollout session init errors (#7336) Title: Improve rollout session initialization error messages Issue: https://github.com/openai/codex/issues/7283 What: add targeted mapping for rollout/session initialization errors so users get actionable messages when Codex cannot access session files. Why: session creation previously returned a generic internal error, hiding permissions/FS issues and making support harder. How: - Added rollout::error::map_session_init_error to translate the more common io::Error kinds into user-facing hints (permission, missing dir, file blocking, corruption). Others are passed through directly with `CodexErr::Fatal`. - Reused the mapper in Codex session creation to preserve root causes instead of returning InternalAgentDied. --- codex-rs/core/src/codex.rs | 5 +-- codex-rs/core/src/rollout/error.rs | 49 ++++++++++++++++++++++++++++++ codex-rs/core/src/rollout/mod.rs | 2 ++ 3 files changed, 54 insertions(+), 2 deletions(-) create mode 100644 codex-rs/core/src/rollout/error.rs diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index e3c5de5762..4b4daf8f19 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -102,6 +102,7 @@ use crate::protocol::TurnDiffEvent; use crate::protocol::WarningEvent; use crate::rollout::RolloutRecorder; use crate::rollout::RolloutRecorderParams; +use crate::rollout::map_session_init_error; use crate::shell; use crate::state::ActiveTurn; use crate::state::SessionServices; @@ -206,7 +207,7 @@ impl Codex { .await .map_err(|e| { error!("Failed to create session: {e:#}"); - CodexErr::InternalAgentDied + map_session_init_error(&e, &config.codex_home) })?; let conversation_id = session.conversation_id; @@ -508,7 +509,7 @@ impl Session { let rollout_recorder = rollout_recorder.map_err(|e| { error!("failed to initialize rollout recorder: {e:#}"); - anyhow::anyhow!("failed to initialize rollout recorder: {e:#}") + anyhow::Error::from(e) })?; let rollout_path = rollout_recorder.rollout_path.clone(); diff --git a/codex-rs/core/src/rollout/error.rs b/codex-rs/core/src/rollout/error.rs new file mode 100644 index 0000000000..e924dd2d28 --- /dev/null +++ b/codex-rs/core/src/rollout/error.rs @@ -0,0 +1,49 @@ +use std::io::ErrorKind; +use std::path::Path; + +use crate::error::CodexErr; +use crate::rollout::SESSIONS_SUBDIR; + +pub(crate) fn map_session_init_error(err: &anyhow::Error, codex_home: &Path) -> CodexErr { + if let Some(mapped) = err + .chain() + .filter_map(|cause| cause.downcast_ref::()) + .find_map(|io_err| map_rollout_io_error(io_err, codex_home)) + { + return mapped; + } + + CodexErr::Fatal(format!("Failed to initialize session: {err:#}")) +} + +fn map_rollout_io_error(io_err: &std::io::Error, codex_home: &Path) -> Option { + let sessions_dir = codex_home.join(SESSIONS_SUBDIR); + let hint = match io_err.kind() { + ErrorKind::PermissionDenied => format!( + "Codex cannot access session files at {} (permission denied). If sessions were created using sudo, fix ownership: sudo chown -R $(whoami) {}", + sessions_dir.display(), + codex_home.display() + ), + ErrorKind::NotFound => format!( + "Session storage missing at {}. Create the directory or choose a different Codex home.", + sessions_dir.display() + ), + ErrorKind::AlreadyExists => format!( + "Session storage path {} is blocked by an existing file. Remove or rename it so Codex can create sessions.", + sessions_dir.display() + ), + ErrorKind::InvalidData | ErrorKind::InvalidInput => format!( + "Session data under {} looks corrupt or unreadable. Clearing the sessions directory may help (this will remove saved conversations).", + sessions_dir.display() + ), + ErrorKind::IsADirectory | ErrorKind::NotADirectory => format!( + "Session storage path {} has an unexpected type. Ensure it is a directory Codex can use for session files.", + sessions_dir.display() + ), + _ => return None, + }; + + Some(CodexErr::Fatal(format!( + "{hint} (underlying error: {io_err})" + ))) +} diff --git a/codex-rs/core/src/rollout/mod.rs b/codex-rs/core/src/rollout/mod.rs index 23410bd4d0..540d204be3 100644 --- a/codex-rs/core/src/rollout/mod.rs +++ b/codex-rs/core/src/rollout/mod.rs @@ -7,11 +7,13 @@ pub const ARCHIVED_SESSIONS_SUBDIR: &str = "archived_sessions"; pub const INTERACTIVE_SESSION_SOURCES: &[SessionSource] = &[SessionSource::Cli, SessionSource::VSCode]; +pub(crate) mod error; pub mod list; pub(crate) mod policy; pub mod recorder; pub use codex_protocol::protocol::SessionMeta; +pub(crate) use error::map_session_init_error; pub use list::find_conversation_path_by_id_str; pub use recorder::RolloutRecorder; pub use recorder::RolloutRecorderParams; From aaec8abf581892303592786b93a143600cc6af57 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Fri, 28 Nov 2025 11:34:57 +0000 Subject: [PATCH 44/61] feat: detached review (#7292) --- .../src/protocol/common.rs | 2 +- .../app-server-protocol/src/protocol/v2.rs | 28 +- codex-rs/app-server/README.md | 41 +-- .../app-server/src/bespoke_event_handling.rs | 46 +++- .../app-server/src/codex_message_processor.rs | 258 ++++++++++++++---- codex-rs/app-server/tests/suite/v2/review.rs | 110 ++++++-- codex-rs/core/src/codex.rs | 47 +--- codex-rs/core/src/context_manager/history.rs | 10 + codex-rs/core/src/review_format.rs | 27 ++ codex-rs/core/src/tasks/review.rs | 99 +++---- codex-rs/core/tests/suite/codex_delegate.rs | 3 - codex-rs/core/tests/suite/review.rs | 69 +++-- codex-rs/protocol/src/protocol.rs | 9 +- codex-rs/tui/src/chatwidget.rs | 5 - codex-rs/tui/src/chatwidget/tests.rs | 3 - 15 files changed, 529 insertions(+), 228 deletions(-) diff --git a/codex-rs/app-server-protocol/src/protocol/common.rs b/codex-rs/app-server-protocol/src/protocol/common.rs index bc7141f008..441efad7b8 100644 --- a/codex-rs/app-server-protocol/src/protocol/common.rs +++ b/codex-rs/app-server-protocol/src/protocol/common.rs @@ -131,7 +131,7 @@ client_request_definitions! { }, ReviewStart => "review/start" { params: v2::ReviewStartParams, - response: v2::TurnStartResponse, + response: v2::ReviewStartResponse, }, ModelList => "model/list" { diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index 222f3e4efc..0a04e8a69a 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -130,6 +130,12 @@ v2_enum_from_core!( } ); +v2_enum_from_core!( + pub enum ReviewDelivery from codex_protocol::protocol::ReviewDelivery { + Inline, Detached + } +); + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] @@ -908,9 +914,22 @@ pub struct ReviewStartParams { pub thread_id: String, pub target: ReviewTarget, - /// When true, also append the final review message to the original thread. + /// Where to run the review: inline (default) on the current thread or + /// detached on a new thread (returned in `reviewThreadId`). #[serde(default)] - pub append_to_original_thread: bool, + pub delivery: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ReviewStartResponse { + pub turn: Turn, + /// Identifies the thread where the review runs. + /// + /// For inline reviews, this is the original thread id. + /// For detached reviews, this is the id of the new review thread. + pub review_thread_id: String, } #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] @@ -1063,7 +1082,10 @@ pub enum ThreadItem { ImageView { id: String, path: String }, #[serde(rename_all = "camelCase")] #[ts(rename_all = "camelCase")] - CodeReview { id: String, review: String }, + EnteredReviewMode { id: String, review: String }, + #[serde(rename_all = "camelCase")] + #[ts(rename_all = "camelCase")] + ExitedReviewMode { id: String, review: String }, } impl From for ThreadItem { diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index 176cd27339..4293544f1b 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -65,7 +65,7 @@ The JSON-RPC API exposes dedicated methods for managing Codex conversations. Thr - `thread/archive` — move a thread’s rollout file into the archived directory; returns `{}` on success. - `turn/start` — add user input to a thread and begin Codex generation; responds with the initial `turn` object and streams `turn/started`, `item/*`, and `turn/completed` notifications. - `turn/interrupt` — request cancellation of an in-flight turn by `(thread_id, turn_id)`; success is an empty `{}` response and the turn finishes with `status: "interrupted"`. -- `review/start` — kick off Codex’s automated reviewer for a thread; responds like `turn/start` and emits a `item/completed` notification with a `codeReview` item when results are ready. +- `review/start` — kick off Codex’s automated reviewer for a thread; responds like `turn/start` and emits `item/started`/`item/completed` notifications with `enteredReviewMode` and `exitedReviewMode` items, plus a final assistant `agentMessage` containing the review. ### 1) Start or resume a thread @@ -190,49 +190,56 @@ Use `review/start` to run Codex’s reviewer on the currently checked-out projec - `{"type":"baseBranch","branch":"main"}` — diff against the provided branch’s upstream (see prompt for the exact `git merge-base`/`git diff` instructions Codex will run). - `{"type":"commit","sha":"abc1234","title":"Optional subject"}` — review a specific commit. - `{"type":"custom","instructions":"Free-form reviewer instructions"}` — fallback prompt equivalent to the legacy manual review request. -- `appendToOriginalThread` (bool, default `false`) — when `true`, Codex also records a final assistant-style message with the review summary in the original thread. When `false`, only the `codeReview` item is emitted for the review run and no extra message is added to the original thread. +- `delivery` (`"inline"` or `"detached"`, default `"inline"`) — where the review runs: + - `"inline"`: run the review as a new turn on the existing thread. The response’s `reviewThreadId` equals the original `threadId`, and no new `thread/started` notification is emitted. + - `"detached"`: fork a new review thread from the parent conversation and run the review there. The response’s `reviewThreadId` is the id of this new review thread, and the server emits a `thread/started` notification for it before streaming review items. Example request/response: ```json { "method": "review/start", "id": 40, "params": { "threadId": "thr_123", - "appendToOriginalThread": true, + "delivery": "inline", "target": { "type": "commit", "sha": "1234567deadbeef", "title": "Polish tui colors" } } } -{ "id": 40, "result": { "turn": { - "id": "turn_900", - "status": "inProgress", - "items": [ - { "type": "userMessage", "id": "turn_900", "content": [ { "type": "text", "text": "Review commit 1234567: Polish tui colors" } ] } - ], - "error": null -} } } +{ "id": 40, "result": { + "turn": { + "id": "turn_900", + "status": "inProgress", + "items": [ + { "type": "userMessage", "id": "turn_900", "content": [ { "type": "text", "text": "Review commit 1234567: Polish tui colors" } ] } + ], + "error": null + }, + "reviewThreadId": "thr_123" +} } ``` +For a detached review, use `"delivery": "detached"`. The response is the same shape, but `reviewThreadId` will be the id of the new review thread (different from the original `threadId`). The server also emits a `thread/started` notification for that new thread before streaming the review turn. + Codex streams the usual `turn/started` notification followed by an `item/started` -with the same `codeReview` item id so clients can show progress: +with an `enteredReviewMode` item so clients can show progress: ```json { "method": "item/started", "params": { "item": { - "type": "codeReview", + "type": "enteredReviewMode", "id": "turn_900", "review": "current changes" } } } ``` -When the reviewer finishes, the server emits `item/completed` containing the same -`codeReview` item with the final review text: +When the reviewer finishes, the server emits `item/started` and `item/completed` +containing an `exitedReviewMode` item with the final review text: ```json { "method": "item/completed", "params": { "item": { - "type": "codeReview", + "type": "exitedReviewMode", "id": "turn_900", "review": "Looks solid overall...\n\n- Prefer Stylize helpers — app.rs:10-20\n ..." } } } ``` -The `review` string is plain text that already bundles the overall explanation plus a bullet list for each structured finding (matching `ThreadItem::CodeReview` in the generated schema). Use this notification to render the reviewer output in your client. +The `review` string is plain text that already bundles the overall explanation plus a bullet list for each structured finding (matching `ThreadItem::ExitedReviewMode` in the generated schema). Use this notification to render the reviewer output in your client. ## Events (work-in-progress) diff --git a/codex-rs/app-server/src/bespoke_event_handling.rs b/codex-rs/app-server/src/bespoke_event_handling.rs index 24c74ebe43..0c68fe8794 100644 --- a/codex-rs/app-server/src/bespoke_event_handling.rs +++ b/codex-rs/app-server/src/bespoke_event_handling.rs @@ -340,16 +340,26 @@ pub(crate) async fn apply_bespoke_event_handling( .await; } EventMsg::EnteredReviewMode(review_request) => { - let notification = ItemStartedNotification { + let review = review_request.user_facing_hint; + let item = ThreadItem::EnteredReviewMode { + id: event_turn_id.clone(), + review, + }; + let started = ItemStartedNotification { thread_id: conversation_id.to_string(), turn_id: event_turn_id.clone(), - item: ThreadItem::CodeReview { - id: event_turn_id.clone(), - review: review_request.user_facing_hint, - }, + item: item.clone(), }; outgoing - .send_server_notification(ServerNotification::ItemStarted(notification)) + .send_server_notification(ServerNotification::ItemStarted(started)) + .await; + let completed = ItemCompletedNotification { + thread_id: conversation_id.to_string(), + turn_id: event_turn_id.clone(), + item, + }; + outgoing + .send_server_notification(ServerNotification::ItemCompleted(completed)) .await; } EventMsg::ItemStarted(item_started_event) => { @@ -375,21 +385,29 @@ pub(crate) async fn apply_bespoke_event_handling( .await; } EventMsg::ExitedReviewMode(review_event) => { - let review_text = match review_event.review_output { + let review = match review_event.review_output { Some(output) => render_review_output_text(&output), None => REVIEW_FALLBACK_MESSAGE.to_string(), }; - let review_item_id = event_turn_id.clone(); - let notification = ItemCompletedNotification { + let item = ThreadItem::ExitedReviewMode { + id: event_turn_id.clone(), + review, + }; + let started = ItemStartedNotification { thread_id: conversation_id.to_string(), turn_id: event_turn_id.clone(), - item: ThreadItem::CodeReview { - id: review_item_id, - review: review_text, - }, + item: item.clone(), }; outgoing - .send_server_notification(ServerNotification::ItemCompleted(notification)) + .send_server_notification(ServerNotification::ItemStarted(started)) + .await; + let completed = ItemCompletedNotification { + thread_id: conversation_id.to_string(), + turn_id: event_turn_id.clone(), + item, + }; + outgoing + .send_server_notification(ServerNotification::ItemCompleted(completed)) .await; } EventMsg::PatchApplyBegin(patch_begin_event) => { diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index b60f2d94c3..80ec99e81c 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -61,7 +61,9 @@ use codex_app_server_protocol::RemoveConversationSubscriptionResponse; use codex_app_server_protocol::RequestId; use codex_app_server_protocol::ResumeConversationParams; use codex_app_server_protocol::ResumeConversationResponse; +use codex_app_server_protocol::ReviewDelivery as ApiReviewDelivery; use codex_app_server_protocol::ReviewStartParams; +use codex_app_server_protocol::ReviewStartResponse; use codex_app_server_protocol::ReviewTarget; use codex_app_server_protocol::SandboxMode; use codex_app_server_protocol::SendUserMessageParams; @@ -120,6 +122,7 @@ use codex_core::git_info::git_diff_to_remote; use codex_core::parse_cursor; use codex_core::protocol::EventMsg; use codex_core::protocol::Op; +use codex_core::protocol::ReviewDelivery as CoreReviewDelivery; use codex_core::protocol::ReviewRequest; use codex_core::protocol::SessionConfiguredEvent; use codex_core::read_head_for_summary; @@ -253,7 +256,6 @@ impl CodexMessageProcessor { fn review_request_from_target( target: ReviewTarget, - append_to_original_thread: bool, ) -> Result<(ReviewRequest, String), JSONRPCErrorError> { fn invalid_request(message: String) -> JSONRPCErrorError { JSONRPCErrorError { @@ -269,7 +271,6 @@ impl CodexMessageProcessor { ReviewRequest { prompt: "Review the current code changes (staged, unstaged, and untracked files) and provide prioritized findings.".to_string(), user_facing_hint: "current changes".to_string(), - append_to_original_thread, }, "Review uncommitted changes".to_string(), )), @@ -285,7 +286,6 @@ impl CodexMessageProcessor { ReviewRequest { prompt, user_facing_hint: hint, - append_to_original_thread, }, display, )) @@ -314,7 +314,6 @@ impl CodexMessageProcessor { ReviewRequest { prompt, user_facing_hint: hint, - append_to_original_thread, }, display, )) @@ -328,7 +327,6 @@ impl CodexMessageProcessor { ReviewRequest { prompt: trimmed.clone(), user_facing_hint: trimmed.clone(), - append_to_original_thread, }, trimmed, )) @@ -2497,60 +2495,220 @@ impl CodexMessageProcessor { } } - async fn review_start(&self, request_id: RequestId, params: ReviewStartParams) { + fn build_review_turn(turn_id: String, display_text: &str) -> Turn { + let items = if display_text.is_empty() { + Vec::new() + } else { + vec![ThreadItem::UserMessage { + id: turn_id.clone(), + content: vec![V2UserInput::Text { + text: display_text.to_string(), + }], + }] + }; + + Turn { + id: turn_id, + items, + status: TurnStatus::InProgress, + } + } + + async fn emit_review_started( + &self, + request_id: &RequestId, + turn: Turn, + parent_thread_id: String, + review_thread_id: String, + ) { + let response = ReviewStartResponse { + turn: turn.clone(), + review_thread_id, + }; + self.outgoing + .send_response(request_id.clone(), response) + .await; + + let notif = TurnStartedNotification { + thread_id: parent_thread_id, + turn, + }; + self.outgoing + .send_server_notification(ServerNotification::TurnStarted(notif)) + .await; + } + + async fn start_inline_review( + &self, + request_id: &RequestId, + parent_conversation: Arc, + review_request: ReviewRequest, + display_text: &str, + parent_thread_id: String, + ) -> std::result::Result<(), JSONRPCErrorError> { + let turn_id = parent_conversation + .submit(Op::Review { review_request }) + .await; + + match turn_id { + Ok(turn_id) => { + let turn = Self::build_review_turn(turn_id, display_text); + self.emit_review_started( + request_id, + turn, + parent_thread_id.clone(), + parent_thread_id, + ) + .await; + Ok(()) + } + Err(err) => Err(JSONRPCErrorError { + code: INTERNAL_ERROR_CODE, + message: format!("failed to start review: {err}"), + data: None, + }), + } + } + + async fn start_detached_review( + &mut self, + request_id: &RequestId, + parent_conversation_id: ConversationId, + review_request: ReviewRequest, + display_text: &str, + ) -> std::result::Result<(), JSONRPCErrorError> { + let rollout_path = find_conversation_path_by_id_str( + &self.config.codex_home, + &parent_conversation_id.to_string(), + ) + .await + .map_err(|err| JSONRPCErrorError { + code: INTERNAL_ERROR_CODE, + message: format!("failed to locate conversation id {parent_conversation_id}: {err}"), + data: None, + })? + .ok_or_else(|| JSONRPCErrorError { + code: INVALID_REQUEST_ERROR_CODE, + message: format!("no rollout found for conversation id {parent_conversation_id}"), + data: None, + })?; + + let mut config = self.config.as_ref().clone(); + config.model = self.config.review_model.clone(); + + let NewConversation { + conversation_id, + conversation, + session_configured, + .. + } = self + .conversation_manager + .fork_conversation(usize::MAX, config, rollout_path) + .await + .map_err(|err| JSONRPCErrorError { + code: INTERNAL_ERROR_CODE, + message: format!("error creating detached review conversation: {err}"), + data: None, + })?; + + if let Err(err) = self + .attach_conversation_listener(conversation_id, false, ApiVersion::V2) + .await + { + tracing::warn!( + "failed to attach listener for review conversation {}: {}", + conversation_id, + err.message + ); + } + + let rollout_path = conversation.rollout_path(); + let fallback_provider = self.config.model_provider_id.as_str(); + match read_summary_from_rollout(rollout_path.as_path(), fallback_provider).await { + Ok(summary) => { + let thread = summary_to_thread(summary); + let notif = ThreadStartedNotification { thread }; + self.outgoing + .send_server_notification(ServerNotification::ThreadStarted(notif)) + .await; + } + Err(err) => { + tracing::warn!( + "failed to load summary for review conversation {}: {}", + session_configured.session_id, + err + ); + } + } + + let turn_id = conversation + .submit(Op::Review { review_request }) + .await + .map_err(|err| JSONRPCErrorError { + code: INTERNAL_ERROR_CODE, + message: format!("failed to start detached review turn: {err}"), + data: None, + })?; + + let turn = Self::build_review_turn(turn_id, display_text); + let review_thread_id = conversation_id.to_string(); + self.emit_review_started(request_id, turn, review_thread_id.clone(), review_thread_id) + .await; + + Ok(()) + } + + async fn review_start(&mut self, request_id: RequestId, params: ReviewStartParams) { let ReviewStartParams { thread_id, target, - append_to_original_thread, + delivery, } = params; - let (_, conversation) = match self.conversation_from_thread_id(&thread_id).await { - Ok(v) => v, - Err(error) => { - self.outgoing.send_error(request_id, error).await; - return; - } - }; - - let (review_request, display_text) = - match Self::review_request_from_target(target, append_to_original_thread) { - Ok(value) => value, - Err(err) => { - self.outgoing.send_error(request_id, err).await; + let (parent_conversation_id, parent_conversation) = + match self.conversation_from_thread_id(&thread_id).await { + Ok(v) => v, + Err(error) => { + self.outgoing.send_error(request_id, error).await; return; } }; - let turn_id = conversation.submit(Op::Review { review_request }).await; - - match turn_id { - Ok(turn_id) => { - let mut items = Vec::new(); - if !display_text.is_empty() { - items.push(ThreadItem::UserMessage { - id: turn_id.clone(), - content: vec![V2UserInput::Text { text: display_text }], - }); - } - let turn = Turn { - id: turn_id.clone(), - items, - status: TurnStatus::InProgress, - }; - let response = TurnStartResponse { turn: turn.clone() }; - self.outgoing.send_response(request_id, response).await; - - let notif = TurnStartedNotification { thread_id, turn }; - self.outgoing - .send_server_notification(ServerNotification::TurnStarted(notif)) - .await; - } + let (review_request, display_text) = match Self::review_request_from_target(target) { + Ok(value) => value, Err(err) => { - let error = JSONRPCErrorError { - code: INTERNAL_ERROR_CODE, - message: format!("failed to start review: {err}"), - data: None, - }; - self.outgoing.send_error(request_id, error).await; + self.outgoing.send_error(request_id, err).await; + return; + } + }; + + let delivery = delivery.unwrap_or(ApiReviewDelivery::Inline).to_core(); + match delivery { + CoreReviewDelivery::Inline => { + if let Err(err) = self + .start_inline_review( + &request_id, + parent_conversation, + review_request, + display_text.as_str(), + thread_id.clone(), + ) + .await + { + self.outgoing.send_error(request_id, err).await; + } + } + CoreReviewDelivery::Detached => { + if let Err(err) = self + .start_detached_review( + &request_id, + parent_conversation_id, + review_request, + display_text.as_str(), + ) + .await + { + self.outgoing.send_error(request_id, err).await; + } } } } diff --git a/codex-rs/app-server/tests/suite/v2/review.rs b/codex-rs/app-server/tests/suite/v2/review.rs index cdb3acd088..0f424b1d75 100644 --- a/codex-rs/app-server/tests/suite/v2/review.rs +++ b/codex-rs/app-server/tests/suite/v2/review.rs @@ -9,12 +9,13 @@ use codex_app_server_protocol::JSONRPCError; use codex_app_server_protocol::JSONRPCNotification; use codex_app_server_protocol::JSONRPCResponse; use codex_app_server_protocol::RequestId; +use codex_app_server_protocol::ReviewDelivery; use codex_app_server_protocol::ReviewStartParams; +use codex_app_server_protocol::ReviewStartResponse; use codex_app_server_protocol::ReviewTarget; use codex_app_server_protocol::ThreadItem; use codex_app_server_protocol::ThreadStartParams; use codex_app_server_protocol::ThreadStartResponse; -use codex_app_server_protocol::TurnStartResponse; use codex_app_server_protocol::TurnStatus; use serde_json::json; use tempfile::TempDir; @@ -59,7 +60,7 @@ async fn review_start_runs_review_turn_and_emits_code_review_item() -> Result<() let review_req = mcp .send_review_start_request(ReviewStartParams { thread_id: thread_id.clone(), - append_to_original_thread: true, + delivery: Some(ReviewDelivery::Inline), target: ReviewTarget::Commit { sha: "1234567deadbeef".to_string(), title: Some("Tidy UI colors".to_string()), @@ -71,7 +72,15 @@ async fn review_start_runs_review_turn_and_emits_code_review_item() -> Result<() mcp.read_stream_until_response_message(RequestId::Integer(review_req)), ) .await??; - let TurnStartResponse { turn } = to_response::(review_resp)?; + let ReviewStartResponse { + turn, + review_thread_id, + } = to_response::(review_resp)?; + assert_eq!( + review_thread_id, + thread_id.clone(), + "expected inline review to run on parent thread id" + ); let turn_id = turn.id.clone(); assert_eq!(turn.status, TurnStatus::InProgress); assert_eq!(turn.items.len(), 1); @@ -91,23 +100,33 @@ async fn review_start_runs_review_turn_and_emits_code_review_item() -> Result<() mcp.read_stream_until_notification_message("turn/started"), ) .await??; - let item_started: JSONRPCNotification = timeout( - DEFAULT_READ_TIMEOUT, - mcp.read_stream_until_notification_message("item/started"), - ) - .await??; - let started: ItemStartedNotification = - serde_json::from_value(item_started.params.expect("params must be present"))?; - match started.item { - ThreadItem::CodeReview { id, review } => { - assert_eq!(id, turn_id); - assert_eq!(review, "commit 1234567"); + let mut saw_entered_review_mode = false; + for _ in 0..10 { + let item_started: JSONRPCNotification = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("item/started"), + ) + .await??; + let started: ItemStartedNotification = + serde_json::from_value(item_started.params.expect("params must be present"))?; + match started.item { + ThreadItem::EnteredReviewMode { id, review } => { + assert_eq!(id, turn_id); + assert_eq!(review, "commit 1234567"); + saw_entered_review_mode = true; + break; + } + ThreadItem::UserMessage { .. } => continue, + other => panic!("unexpected item/started payload: {other:?}"), } - other => panic!("expected code review item, got {other:?}"), } + assert!( + saw_entered_review_mode, + "did not observe enteredReviewMode item" + ); let mut review_body: Option = None; - for _ in 0..5 { + for _ in 0..10 { let review_notif: JSONRPCNotification = timeout( DEFAULT_READ_TIMEOUT, mcp.read_stream_until_notification_message("item/completed"), @@ -116,12 +135,13 @@ async fn review_start_runs_review_turn_and_emits_code_review_item() -> Result<() let completed: ItemCompletedNotification = serde_json::from_value(review_notif.params.expect("params must be present"))?; match completed.item { - ThreadItem::CodeReview { id, review } => { + ThreadItem::ExitedReviewMode { id, review } => { assert_eq!(id, turn_id); review_body = Some(review); break; } ThreadItem::UserMessage { .. } => continue, + ThreadItem::EnteredReviewMode { .. } => continue, other => panic!("unexpected item/completed payload: {other:?}"), } } @@ -146,7 +166,7 @@ async fn review_start_rejects_empty_base_branch() -> Result<()> { let request_id = mcp .send_review_start_request(ReviewStartParams { thread_id, - append_to_original_thread: true, + delivery: Some(ReviewDelivery::Inline), target: ReviewTarget::BaseBranch { branch: " ".to_string(), }, @@ -167,6 +187,56 @@ async fn review_start_rejects_empty_base_branch() -> Result<()> { Ok(()) } +#[tokio::test] +async fn review_start_with_detached_delivery_returns_new_thread_id() -> Result<()> { + let review_payload = json!({ + "findings": [], + "overall_correctness": "ok", + "overall_explanation": "detached review", + "overall_confidence_score": 0.5 + }) + .to_string(); + let responses = vec![create_final_assistant_message_sse_response( + &review_payload, + )?]; + let server = create_mock_chat_completions_server_unchecked(responses).await; + + let codex_home = TempDir::new()?; + create_config_toml(codex_home.path(), &server.uri())?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let thread_id = start_default_thread(&mut mcp).await?; + + let review_req = mcp + .send_review_start_request(ReviewStartParams { + thread_id: thread_id.clone(), + delivery: Some(ReviewDelivery::Detached), + target: ReviewTarget::Custom { + instructions: "detached review".to_string(), + }, + }) + .await?; + let review_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(review_req)), + ) + .await??; + let ReviewStartResponse { + turn, + review_thread_id, + } = to_response::(review_resp)?; + + assert_eq!(turn.status, TurnStatus::InProgress); + assert_ne!( + review_thread_id, thread_id, + "detached review should run on a different thread" + ); + + Ok(()) +} + #[tokio::test] async fn review_start_rejects_empty_commit_sha() -> Result<()> { let server = create_mock_chat_completions_server_unchecked(vec![]).await; @@ -180,7 +250,7 @@ async fn review_start_rejects_empty_commit_sha() -> Result<()> { let request_id = mcp .send_review_start_request(ReviewStartParams { thread_id, - append_to_original_thread: true, + delivery: Some(ReviewDelivery::Inline), target: ReviewTarget::Commit { sha: "\t".to_string(), title: None, @@ -215,7 +285,7 @@ async fn review_start_rejects_empty_custom_instructions() -> Result<()> { let request_id = mcp .send_review_start_request(ReviewStartParams { thread_id, - append_to_original_thread: true, + delivery: Some(ReviewDelivery::Inline), target: ReviewTarget::Custom { instructions: "\n\n".to_string(), }, diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 4b4daf8f19..a3d21ba8ca 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -1190,22 +1190,17 @@ impl Session { } } - /// Record a user input item to conversation history and also persist a - /// corresponding UserMessage EventMsg to rollout. - async fn record_input_and_rollout_usermsg( + pub(crate) async fn record_response_item_and_emit_turn_item( &self, turn_context: &TurnContext, - response_input: &ResponseInputItem, + response_item: ResponseItem, ) { - let response_item: ResponseItem = response_input.clone().into(); - // Add to conversation history and persist response item to rollout + // Add to conversation history and persist response item to rollout. self.record_conversation_items(turn_context, std::slice::from_ref(&response_item)) .await; - // Derive user message events and persist only UserMessage to rollout - let turn_item = parse_turn_item(&response_item); - - if let Some(item @ TurnItem::UserMessage(_)) = turn_item { + // Derive a turn item and emit lifecycle events if applicable. + if let Some(item) = parse_turn_item(&response_item) { self.emit_turn_item_started(turn_context, &item).await; self.emit_turn_item_completed(turn_context, item).await; } @@ -1881,12 +1876,7 @@ async fn spawn_review_thread( text: review_prompt, }]; let tc = Arc::new(review_turn_context); - sess.spawn_task( - tc.clone(), - input, - ReviewTask::new(review_request.append_to_original_thread), - ) - .await; + sess.spawn_task(tc.clone(), input, ReviewTask::new()).await; // Announce entering review mode so UIs can switch modes. sess.send_event(&tc, EventMsg::EnteredReviewMode(review_request)) @@ -1922,7 +1912,8 @@ pub(crate) async fn run_task( sess.send_event(&turn_context, event).await; let initial_input_for_turn: ResponseInputItem = ResponseInputItem::from(input); - sess.record_input_and_rollout_usermsg(turn_context.as_ref(), &initial_input_for_turn) + let response_item: ResponseItem = initial_input_for_turn.clone().into(); + sess.record_response_item_and_emit_turn_item(turn_context.as_ref(), response_item) .await; sess.maybe_start_ghost_snapshot(Arc::clone(&turn_context), cancellation_token.child_token()) @@ -2887,7 +2878,7 @@ mod tests { let input = vec![UserInput::Text { text: "start review".to_string(), }]; - sess.spawn_task(Arc::clone(&tc), input, ReviewTask::new(true)) + sess.spawn_task(Arc::clone(&tc), input, ReviewTask::new()) .await; sess.abort_all_tasks(TurnAbortReason::Interrupted).await; @@ -2915,6 +2906,8 @@ mod tests { .expect("event"); match evt.msg { EventMsg::RawResponseItem(_) => continue, + EventMsg::ItemStarted(_) | EventMsg::ItemCompleted(_) => continue, + EventMsg::AgentMessage(_) => continue, EventMsg::TurnAborted(e) => { assert_eq!(TurnAbortReason::Interrupted, e.reason); break; @@ -2924,23 +2917,7 @@ mod tests { } let history = sess.clone_history().await.get_history(); - let found = history.iter().any(|item| match item { - ResponseItem::Message { role, content, .. } if role == "user" => { - content.iter().any(|ci| match ci { - ContentItem::InputText { text } => { - text.contains("") - && text.contains("review") - && text.contains("interrupted") - } - _ => false, - }) - } - _ => false, - }); - assert!( - found, - "synthetic review interruption not recorded in history" - ); + let _ = history; } #[tokio::test] diff --git a/codex-rs/core/src/context_manager/history.rs b/codex-rs/core/src/context_manager/history.rs index 7277f0b0c6..a1b81ca8e7 100644 --- a/codex-rs/core/src/context_manager/history.rs +++ b/codex-rs/core/src/context_manager/history.rs @@ -71,6 +71,7 @@ impl ContextManager { // With extra response items filtered out and GhostCommits removed. pub(crate) fn get_history_for_prompt(&mut self) -> Vec { let mut history = self.get_history(); + history.retain(|item| !is_review_rollout_item(item)); Self::remove_ghost_snapshots(&mut history); history } @@ -249,6 +250,15 @@ fn is_api_message(message: &ResponseItem) -> bool { } } +fn is_review_rollout_item(item: &ResponseItem) -> bool { + matches!(item, + ResponseItem::Message { + id: Some(id), + .. + } if id.starts_with("review:rollout:") + ) +} + fn estimate_reasoning_length(encoded_len: usize) -> usize { encoded_len .saturating_mul(3) diff --git a/codex-rs/core/src/review_format.rs b/codex-rs/core/src/review_format.rs index 6a64d06fa1..10f53066a7 100644 --- a/codex-rs/core/src/review_format.rs +++ b/codex-rs/core/src/review_format.rs @@ -1,4 +1,5 @@ use crate::protocol::ReviewFinding; +use crate::protocol::ReviewOutputEvent; // Note: We keep this module UI-agnostic. It returns plain strings that // higher layers (e.g., TUI) may style as needed. @@ -10,6 +11,8 @@ fn format_location(item: &ReviewFinding) -> String { format!("{path}:{start}-{end}") } +const REVIEW_FALLBACK_MESSAGE: &str = "Reviewer failed to output a response."; + /// Format a full review findings block as plain text lines. /// /// - When `selection` is `Some`, each item line includes a checkbox marker: @@ -53,3 +56,27 @@ pub fn format_review_findings_block( lines.join("\n") } + +/// Render a human-readable review summary suitable for a user-facing message. +/// +/// Returns either the explanation, the formatted findings block, or both +/// separated by a blank line. If neither is present, emits a fallback message. +pub fn render_review_output_text(output: &ReviewOutputEvent) -> String { + let mut sections = Vec::new(); + let explanation = output.overall_explanation.trim(); + if !explanation.is_empty() { + sections.push(explanation.to_string()); + } + if !output.findings.is_empty() { + let findings = format_review_findings_block(&output.findings, None); + let trimmed = findings.trim(); + if !trimmed.is_empty() { + sections.push(trimmed.to_string()); + } + } + if sections.is_empty() { + REVIEW_FALLBACK_MESSAGE.to_string() + } else { + sections.join("\n\n") + } +} diff --git a/codex-rs/core/src/tasks/review.rs b/codex-rs/core/src/tasks/review.rs index 14a95dba5c..a6ec840a84 100644 --- a/codex-rs/core/src/tasks/review.rs +++ b/codex-rs/core/src/tasks/review.rs @@ -17,6 +17,7 @@ use crate::codex::Session; use crate::codex::TurnContext; use crate::codex_delegate::run_codex_conversation_one_shot; use crate::review_format::format_review_findings_block; +use crate::review_format::render_review_output_text; use crate::state::TaskKind; use codex_protocol::user_input::UserInput; @@ -24,15 +25,11 @@ use super::SessionTask; use super::SessionTaskContext; #[derive(Clone, Copy)] -pub(crate) struct ReviewTask { - append_to_original_thread: bool, -} +pub(crate) struct ReviewTask; impl ReviewTask { - pub(crate) fn new(append_to_original_thread: bool) -> Self { - Self { - append_to_original_thread, - } + pub(crate) fn new() -> Self { + Self } } @@ -62,25 +59,13 @@ impl SessionTask for ReviewTask { None => None, }; if !cancellation_token.is_cancelled() { - exit_review_mode( - session.clone_session(), - output.clone(), - ctx.clone(), - self.append_to_original_thread, - ) - .await; + exit_review_mode(session.clone_session(), output.clone(), ctx.clone()).await; } None } async fn abort(&self, session: Arc, ctx: Arc) { - exit_review_mode( - session.clone_session(), - None, - ctx, - self.append_to_original_thread, - ) - .await; + exit_review_mode(session.clone_session(), None, ctx).await; } } @@ -197,39 +182,57 @@ pub(crate) async fn exit_review_mode( session: Arc, review_output: Option, ctx: Arc, - append_to_original_thread: bool, ) { - if append_to_original_thread { - let user_message = if let Some(out) = review_output.clone() { - let mut findings_str = String::new(); - let text = out.overall_explanation.trim(); - if !text.is_empty() { - findings_str.push_str(text); - } - if !out.findings.is_empty() { - let block = format_review_findings_block(&out.findings, None); - findings_str.push_str(&format!("\n{block}")); - } - crate::client_common::REVIEW_EXIT_SUCCESS_TMPL.replace("{results}", &findings_str) - } else { - crate::client_common::REVIEW_EXIT_INTERRUPTED_TMPL.to_string() - }; + const REVIEW_USER_MESSAGE_ID: &str = "review:rollout:user"; + const REVIEW_ASSISTANT_MESSAGE_ID: &str = "review:rollout:assistant"; + let (user_message, assistant_message) = if let Some(out) = review_output.clone() { + let mut findings_str = String::new(); + let text = out.overall_explanation.trim(); + if !text.is_empty() { + findings_str.push_str(text); + } + if !out.findings.is_empty() { + let block = format_review_findings_block(&out.findings, None); + findings_str.push_str(&format!("\n{block}")); + } + let rendered = + crate::client_common::REVIEW_EXIT_SUCCESS_TMPL.replace("{results}", &findings_str); + let assistant_message = render_review_output_text(&out); + (rendered, assistant_message) + } else { + let rendered = crate::client_common::REVIEW_EXIT_INTERRUPTED_TMPL.to_string(); + let assistant_message = + "Review was interrupted. Please re-run /review and wait for it to complete." + .to_string(); + (rendered, assistant_message) + }; - session - .record_conversation_items( - &ctx, - &[ResponseItem::Message { - id: None, - role: "user".to_string(), - content: vec![ContentItem::InputText { text: user_message }], - }], - ) - .await; - } + session + .record_conversation_items( + &ctx, + &[ResponseItem::Message { + id: Some(REVIEW_USER_MESSAGE_ID.to_string()), + role: "user".to_string(), + content: vec![ContentItem::InputText { text: user_message }], + }], + ) + .await; session .send_event( ctx.as_ref(), EventMsg::ExitedReviewMode(ExitedReviewModeEvent { review_output }), ) .await; + session + .record_response_item_and_emit_turn_item( + ctx.as_ref(), + ResponseItem::Message { + id: Some(REVIEW_ASSISTANT_MESSAGE_ID.to_string()), + role: "assistant".to_string(), + content: vec![ContentItem::OutputText { + text: assistant_message, + }], + }, + ) + .await; } diff --git a/codex-rs/core/tests/suite/codex_delegate.rs b/codex-rs/core/tests/suite/codex_delegate.rs index ca9dc35f00..31073cd669 100644 --- a/codex-rs/core/tests/suite/codex_delegate.rs +++ b/codex-rs/core/tests/suite/codex_delegate.rs @@ -70,7 +70,6 @@ async fn codex_delegate_forwards_exec_approval_and_proceeds_on_approval() { review_request: ReviewRequest { prompt: "Please review".to_string(), user_facing_hint: "review".to_string(), - append_to_original_thread: true, }, }) .await @@ -146,7 +145,6 @@ async fn codex_delegate_forwards_patch_approval_and_proceeds_on_decision() { review_request: ReviewRequest { prompt: "Please review".to_string(), user_facing_hint: "review".to_string(), - append_to_original_thread: true, }, }) .await @@ -201,7 +199,6 @@ async fn codex_delegate_ignores_legacy_deltas() { review_request: ReviewRequest { prompt: "Please review".to_string(), user_facing_hint: "review".to_string(), - append_to_original_thread: true, }, }) .await diff --git a/codex-rs/core/tests/suite/review.rs b/codex-rs/core/tests/suite/review.rs index 3904f18f8e..562cbf0ddf 100644 --- a/codex-rs/core/tests/suite/review.rs +++ b/codex-rs/core/tests/suite/review.rs @@ -18,6 +18,7 @@ use codex_core::protocol::ReviewOutputEvent; use codex_core::protocol::ReviewRequest; use codex_core::protocol::RolloutItem; use codex_core::protocol::RolloutLine; +use codex_core::review_format::render_review_output_text; use codex_protocol::user_input::UserInput; use core_test_support::load_default_config_for_test; use core_test_support::load_sse_fixture_with_id_from_str; @@ -82,7 +83,6 @@ async fn review_op_emits_lifecycle_and_review_output() { review_request: ReviewRequest { prompt: "Please review my changes".to_string(), user_facing_hint: "my changes".to_string(), - append_to_original_thread: true, }, }) .await @@ -124,22 +124,36 @@ async fn review_op_emits_lifecycle_and_review_output() { let mut saw_header = false; let mut saw_finding_line = false; + let expected_assistant_text = render_review_output_text(&expected); + let mut saw_assistant_plain = false; + let mut saw_assistant_xml = false; for line in text.lines() { if line.trim().is_empty() { continue; } let v: serde_json::Value = serde_json::from_str(line).expect("jsonl line"); let rl: RolloutLine = serde_json::from_value(v).expect("rollout line"); - if let RolloutItem::ResponseItem(ResponseItem::Message { role, content, .. }) = rl.item - && role == "user" - { - for c in content { - if let ContentItem::InputText { text } = c { - if text.contains("full review output from reviewer model") { - saw_header = true; + if let RolloutItem::ResponseItem(ResponseItem::Message { role, content, .. }) = rl.item { + if role == "user" { + for c in content { + if let ContentItem::InputText { text } = c { + if text.contains("full review output from reviewer model") { + saw_header = true; + } + if text.contains("- Prefer Stylize helpers — /tmp/file.rs:10-20") { + saw_finding_line = true; + } } - if text.contains("- Prefer Stylize helpers — /tmp/file.rs:10-20") { - saw_finding_line = true; + } + } else if role == "assistant" { + for c in content { + if let ContentItem::OutputText { text } = c { + if text.contains("") { + saw_assistant_xml = true; + } + if text == expected_assistant_text { + saw_assistant_plain = true; + } } } } @@ -150,6 +164,14 @@ async fn review_op_emits_lifecycle_and_review_output() { saw_finding_line, "formatted finding line missing from rollout" ); + assert!( + saw_assistant_plain, + "assistant review output missing from rollout" + ); + assert!( + !saw_assistant_xml, + "assistant review output contains user_action markup" + ); server.verify().await; } @@ -179,7 +201,6 @@ async fn review_op_with_plain_text_emits_review_fallback() { review_request: ReviewRequest { prompt: "Plain text review".to_string(), user_facing_hint: "plain text review".to_string(), - append_to_original_thread: true, }, }) .await @@ -238,7 +259,6 @@ async fn review_filters_agent_message_related_events() { review_request: ReviewRequest { prompt: "Filter streaming events".to_string(), user_facing_hint: "Filter streaming events".to_string(), - append_to_original_thread: true, }, }) .await @@ -247,7 +267,7 @@ async fn review_filters_agent_message_related_events() { let mut saw_entered = false; let mut saw_exited = false; - // Drain until TaskComplete; assert filtered events never surface. + // Drain until TaskComplete; assert streaming-related events never surface. wait_for_event(&codex, |event| match event { EventMsg::TaskComplete(_) => true, EventMsg::EnteredReviewMode(_) => { @@ -265,12 +285,6 @@ async fn review_filters_agent_message_related_events() { EventMsg::AgentMessageDelta(_) => { panic!("unexpected AgentMessageDelta surfaced during review") } - EventMsg::ItemCompleted(ev) => match &ev.item { - codex_protocol::items::TurnItem::AgentMessage(_) => { - panic!("unexpected ItemCompleted for TurnItem::AgentMessage surfaced during review") - } - _ => false, - }, _ => false, }) .await; @@ -279,8 +293,9 @@ async fn review_filters_agent_message_related_events() { server.verify().await; } -/// When the model returns structured JSON in a review, ensure no AgentMessage -/// is emitted; the UI consumes the structured result via ExitedReviewMode. +/// When the model returns structured JSON in a review, ensure only a single +/// non-streaming AgentMessage is emitted; the UI consumes the structured +/// result via ExitedReviewMode plus a final assistant message. // Windows CI only: bump to 4 workers to prevent SSE/event starvation and test timeouts. #[cfg_attr(windows, tokio::test(flavor = "multi_thread", worker_threads = 4))] #[cfg_attr(not(windows), tokio::test(flavor = "multi_thread", worker_threads = 2))] @@ -323,19 +338,21 @@ async fn review_does_not_emit_agent_message_on_structured_output() { review_request: ReviewRequest { prompt: "check structured".to_string(), user_facing_hint: "check structured".to_string(), - append_to_original_thread: true, }, }) .await .unwrap(); - // Drain events until TaskComplete; ensure none are AgentMessage. + // Drain events until TaskComplete; ensure we only see a final + // AgentMessage (no streaming assistant messages). let mut saw_entered = false; let mut saw_exited = false; + let mut agent_messages = 0; wait_for_event(&codex, |event| match event { EventMsg::TaskComplete(_) => true, EventMsg::AgentMessage(_) => { - panic!("unexpected AgentMessage during review with structured output") + agent_messages += 1; + false } EventMsg::EnteredReviewMode(_) => { saw_entered = true; @@ -348,6 +365,7 @@ async fn review_does_not_emit_agent_message_on_structured_output() { _ => false, }) .await; + assert_eq!(1, agent_messages, "expected exactly one AgentMessage event"); assert!(saw_entered && saw_exited, "missing review lifecycle events"); server.verify().await; @@ -377,7 +395,6 @@ async fn review_uses_custom_review_model_from_config() { review_request: ReviewRequest { prompt: "use custom model".to_string(), user_facing_hint: "use custom model".to_string(), - append_to_original_thread: true, }, }) .await @@ -495,7 +512,6 @@ async fn review_input_isolated_from_parent_history() { review_request: ReviewRequest { prompt: review_prompt.clone(), user_facing_hint: review_prompt.clone(), - append_to_original_thread: true, }, }) .await @@ -608,7 +624,6 @@ async fn review_history_does_not_leak_into_parent_session() { review_request: ReviewRequest { prompt: "Start a review".to_string(), user_facing_hint: "Start a review".to_string(), - append_to_original_thread: true, }, }) .await diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index 60c4dffb2a..0f5541c2d5 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -1256,13 +1256,18 @@ pub struct GitInfo { pub repository_url: Option, } +#[derive(Debug, Clone, Copy, Deserialize, Serialize, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "snake_case")] +pub enum ReviewDelivery { + Inline, + Detached, +} + #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, JsonSchema, TS)] /// Review request sent to the review session. pub struct ReviewRequest { pub prompt: String, pub user_facing_hint: String, - #[serde(default)] - pub append_to_original_thread: bool, } /// Structured review result produced by a child review session. diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index fbabbc300e..d11d983cf1 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -2895,7 +2895,6 @@ impl ChatWidget { review_request: ReviewRequest { prompt: "Review the current code changes (staged, unstaged, and untracked files) and provide prioritized findings.".to_string(), user_facing_hint: "current changes".to_string(), - append_to_original_thread: true, }, })); }, @@ -2952,7 +2951,6 @@ impl ChatWidget { "Review the code changes against the base branch '{branch}'. Start by finding the merge diff between the current branch and {branch}'s upstream e.g. (`git merge-base HEAD \"$(git rev-parse --abbrev-ref \"{branch}@{{upstream}}\")\"`), then run `git diff` against that SHA to see what changes we would merge into the {branch} branch. Provide prioritized, actionable findings." ), user_facing_hint: format!("changes against '{branch}'"), - append_to_original_thread: true, }, })); })], @@ -2993,7 +2991,6 @@ impl ChatWidget { review_request: ReviewRequest { prompt, user_facing_hint: hint, - append_to_original_thread: true, }, })); })], @@ -3028,7 +3025,6 @@ impl ChatWidget { review_request: ReviewRequest { prompt: trimmed.clone(), user_facing_hint: trimmed, - append_to_original_thread: true, }, })); }), @@ -3244,7 +3240,6 @@ pub(crate) fn show_review_commit_picker_with_entries( review_request: ReviewRequest { prompt, user_facing_hint: hint, - append_to_original_thread: true, }, })); })], diff --git a/codex-rs/tui/src/chatwidget/tests.rs b/codex-rs/tui/src/chatwidget/tests.rs index 2eebdaf237..e0a1cc3d3b 100644 --- a/codex-rs/tui/src/chatwidget/tests.rs +++ b/codex-rs/tui/src/chatwidget/tests.rs @@ -155,7 +155,6 @@ fn entered_review_mode_uses_request_hint() { msg: EventMsg::EnteredReviewMode(ReviewRequest { prompt: "Review the latest changes".to_string(), user_facing_hint: "feature branch".to_string(), - append_to_original_thread: true, }), }); @@ -175,7 +174,6 @@ fn entered_review_mode_defaults_to_current_changes_banner() { msg: EventMsg::EnteredReviewMode(ReviewRequest { prompt: "Review the current changes".to_string(), user_facing_hint: "current changes".to_string(), - append_to_original_thread: true, }), }); @@ -243,7 +241,6 @@ fn review_restores_context_window_indicator() { msg: EventMsg::EnteredReviewMode(ReviewRequest { prompt: "Review the latest changes".to_string(), user_facing_hint: "feature branch".to_string(), - append_to_original_thread: true, }), }); From 6eeaf46ac136f6ea0352641a5daea5006195e09f Mon Sep 17 00:00:00 2001 From: jif-oai Date: Fri, 28 Nov 2025 15:29:44 +0000 Subject: [PATCH 45/61] fix: other flaky tests (#7372) --- codex-rs/app-server/tests/suite/v2/review.rs | 32 ++++---------------- 1 file changed, 6 insertions(+), 26 deletions(-) diff --git a/codex-rs/app-server/tests/suite/v2/review.rs b/codex-rs/app-server/tests/suite/v2/review.rs index 0f424b1d75..6da16b53b0 100644 --- a/codex-rs/app-server/tests/suite/v2/review.rs +++ b/codex-rs/app-server/tests/suite/v2/review.rs @@ -76,30 +76,11 @@ async fn review_start_runs_review_turn_and_emits_code_review_item() -> Result<() turn, review_thread_id, } = to_response::(review_resp)?; - assert_eq!( - review_thread_id, - thread_id.clone(), - "expected inline review to run on parent thread id" - ); + assert_eq!(review_thread_id, thread_id.clone()); let turn_id = turn.id.clone(); assert_eq!(turn.status, TurnStatus::InProgress); - assert_eq!(turn.items.len(), 1); - match &turn.items[0] { - ThreadItem::UserMessage { content, .. } => { - assert_eq!(content.len(), 1); - assert!(matches!( - &content[0], - codex_app_server_protocol::UserInput::Text { .. } - )); - } - other => panic!("expected user message, got {other:?}"), - } - let _started: JSONRPCNotification = timeout( - DEFAULT_READ_TIMEOUT, - mcp.read_stream_until_notification_message("turn/started"), - ) - .await??; + // Confirm we see the EnteredReviewMode marker on the main thread. let mut saw_entered_review_mode = false; for _ in 0..10 { let item_started: JSONRPCNotification = timeout( @@ -116,8 +97,7 @@ async fn review_start_runs_review_turn_and_emits_code_review_item() -> Result<() saw_entered_review_mode = true; break; } - ThreadItem::UserMessage { .. } => continue, - other => panic!("unexpected item/started payload: {other:?}"), + _ => continue, } } assert!( @@ -125,6 +105,8 @@ async fn review_start_runs_review_turn_and_emits_code_review_item() -> Result<() "did not observe enteredReviewMode item" ); + // Confirm we see the ExitedReviewMode marker (with review text) + // on the same turn. Ignore any other items the stream surfaces. let mut review_body: Option = None; for _ in 0..10 { let review_notif: JSONRPCNotification = timeout( @@ -140,9 +122,7 @@ async fn review_start_runs_review_turn_and_emits_code_review_item() -> Result<() review_body = Some(review); break; } - ThreadItem::UserMessage { .. } => continue, - ThreadItem::EnteredReviewMode { .. } => continue, - other => panic!("unexpected item/completed payload: {other:?}"), + _ => continue, } } From 457c9fdb87853ce8c54dd3c65ed32b468329b31d Mon Sep 17 00:00:00 2001 From: jif-oai Date: Sun, 30 Nov 2025 20:42:26 +0000 Subject: [PATCH 46/61] chore: better session recycling (#7368) --- codex-rs/core/src/unified_exec/mod.rs | 28 ++++++++++-- .../core/src/unified_exec/session_manager.rs | 45 ++++++++++--------- 2 files changed, 49 insertions(+), 24 deletions(-) diff --git a/codex-rs/core/src/unified_exec/mod.rs b/codex-rs/core/src/unified_exec/mod.rs index 35df80255b..8c33548065 100644 --- a/codex-rs/core/src/unified_exec/mod.rs +++ b/codex-rs/core/src/unified_exec/mod.rs @@ -98,8 +98,26 @@ pub(crate) struct UnifiedExecResponse { #[derive(Default)] pub(crate) struct UnifiedExecSessionManager { - sessions: Mutex>, - used_session_ids: Mutex>, + session_store: Mutex, +} + +// Required for mutex sharing. +#[derive(Default)] +pub(crate) struct SessionStore { + sessions: HashMap, + reserved_sessions_id: HashSet, +} + +impl SessionStore { + fn remove(&mut self, session_id: &str) -> Option { + self.reserved_sessions_id.remove(session_id); + self.sessions.remove(session_id) + } + + pub(crate) fn clear(&mut self) { + self.reserved_sessions_id.clear(); + self.sessions.clear(); + } } struct SessionEntry { @@ -384,9 +402,10 @@ mod tests { session .services .unified_exec_manager - .sessions + .session_store .lock() .await + .sessions .is_empty() ); @@ -425,9 +444,10 @@ mod tests { session .services .unified_exec_manager - .sessions + .session_store .lock() .await + .sessions .is_empty() ); diff --git a/codex-rs/core/src/unified_exec/session_manager.rs b/codex-rs/core/src/unified_exec/session_manager.rs index df8f543212..23044c0f78 100644 --- a/codex-rs/core/src/unified_exec/session_manager.rs +++ b/codex-rs/core/src/unified_exec/session_manager.rs @@ -36,6 +36,7 @@ use crate::truncate::formatted_truncate_text; use super::ExecCommandRequest; use super::MAX_UNIFIED_EXEC_SESSIONS; use super::SessionEntry; +use super::SessionStore; use super::UnifiedExecContext; use super::UnifiedExecError; use super::UnifiedExecResponse; @@ -81,7 +82,7 @@ struct PreparedSessionHandles { impl UnifiedExecSessionManager { pub(crate) async fn allocate_process_id(&self) -> String { loop { - let mut store = self.used_session_ids.lock().await; + let mut store = self.session_store.lock().await; let process_id = if !cfg!(test) && !cfg!(feature = "deterministic_process_ids") { // production mode → random @@ -89,6 +90,7 @@ impl UnifiedExecSessionManager { } else { // test or deterministic mode let next = store + .reserved_sessions_id .iter() .filter_map(|s| s.parse::().ok()) .max() @@ -98,11 +100,11 @@ impl UnifiedExecSessionManager { next.to_string() }; - if store.contains(&process_id) { + if store.reserved_sessions_id.contains(&process_id) { continue; } - store.insert(process_id.clone()); + store.reserved_sessions_id.insert(process_id.clone()); return process_id; } } @@ -331,8 +333,8 @@ impl UnifiedExecSessionManager { } async fn refresh_session_state(&self, process_id: &str) -> SessionStatus { - let mut sessions = self.sessions.lock().await; - let Some(entry) = sessions.get(process_id) else { + let mut store = self.session_store.lock().await; + let Some(entry) = store.sessions.get(process_id) else { return SessionStatus::Unknown; }; @@ -340,7 +342,7 @@ impl UnifiedExecSessionManager { let process_id = entry.process_id.clone(); if entry.session.has_exited() { - let Some(entry) = sessions.remove(&process_id) else { + let Some(entry) = store.remove(&process_id) else { return SessionStatus::Unknown; }; SessionStatus::Exited { @@ -360,12 +362,14 @@ impl UnifiedExecSessionManager { &self, process_id: &str, ) -> Result { - let mut sessions = self.sessions.lock().await; - let entry = sessions - .get_mut(process_id) - .ok_or(UnifiedExecError::UnknownSessionId { - process_id: process_id.to_string(), - })?; + let mut store = self.session_store.lock().await; + let entry = + store + .sessions + .get_mut(process_id) + .ok_or(UnifiedExecError::UnknownSessionId { + process_id: process_id.to_string(), + })?; entry.last_used = Instant::now(); let OutputHandles { output_buffer, @@ -417,9 +421,9 @@ impl UnifiedExecSessionManager { started_at, last_used: started_at, }; - let mut sessions = self.sessions.lock().await; - Self::prune_sessions_if_needed(&mut sessions); - sessions.insert(process_id, entry); + let mut store = self.session_store.lock().await; + Self::prune_sessions_if_needed(&mut store); + store.sessions.insert(process_id, entry); } async fn emit_exec_end_from_entry( @@ -629,18 +633,19 @@ impl UnifiedExecSessionManager { collected } - fn prune_sessions_if_needed(sessions: &mut HashMap) { - if sessions.len() < MAX_UNIFIED_EXEC_SESSIONS { + fn prune_sessions_if_needed(store: &mut SessionStore) { + if store.sessions.len() < MAX_UNIFIED_EXEC_SESSIONS { return; } - let meta: Vec<(String, Instant, bool)> = sessions + let meta: Vec<(String, Instant, bool)> = store + .sessions .iter() .map(|(id, entry)| (id.clone(), entry.last_used, entry.session.has_exited())) .collect(); if let Some(session_id) = Self::session_id_to_prune_from_meta(&meta) { - sessions.remove(&session_id); + store.remove(&session_id); } } @@ -674,7 +679,7 @@ impl UnifiedExecSessionManager { } pub(crate) async fn terminate_all_sessions(&self) { - let mut sessions = self.sessions.lock().await; + let mut sessions = self.session_store.lock().await; sessions.clear(); } } From ebd485b1a009ee2afc39bf302dc81b2ecfae357b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 30 Nov 2025 20:20:45 -0800 Subject: [PATCH 47/61] chore(deps): bump arboard from 3.6.0 to 3.6.1 in /codex-rs (#7426) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [arboard](https://github.com/1Password/arboard) from 3.6.0 to 3.6.1.
Release notes

Sourced from arboard's releases.

v3.6.1

This release focuses on improving compatibility with data in the real world and bug fixes. It also includes a new Set API for working with file paths via drag-and-drop interfaces across Linux, macOS, and Windows.

This release also marks the start of exclusively publishing changelogs via GitHub Releases. The old CHANGELOG.md has been removed due to maintenance overhead and duplication. v3.6.0 is the last revision to include this file.

Added

Changed

  • Fix grammar and typos by @​complexspaces and @​gagath in 1Password/arboard#194 and 1Password/arboard#196
  • Prefer PNG when pasting images on Windows by @​wcassels in 1Password/arboard#198
    • Note: This change greatly increases compatibility for "complicated" images that contain alpha values and/or transparent pixels. Support for transparency in BITMAP formats is ill-defined and inconsistently implemented in the wild, but is consistent in PNG. Most applications loading images onto the clipboard include PNG-encoded data already.
  • Bitmap images pasted on Windows now use the image crate instead of a homegrown internal parser.
    • This should not regress any existing Bitmap use cases and instead will provide more consistent and robust parsing. If you notice something now broken, please open an issue!

Fixed

New Contributors

Full Changelog: https://github.com/1Password/arboard/compare/v3.6.0...v3.6.1

Commits
  • a3750c7 Release 3.6.1
  • edcce2c Remove CHANGELOG.md in favor of GitHub releases
  • 26a96a6 Bump windows-sys semver range to support 0.60.x
  • 7bdd1c1 Update errno for windows-sys 0.60 flexibility
  • 55c0b26 read/write_unaligned rather than using manual field offsets
  • ff15a09 Return conversionFailure instead of adhoc errors
  • 16ef181 Implement fetching PNG on Windows and prefer over DIB when available
  • a3c64f9 Add a couple of end-to-end DIBV5 tests
  • e6008ea Use image for reading DIB and try to make it do the right thing for 32-bit BI...
  • 17ef05c add file_list to Set interface (#181)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=arboard&package-manager=cargo&previous-version=3.6.0&new-version=3.6.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- codex-rs/Cargo.lock | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 4c87b8dc05..36eff3c569 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -198,9 +198,9 @@ dependencies = [ [[package]] name = "arboard" -version = "3.6.0" +version = "3.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55f533f8e0af236ffe5eb979b99381df3258853f00ba2e44b6e1955292c75227" +checksum = "0348a1c054491f4bfe6ab86a7b6ab1e44e45d899005de92f58b3df180b36ddaf" dependencies = [ "clipboard-win", "image", @@ -212,7 +212,7 @@ dependencies = [ "objc2-foundation", "parking_lot", "percent-encoding", - "windows-sys 0.52.0", + "windows-sys 0.60.2", "wl-clipboard-rs", "x11rb", ] @@ -2536,7 +2536,7 @@ checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" dependencies = [ "cfg-if", "rustix 1.0.8", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3440,7 +3440,7 @@ checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -5217,7 +5217,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.4.15", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -7369,7 +7369,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] From c22cd2e953901e2969d55b544ebd8db8e70cacbc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 30 Nov 2025 20:35:32 -0800 Subject: [PATCH 48/61] chore(deps): bump serde_with from 3.14.0 to 3.16.1 in /codex-rs (#7422) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [serde_with](https://github.com/jonasbb/serde_with) from 3.14.0 to 3.16.1.
Release notes

Sourced from serde_with's releases.

serde_with v3.16.1

Fixed

  • Fix JsonSchemaAs of SetPreventDuplicates and SetLastValueWins. (#906, #907)

serde_with v3.16.0

Added

serde_with v3.15.1

Fixed

  • Fix building of the documentation by updating references to use serde_core.

serde_with v3.15.0

Added

  • Added error inspection to VecSkipError and MapSkipError by @​michelhe (#878) This allows interacting with the previously hidden error, for example for logging. Checkout the newly added example to both types.

  • Allow documenting the types generated by serde_conv!. The serde_conv! macro now acceps outer attributes before the optional visibility modifier. This allow adding doc comments in the shape of #[doc = "..."] or any other attributes, such as lint modifiers.

    serde_conv!(
    #[doc = "Serialize bools as string"]
    #[allow(dead_code)]
    pub BoolAsString,
    bool,
    |x: &bool| ::std::string::ToString::to_string(x),
    |x: ::std::string::String| x.parse()
);

  • Add support for hashbrown v0.16 (#877)

    This extends the existing support for hashbrown v0.14 and v0.15 to the newly released version.

Changed

  • Bump MSRV to 1.76, since that is required for toml dev-dependency.

serde_with v3.14.1

Fixed

  • Show macro expansion in the docs.rs generated rustdoc.

... (truncated)

Commits
  • 8513323 Bump version to 3.16.1 (#908)
  • 5392bbe Bump version to 3.16.1
  • 1e54f1c Fix duplicate schema set definitions for schemars 0.8, 0.9, and 1.0 (#907)
  • 0650180 Fix duplicate schema set definitions for schemars 0.8, 0.9, and 1.0
  • 41d1033 Fix test conditions for schemars tests to include "hex" feature
  • 2eed58a Bump the github-actions group across 1 directory with 2 updates (#905)
  • ed040f2 Bump the github-actions group across 1 directory with 2 updates
  • fa2129b Bump ron from 0.11.0 to 0.12.0 (#904)
  • b55cb99 Bump ron from 0.11.0 to 0.12.0
  • 066b9d4 Bump version to 3.16.0 (#903)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=serde_with&package-manager=cargo&previous-version=3.14.0&new-version=3.16.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Eric Traut --- codex-rs/Cargo.lock | 13 ++++++------- codex-rs/Cargo.toml | 2 +- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 36eff3c569..5e86b93fc8 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -5736,9 +5736,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.14.0" +version = "3.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2c45cd61fefa9db6f254525d46e392b852e0e61d9a1fd36e5bd183450a556d5" +checksum = "4fa237f2807440d238e0364a218270b98f767a00d3dada77b1c53ae88940e2e7" dependencies = [ "base64", "chrono", @@ -5747,8 +5747,7 @@ dependencies = [ "indexmap 2.12.0", "schemars 0.9.0", "schemars 1.0.4", - "serde", - "serde_derive", + "serde_core", "serde_json", "serde_with_macros", "time", @@ -5756,11 +5755,11 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.14.0" +version = "3.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de90945e6565ce0d9a25098082ed4ee4002e047cb59892c318d66821e14bb30f" +checksum = "52a8e3ca0ca629121f70ab50f95249e5a6f925cc0f6ffe8256c45b728875706c" dependencies = [ - "darling 0.20.11", + "darling 0.21.3", "proc-macro2", "quote", "syn 2.0.104", diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index 053d79d7e0..1ca55f98ab 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -178,7 +178,7 @@ seccompiler = "0.5.0" sentry = "0.34.0" serde = "1" serde_json = "1" -serde_with = "3.14" +serde_with = "3.16" serial_test = "3.2.0" sha1 = "0.10.6" sha2 = "0.10" From 3f12f1140fd5cde0dab80237072cb3bbea245b66 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 30 Nov 2025 20:35:49 -0800 Subject: [PATCH 49/61] chore(deps): bump reqwest from 0.12.23 to 0.12.24 in /codex-rs (#7424) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [reqwest](https://github.com/seanmonstar/reqwest) from 0.12.23 to 0.12.24.
Release notes

Sourced from reqwest's releases.

v0.12.24

Highlights

  • Refactor cookie handling to an internal middleware.
  • Refactor internal random generator.
  • Refactor base64 encoding to reduce a copy.
  • Documentation updates.

What's Changed

New Contributors

Full Changelog: https://github.com/seanmonstar/reqwest/compare/v0.12.23...v0.12.24

Changelog

Sourced from reqwest's changelog.

v0.12.24

  • Refactor cookie handling to an internal middleware.
  • Refactor internal random generator.
  • Refactor base64 encoding to reduce a copy.
  • Documentation updates.
Commits
  • b126ca4 v0.12.24
  • 4023493 refactor: change fast_random from xorshift to siphash a counter
  • fd61bc9 refactor(cookie): avoid duplicate cookie insertion (#2834)
  • 0bfa526 test(multipart): fix build failure with no-default-features (#2801)
  • 994b8a0 docs: typo in retry max_retries_per_request (#2824)
  • da0702b refactor(cookie): de-duplicate cookie support as CookieService middleware (...
  • 7ebddea chore: align internal name usage of TotalTimeout (#2657)
  • b540a4e chore(readme): use correct CI status badge
  • e4550c4 docs: fix method name in changelog entry (#2807)
  • f4694a2 perf(util): avoid extra copy when base64 encoding (#2805)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=reqwest&package-manager=cargo&previous-version=0.12.23&new-version=0.12.24)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Eric Traut --- codex-rs/Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 5e86b93fc8..b98394b85d 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -5079,9 +5079,9 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "reqwest" -version = "0.12.23" +version = "0.12.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d429f34c8092b2d42c7c93cec323bb4adeb7c67698f70839adec842ec10c7ceb" +checksum = "9d0946410b9f7b082a427e4ef5c8ff541a88b357bc6c637c40db3a68ac70a36f" dependencies = [ "base64", "bytes", From cb85a7b96eba9ee75769db894ca397decb7c2019 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 30 Nov 2025 20:36:03 -0800 Subject: [PATCH 50/61] chore(deps): bump tracing from 0.1.41 to 0.1.43 in /codex-rs (#7428) Bumps [tracing](https://github.com/tokio-rs/tracing) from 0.1.41 to 0.1.43.
Release notes

Sourced from tracing's releases.

tracing 0.1.43

Important

The previous release [0.1.42] was yanked because #3382 was a breaking change. See further details in #3424. This release contains all the changes from that version, plus a revert for the problematic part of the breaking PR.

Fixed

  • Revert "make valueset macro sanitary" (#3425)

#3382: tokio-rs/tracing#3382 #3424: tokio-rs/tracing#3424 #3425: tokio-rs/tracing#3425 [0.1.42]: https://github.com/tokio-rs/tracing/releases/tag/tracing-0.1.42

tracing 0.1.42

Important

The [Span::record_all] method has been removed from the documented API. It was always unsuable via the documented API as it requried a ValueSet which has no publically documented constructors. The method remains, but should not be used outside of tracing macros.

Added

  • attributes: Support constant expressions as instrument field names (#3158)
  • Add record_all! macro for recording multiple values in one call (#3227)
  • core: Improve code generation at trace points significantly (#3398)

Changed

  • tracing-core: updated to 0.1.35 (#3414)
  • tracing-attributes: updated to 0.1.31 (#3417)

Fixed

  • Fix "name / parent" variant of event! (#2983)
  • Remove 'r#' prefix from raw identifiers in field names (#3130)
  • Fix perf regression when release_max_level_* not set (#3373)
  • Use imported instead of fully qualified path (#3374)
  • Make valueset macro sanitary (#3382)

Documented

  • core: Add missing dyn keyword in Visit documentation code sample (#3387)

#2983: tokio-rs/tracing#2983 #3130: tokio-rs/tracing#3130 #3158: tokio-rs/tracing#3158

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=tracing&package-manager=cargo&previous-version=0.1.41&new-version=0.1.43)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Eric Traut --- codex-rs/Cargo.lock | 12 ++++++------ codex-rs/Cargo.toml | 2 +- codex-rs/lmstudio/Cargo.toml | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index b98394b85d..511831a19f 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -6713,9 +6713,9 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" -version = "0.1.41" +version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +checksum = "2d15d90a0b5c19378952d479dc858407149d7bb45a14de0142f6c534b16fc647" dependencies = [ "log", "pin-project-lite", @@ -6737,9 +6737,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", @@ -6748,9 +6748,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" +checksum = "7a04e24fab5c89c6a36eb8558c9656f30d81de51dfa4d3b45f26b21d61fa0a6c" dependencies = [ "once_cell", "valuable", diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index 1ca55f98ab..49f9e9d163 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -203,7 +203,7 @@ tokio-util = "0.7.16" toml = "0.9.5" toml_edit = "0.23.5" tonic = "0.13.1" -tracing = "0.1.41" +tracing = "0.1.43" tracing-appender = "0.2.3" tracing-subscriber = "0.3.20" tracing-test = "0.2.5" diff --git a/codex-rs/lmstudio/Cargo.toml b/codex-rs/lmstudio/Cargo.toml index 597ad3033f..3ac7614c22 100644 --- a/codex-rs/lmstudio/Cargo.toml +++ b/codex-rs/lmstudio/Cargo.toml @@ -14,7 +14,7 @@ codex-core = { path = "../core" } reqwest = { version = "0.12", features = ["json", "stream"] } serde_json = "1" tokio = { version = "1", features = ["rt"] } -tracing = { version = "0.1.41", features = ["log"] } +tracing = { version = "0.1.43", features = ["log"] } which = "6.0" [dev-dependencies] From 14df5c9492827d85e57d78817831990b6c21dc0a Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Sun, 30 Nov 2025 22:45:17 -0600 Subject: [PATCH 51/61] Fixed CLA action to properly exempt dependabot (#7429) --- .github/workflows/cla.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/cla.yml b/.github/workflows/cla.yml index 12ee971c4b..bab34d0365 100644 --- a/.github/workflows/cla.yml +++ b/.github/workflows/cla.yml @@ -46,7 +46,4 @@ jobs: path-to-document: https://github.com/openai/codex/blob/main/docs/CLA.md path-to-signatures: signatures/cla.json branch: cla-signatures - allowlist: | - codex - dependabot - dependabot[bot] + allowlist: codex,dependabot,dependabot[bot],github-actions[bot] From ba581843497505638286fb084f97d825e37dacb0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 30 Nov 2025 20:50:51 -0800 Subject: [PATCH 52/61] chore(deps): bump image from 0.25.8 to 0.25.9 in /codex-rs (#7421) Bumps [image](https://github.com/image-rs/image) from 0.25.8 to 0.25.9.
Changelog

Sourced from image's changelog.

Version 0.25.9

Features:

  • Support extracting XMP metadata from PNG, JPEG, GIF, WebP and TIFF files (#2567, #2634, #2644)
  • Support reading IPTC metadata from PNG and JPG files (#2611)
  • Support reading ICC profile from GIF files (#2644)
  • Allow setting a specific DEFLATE compression level when writing PNG (#2583)
  • Initial support for 16-bit CMYK TIFF files (#2588)
  • Allow extracting the alpha channel of a Pixel in a generic way (#2638)

Structural changes:

  • EXR format decoding now only uses multi-threading via Rayon when the rayon feature is enabled (#2643)
  • Upgraded zune-jpeg to 0.5.x, ravif to 0.12.x, gif to 0.14.x
  • pnm: parse integers in PBM/PGM/PPM headers without allocations (#2620)
  • Replace doc_auto_cfg with doc_cfg (#2637)

Bug fixes:

  • Do not encode empty JPEG images (#2624)
  • tga: reject empty images (#2614)
  • tga: fix orientation flip for color mapped images (#2607)
  • tga: adjust colormap lookup to match tga 2.0 spec (#2608)
Commits
  • 5ceb6af Merge pull request #2640 from Shnatsel/release-v0.25.9
  • 282d7b3 Merge pull request #2646 from oligamiq/main
  • 5412aee Amend the note in accordance with the advice of 197g.
  • 4e8a4ed Clarify default features in README and add usage note
  • ca8fa52 Merge pull request #2644 from image-rs/gif-0.14
  • d9bc8fe mention GIF 0.14 changes
  • 053220a Provide gif's XMP and ICC metadata
  • 2ec20b3 Prepare codec with gif@0.14
  • 31939fa Mention EXR rayon change
  • c7f68be Merge pull request #2643 from Shnatsel/really-optional-rayon
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=image&package-manager=cargo&previous-version=0.25.8&new-version=0.25.9)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- codex-rs/Cargo.lock | 27 +++++++++++++++++++++------ codex-rs/Cargo.toml | 2 +- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 511831a19f..2320afeecb 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -3293,9 +3293,9 @@ dependencies = [ [[package]] name = "image" -version = "0.25.8" +version = "0.25.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "529feb3e6769d234375c4cf1ee2ce713682b8e76538cb13f9fc23e1400a591e7" +checksum = "e6506c6c10786659413faa717ceebcb8f70731c0a60cbae39795fdf114519c1a" dependencies = [ "bytemuck", "byteorder-lite", @@ -3303,8 +3303,8 @@ dependencies = [ "num-traits", "png", "tiff", - "zune-core", - "zune-jpeg", + "zune-core 0.5.0", + "zune-jpeg 0.5.5", ] [[package]] @@ -6408,7 +6408,7 @@ dependencies = [ "half", "quick-error", "weezl", - "zune-jpeg", + "zune-jpeg 0.4.19", ] [[package]] @@ -8093,13 +8093,28 @@ version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f423a2c17029964870cfaabb1f13dfab7d092a62a29a89264f4d36990ca414a" +[[package]] +name = "zune-core" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "111f7d9820f05fd715df3144e254d6fc02ee4088b0644c0ffd0efc9e6d9d2773" + [[package]] name = "zune-jpeg" version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c9e525af0a6a658e031e95f14b7f889976b74a11ba0eca5a5fc9ac8a1c43a6a" dependencies = [ - "zune-core", + "zune-core 0.4.12", +] + +[[package]] +name = "zune-jpeg" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6fb7703e32e9a07fb3f757360338b3a567a5054f21b5f52a666752e333d58e" +dependencies = [ + "zune-core 0.5.0", ] [[package]] diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index 49f9e9d163..44270c4eb4 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -136,7 +136,7 @@ icu_decimal = "2.1" icu_locale_core = "2.1" icu_provider = { version = "2.1", features = ["sync"] } ignore = "0.4.23" -image = { version = "^0.25.8", default-features = false } +image = { version = "^0.25.9", default-features = false } indexmap = "2.12.0" insta = "1.43.2" itertools = "0.14.0" From 40006808a396a35aab7d55cce9c4d551f5cadd55 Mon Sep 17 00:00:00 2001 From: Celia Chen Date: Sun, 30 Nov 2025 21:09:59 -0800 Subject: [PATCH 53/61] [app-server] add `turn/plan/updated` event (#7329) transform `EventMsg::PlanDate` to v2 `turn/plan/updated` event. similar to `turn/diff/updated`. --- .../src/protocol/common.rs | 1 + .../app-server-protocol/src/protocol/v2.rs | 47 +++++++++++ codex-rs/app-server/README.md | 1 + .../app-server/src/bespoke_event_handling.rs | 77 +++++++++++++++++++ 4 files changed, 126 insertions(+) diff --git a/codex-rs/app-server-protocol/src/protocol/common.rs b/codex-rs/app-server-protocol/src/protocol/common.rs index 441efad7b8..f8483e9edb 100644 --- a/codex-rs/app-server-protocol/src/protocol/common.rs +++ b/codex-rs/app-server-protocol/src/protocol/common.rs @@ -506,6 +506,7 @@ server_notification_definitions! { TurnStarted => "turn/started" (v2::TurnStartedNotification), TurnCompleted => "turn/completed" (v2::TurnCompletedNotification), TurnDiffUpdated => "turn/diff/updated" (v2::TurnDiffUpdatedNotification), + TurnPlanUpdated => "turn/plan/updated" (v2::TurnPlanUpdatedNotification), ItemStarted => "item/started" (v2::ItemStartedNotification), ItemCompleted => "item/completed" (v2::ItemCompletedNotification), AgentMessageDelta => "item/agentMessage/delta" (v2::AgentMessageDeltaNotification), diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index 0a04e8a69a..71d38a3dd4 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -11,6 +11,8 @@ use codex_protocol::items::AgentMessageContent as CoreAgentMessageContent; use codex_protocol::items::TurnItem as CoreTurnItem; use codex_protocol::models::ResponseItem; use codex_protocol::parse_command::ParsedCommand as CoreParsedCommand; +use codex_protocol::plan_tool::PlanItemArg as CorePlanItemArg; +use codex_protocol::plan_tool::StepStatus as CorePlanStepStatus; use codex_protocol::protocol::CodexErrorInfo as CoreCodexErrorInfo; use codex_protocol::protocol::CreditsSnapshot as CoreCreditsSnapshot; use codex_protocol::protocol::RateLimitSnapshot as CoreRateLimitSnapshot; @@ -1234,6 +1236,51 @@ pub struct TurnDiffUpdatedNotification { pub diff: String, } +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct TurnPlanUpdatedNotification { + pub turn_id: String, + pub explanation: Option, + pub plan: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct TurnPlanStep { + pub step: String, + pub status: TurnPlanStepStatus, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub enum TurnPlanStepStatus { + Pending, + InProgress, + Completed, +} + +impl From for TurnPlanStep { + fn from(value: CorePlanItemArg) -> Self { + Self { + step: value.step, + status: value.status.into(), + } + } +} + +impl From for TurnPlanStepStatus { + fn from(value: CorePlanStepStatus) -> Self { + match value { + CorePlanStepStatus::Pending => Self::Pending, + CorePlanStepStatus::InProgress => Self::InProgress, + CorePlanStepStatus::Completed => Self::Completed, + } + } +} + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index 4293544f1b..7395419aee 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -251,6 +251,7 @@ The app-server streams JSON-RPC notifications while a turn is running. Each turn - `turn/started` — `{ turn }` with the turn id, empty `items`, and `status: "inProgress"`. - `turn/completed` — `{ turn }` where `turn.status` is `completed`, `interrupted`, or `failed`; failures carry `{ error: { message, codexErrorInfo? } }`. +- `turn/plan/updated` — `{ turnId, explanation?, plan }` whenever the agent shares or changes its plan; each `plan` entry is `{ step, status }` with `status` in `pending`, `inProgress`, or `completed`. Today both notifications carry an empty `items` array even when item events were streamed; rely on `item/*` notifications for the canonical item list until this is fixed. diff --git a/codex-rs/app-server/src/bespoke_event_handling.rs b/codex-rs/app-server/src/bespoke_event_handling.rs index 0c68fe8794..a32a855b92 100644 --- a/codex-rs/app-server/src/bespoke_event_handling.rs +++ b/codex-rs/app-server/src/bespoke_event_handling.rs @@ -43,6 +43,8 @@ use codex_app_server_protocol::TurnCompletedNotification; use codex_app_server_protocol::TurnDiffUpdatedNotification; use codex_app_server_protocol::TurnError; use codex_app_server_protocol::TurnInterruptResponse; +use codex_app_server_protocol::TurnPlanStep; +use codex_app_server_protocol::TurnPlanUpdatedNotification; use codex_app_server_protocol::TurnStatus; use codex_core::CodexConversation; use codex_core::parse_command::shlex_join; @@ -60,6 +62,7 @@ use codex_core::protocol::TokenCountEvent; use codex_core::protocol::TurnDiffEvent; use codex_core::review_format::format_review_findings_block; use codex_protocol::ConversationId; +use codex_protocol::plan_tool::UpdatePlanArgs; use codex_protocol::protocol::ReviewOutputEvent; use std::collections::HashMap; use std::convert::TryFrom; @@ -592,6 +595,15 @@ pub(crate) async fn apply_bespoke_event_handling( ) .await; } + EventMsg::PlanUpdate(plan_update_event) => { + handle_turn_plan_update( + &event_turn_id, + plan_update_event, + api_version, + outgoing.as_ref(), + ) + .await; + } _ => {} } @@ -614,6 +626,28 @@ async fn handle_turn_diff( } } +async fn handle_turn_plan_update( + event_turn_id: &str, + plan_update_event: UpdatePlanArgs, + api_version: ApiVersion, + outgoing: &OutgoingMessageSender, +) { + if let ApiVersion::V2 = api_version { + let notification = TurnPlanUpdatedNotification { + turn_id: event_turn_id.to_string(), + explanation: plan_update_event.explanation, + plan: plan_update_event + .plan + .into_iter() + .map(TurnPlanStep::from) + .collect(), + }; + outgoing + .send_server_notification(ServerNotification::TurnPlanUpdated(notification)) + .await; + } +} + async fn emit_turn_completed_with_status( conversation_id: ConversationId, event_turn_id: String, @@ -1133,12 +1167,15 @@ mod tests { use anyhow::Result; use anyhow::anyhow; use anyhow::bail; + use codex_app_server_protocol::TurnPlanStepStatus; use codex_core::protocol::CreditsSnapshot; use codex_core::protocol::McpInvocation; use codex_core::protocol::RateLimitSnapshot; use codex_core::protocol::RateLimitWindow; use codex_core::protocol::TokenUsage; use codex_core::protocol::TokenUsageInfo; + use codex_protocol::plan_tool::PlanItemArg; + use codex_protocol::plan_tool::StepStatus; use mcp_types::CallToolResult; use mcp_types::ContentBlock; use mcp_types::TextContent; @@ -1298,6 +1335,46 @@ mod tests { Ok(()) } + #[tokio::test] + async fn test_handle_turn_plan_update_emits_notification_for_v2() -> Result<()> { + let (tx, mut rx) = mpsc::channel(CHANNEL_CAPACITY); + let outgoing = OutgoingMessageSender::new(tx); + let update = UpdatePlanArgs { + explanation: Some("need plan".to_string()), + plan: vec![ + PlanItemArg { + step: "first".to_string(), + status: StepStatus::Pending, + }, + PlanItemArg { + step: "second".to_string(), + status: StepStatus::Completed, + }, + ], + }; + + handle_turn_plan_update("turn-123", update, ApiVersion::V2, &outgoing).await; + + let msg = rx + .recv() + .await + .ok_or_else(|| anyhow!("should send one notification"))?; + match msg { + OutgoingMessage::AppServerNotification(ServerNotification::TurnPlanUpdated(n)) => { + assert_eq!(n.turn_id, "turn-123"); + assert_eq!(n.explanation.as_deref(), Some("need plan")); + assert_eq!(n.plan.len(), 2); + assert_eq!(n.plan[0].step, "first"); + assert_eq!(n.plan[0].status, TurnPlanStepStatus::Pending); + assert_eq!(n.plan[1].step, "second"); + assert_eq!(n.plan[1].status, TurnPlanStepStatus::Completed); + } + other => bail!("unexpected message: {other:?}"), + } + assert!(rx.try_recv().is_err(), "no extra messages expected"); + Ok(()) + } + #[tokio::test] async fn test_handle_token_count_event_emits_usage_and_rate_limits() -> Result<()> { let conversation_id = ConversationId::new(); From a421eba31f45d162e20b2c78ac899f78f559e3f6 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Mon, 1 Dec 2025 09:04:13 +0000 Subject: [PATCH 54/61] fix: disable review rollout filtering (#7371) --- codex-rs/core/src/context_manager/history.rs | 10 ------- codex-rs/core/tests/suite/review.rs | 31 ++++++++++++-------- 2 files changed, 18 insertions(+), 23 deletions(-) diff --git a/codex-rs/core/src/context_manager/history.rs b/codex-rs/core/src/context_manager/history.rs index a1b81ca8e7..7277f0b0c6 100644 --- a/codex-rs/core/src/context_manager/history.rs +++ b/codex-rs/core/src/context_manager/history.rs @@ -71,7 +71,6 @@ impl ContextManager { // With extra response items filtered out and GhostCommits removed. pub(crate) fn get_history_for_prompt(&mut self) -> Vec { let mut history = self.get_history(); - history.retain(|item| !is_review_rollout_item(item)); Self::remove_ghost_snapshots(&mut history); history } @@ -250,15 +249,6 @@ fn is_api_message(message: &ResponseItem) -> bool { } } -fn is_review_rollout_item(item: &ResponseItem) -> bool { - matches!(item, - ResponseItem::Message { - id: Some(id), - .. - } if id.starts_with("review:rollout:") - ) -} - fn estimate_reasoning_length(encoded_len: usize) -> usize { encoded_len .saturating_mul(3) diff --git a/codex-rs/core/tests/suite/review.rs b/codex-rs/core/tests/suite/review.rs index 562cbf0ddf..43d3dfa1df 100644 --- a/codex-rs/core/tests/suite/review.rs +++ b/codex-rs/core/tests/suite/review.rs @@ -599,11 +599,10 @@ async fn review_input_isolated_from_parent_history() { server.verify().await; } -/// After a review thread finishes, its conversation should not leak into the -/// parent session. A subsequent parent turn must not include any review -/// messages in its request `input`. +/// After a review thread finishes, its conversation should be visible in the +/// parent session so later turns can reference the results. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn review_history_does_not_leak_into_parent_session() { +async fn review_history_surfaces_in_parent_session() { skip_if_no_network!(); // Respond to both the review request and the subsequent parent request. @@ -666,20 +665,26 @@ async fn review_history_does_not_leak_into_parent_session() { let last_text = last["content"][0]["text"].as_str().unwrap(); assert_eq!(last_text, followup); - // Ensure no review-thread content leaked into the parent request - let contains_review_prompt = input - .iter() - .any(|msg| msg["content"][0]["text"].as_str().unwrap_or_default() == "Start a review"); + // Ensure review-thread content is present for downstream turns. + let contains_review_rollout_user = input.iter().any(|msg| { + msg["content"][0]["text"] + .as_str() + .unwrap_or_default() + .contains("User initiated a review task.") + }); let contains_review_assistant = input.iter().any(|msg| { - msg["content"][0]["text"].as_str().unwrap_or_default() == "review assistant output" + msg["content"][0]["text"] + .as_str() + .unwrap_or_default() + .contains("review assistant output") }); assert!( - !contains_review_prompt, - "review prompt leaked into parent turn input" + contains_review_rollout_user, + "review rollout user message missing from parent turn input" ); assert!( - !contains_review_assistant, - "review assistant output leaked into parent turn input" + contains_review_assistant, + "review assistant output missing from parent turn input" ); server.verify().await; From 44d92675ebde7a2b4600ac86441dc120fb22acf9 Mon Sep 17 00:00:00 2001 From: Owen Lin Date: Mon, 1 Dec 2025 08:50:47 -0800 Subject: [PATCH 55/61] [app-server] fix: ensure thread_id and turn_id are on all events (#7408) This is an improvement for client-side developer ergonomics by simplifying the state the client needs to keep track of. --- .../app-server-protocol/src/protocol/v2.rs | 13 +++++++++++++ .../app-server/src/bespoke_event_handling.rs | 18 ++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index 71d38a3dd4..fd148b256f 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -1232,6 +1232,7 @@ pub struct TurnCompletedNotification { /// Notification that the turn-level unified diff has changed. /// Contains the latest aggregated diff across all file changes in the turn. pub struct TurnDiffUpdatedNotification { + pub thread_id: String, pub turn_id: String, pub diff: String, } @@ -1304,6 +1305,8 @@ pub struct ItemCompletedNotification { #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct AgentMessageDeltaNotification { + pub thread_id: String, + pub turn_id: String, pub item_id: String, pub delta: String, } @@ -1312,6 +1315,8 @@ pub struct AgentMessageDeltaNotification { #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct ReasoningSummaryTextDeltaNotification { + pub thread_id: String, + pub turn_id: String, pub item_id: String, pub delta: String, pub summary_index: i64, @@ -1321,6 +1326,8 @@ pub struct ReasoningSummaryTextDeltaNotification { #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct ReasoningSummaryPartAddedNotification { + pub thread_id: String, + pub turn_id: String, pub item_id: String, pub summary_index: i64, } @@ -1329,6 +1336,8 @@ pub struct ReasoningSummaryPartAddedNotification { #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct ReasoningTextDeltaNotification { + pub thread_id: String, + pub turn_id: String, pub item_id: String, pub delta: String, pub content_index: i64, @@ -1338,6 +1347,8 @@ pub struct ReasoningTextDeltaNotification { #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct CommandExecutionOutputDeltaNotification { + pub thread_id: String, + pub turn_id: String, pub item_id: String, pub delta: String, } @@ -1346,6 +1357,8 @@ pub struct CommandExecutionOutputDeltaNotification { #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub struct McpToolCallProgressNotification { + pub thread_id: String, + pub turn_id: String, pub item_id: String, pub message: String, } diff --git a/codex-rs/app-server/src/bespoke_event_handling.rs b/codex-rs/app-server/src/bespoke_event_handling.rs index a32a855b92..c34c7b281e 100644 --- a/codex-rs/app-server/src/bespoke_event_handling.rs +++ b/codex-rs/app-server/src/bespoke_event_handling.rs @@ -260,6 +260,8 @@ pub(crate) async fn apply_bespoke_event_handling( } EventMsg::AgentMessageContentDelta(event) => { let notification = AgentMessageDeltaNotification { + thread_id: conversation_id.to_string(), + turn_id: event_turn_id.clone(), item_id: event.item_id, delta: event.delta, }; @@ -278,6 +280,8 @@ pub(crate) async fn apply_bespoke_event_handling( } EventMsg::ReasoningContentDelta(event) => { let notification = ReasoningSummaryTextDeltaNotification { + thread_id: conversation_id.to_string(), + turn_id: event_turn_id.clone(), item_id: event.item_id, delta: event.delta, summary_index: event.summary_index, @@ -290,6 +294,8 @@ pub(crate) async fn apply_bespoke_event_handling( } EventMsg::ReasoningRawContentDelta(event) => { let notification = ReasoningTextDeltaNotification { + thread_id: conversation_id.to_string(), + turn_id: event_turn_id.clone(), item_id: event.item_id, delta: event.delta, content_index: event.content_index, @@ -300,6 +306,8 @@ pub(crate) async fn apply_bespoke_event_handling( } EventMsg::AgentReasoningSectionBreak(event) => { let notification = ReasoningSummaryPartAddedNotification { + thread_id: conversation_id.to_string(), + turn_id: event_turn_id.clone(), item_id: event.item_id, summary_index: event.summary_index, }; @@ -494,6 +502,8 @@ pub(crate) async fn apply_bespoke_event_handling( } EventMsg::ExecCommandOutputDelta(exec_command_output_delta_event) => { let notification = CommandExecutionOutputDeltaNotification { + thread_id: conversation_id.to_string(), + turn_id: event_turn_id.clone(), item_id: exec_command_output_delta_event.call_id.clone(), delta: String::from_utf8_lossy(&exec_command_output_delta_event.chunk).to_string(), }; @@ -588,6 +598,7 @@ pub(crate) async fn apply_bespoke_event_handling( } EventMsg::TurnDiff(turn_diff_event) => { handle_turn_diff( + conversation_id, &event_turn_id, turn_diff_event, api_version, @@ -610,6 +621,7 @@ pub(crate) async fn apply_bespoke_event_handling( } async fn handle_turn_diff( + conversation_id: ConversationId, event_turn_id: &str, turn_diff_event: TurnDiffEvent, api_version: ApiVersion, @@ -617,6 +629,7 @@ async fn handle_turn_diff( ) { if let ApiVersion::V2 = api_version { let notification = TurnDiffUpdatedNotification { + thread_id: conversation_id.to_string(), turn_id: event_turn_id.to_string(), diff: turn_diff_event.unified_diff, }; @@ -1774,8 +1787,10 @@ mod tests { let (tx, mut rx) = mpsc::channel(CHANNEL_CAPACITY); let outgoing = OutgoingMessageSender::new(tx); let unified_diff = "--- a\n+++ b\n".to_string(); + let conversation_id = ConversationId::new(); handle_turn_diff( + conversation_id, "turn-1", TurnDiffEvent { unified_diff: unified_diff.clone(), @@ -1793,6 +1808,7 @@ mod tests { OutgoingMessage::AppServerNotification(ServerNotification::TurnDiffUpdated( notification, )) => { + assert_eq!(notification.thread_id, conversation_id.to_string()); assert_eq!(notification.turn_id, "turn-1"); assert_eq!(notification.diff, unified_diff); } @@ -1806,8 +1822,10 @@ mod tests { async fn test_handle_turn_diff_is_noop_for_v1() -> Result<()> { let (tx, mut rx) = mpsc::channel(CHANNEL_CAPACITY); let outgoing = OutgoingMessageSender::new(tx); + let conversation_id = ConversationId::new(); handle_turn_diff( + conversation_id, "turn-1", TurnDiffEvent { unified_diff: "diff".to_string(), From 8532876ad80c3ba3067dcfb970de1106897aad06 Mon Sep 17 00:00:00 2001 From: Owen Lin Date: Mon, 1 Dec 2025 09:52:34 -0800 Subject: [PATCH 56/61] [app-server] fix: emit item/fileChange/outputDelta for file change items (#7399) --- .../src/protocol/common.rs | 1 + .../app-server-protocol/src/protocol/v2.rs | 10 ++++ .../app-server/src/bespoke_event_handling.rs | 48 +++++++++++++++---- .../app-server/tests/suite/v2/turn_start.rs | 21 ++++++++ 4 files changed, 70 insertions(+), 10 deletions(-) diff --git a/codex-rs/app-server-protocol/src/protocol/common.rs b/codex-rs/app-server-protocol/src/protocol/common.rs index f8483e9edb..5eba9b5130 100644 --- a/codex-rs/app-server-protocol/src/protocol/common.rs +++ b/codex-rs/app-server-protocol/src/protocol/common.rs @@ -511,6 +511,7 @@ server_notification_definitions! { ItemCompleted => "item/completed" (v2::ItemCompletedNotification), AgentMessageDelta => "item/agentMessage/delta" (v2::AgentMessageDeltaNotification), CommandExecutionOutputDelta => "item/commandExecution/outputDelta" (v2::CommandExecutionOutputDeltaNotification), + FileChangeOutputDelta => "item/fileChange/outputDelta" (v2::FileChangeOutputDeltaNotification), McpToolCallProgress => "item/mcpToolCall/progress" (v2::McpToolCallProgressNotification), AccountUpdated => "account/updated" (v2::AccountUpdatedNotification), AccountRateLimitsUpdated => "account/rateLimits/updated" (v2::AccountRateLimitsUpdatedNotification), diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index fd148b256f..bd38539bb2 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -1353,6 +1353,16 @@ pub struct CommandExecutionOutputDeltaNotification { pub delta: String, } +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct FileChangeOutputDeltaNotification { + pub thread_id: String, + pub turn_id: String, + pub item_id: String, + pub delta: String, +} + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] diff --git a/codex-rs/app-server/src/bespoke_event_handling.rs b/codex-rs/app-server/src/bespoke_event_handling.rs index c34c7b281e..efdce3162d 100644 --- a/codex-rs/app-server/src/bespoke_event_handling.rs +++ b/codex-rs/app-server/src/bespoke_event_handling.rs @@ -18,6 +18,7 @@ use codex_app_server_protocol::ContextCompactedNotification; use codex_app_server_protocol::ErrorNotification; use codex_app_server_protocol::ExecCommandApprovalParams; use codex_app_server_protocol::ExecCommandApprovalResponse; +use codex_app_server_protocol::FileChangeOutputDeltaNotification; use codex_app_server_protocol::FileChangeRequestApprovalParams; use codex_app_server_protocol::FileChangeRequestApprovalResponse; use codex_app_server_protocol::FileUpdateChange; @@ -501,17 +502,44 @@ pub(crate) async fn apply_bespoke_event_handling( .await; } EventMsg::ExecCommandOutputDelta(exec_command_output_delta_event) => { - let notification = CommandExecutionOutputDeltaNotification { - thread_id: conversation_id.to_string(), - turn_id: event_turn_id.clone(), - item_id: exec_command_output_delta_event.call_id.clone(), - delta: String::from_utf8_lossy(&exec_command_output_delta_event.chunk).to_string(), + let item_id = exec_command_output_delta_event.call_id.clone(); + let delta = String::from_utf8_lossy(&exec_command_output_delta_event.chunk).to_string(); + // The underlying EventMsg::ExecCommandOutputDelta is used for shell, unified_exec, + // and apply_patch tool calls. We represent apply_patch with the FileChange item, and + // everything else with the CommandExecution item. + // + // We need to detect which item type it is so we can emit the right notification. + // We already have state tracking FileChange items on item/started, so let's use that. + let is_file_change = { + let map = turn_summary_store.lock().await; + map.get(&conversation_id) + .is_some_and(|summary| summary.file_change_started.contains(&item_id)) }; - outgoing - .send_server_notification(ServerNotification::CommandExecutionOutputDelta( - notification, - )) - .await; + if is_file_change { + let notification = FileChangeOutputDeltaNotification { + thread_id: conversation_id.to_string(), + turn_id: event_turn_id.clone(), + item_id, + delta, + }; + outgoing + .send_server_notification(ServerNotification::FileChangeOutputDelta( + notification, + )) + .await; + } else { + let notification = CommandExecutionOutputDeltaNotification { + thread_id: conversation_id.to_string(), + turn_id: event_turn_id.clone(), + item_id, + delta, + }; + outgoing + .send_server_notification(ServerNotification::CommandExecutionOutputDelta( + notification, + )) + .await; + } } EventMsg::ExecCommandEnd(exec_command_end_event) => { let ExecCommandEndEvent { diff --git a/codex-rs/app-server/tests/suite/v2/turn_start.rs b/codex-rs/app-server/tests/suite/v2/turn_start.rs index cd35e31adc..03ee279e51 100644 --- a/codex-rs/app-server/tests/suite/v2/turn_start.rs +++ b/codex-rs/app-server/tests/suite/v2/turn_start.rs @@ -11,6 +11,7 @@ use app_test_support::to_response; use codex_app_server_protocol::ApprovalDecision; use codex_app_server_protocol::CommandExecutionRequestApprovalResponse; use codex_app_server_protocol::CommandExecutionStatus; +use codex_app_server_protocol::FileChangeOutputDeltaNotification; use codex_app_server_protocol::FileChangeRequestApprovalResponse; use codex_app_server_protocol::ItemCompletedNotification; use codex_app_server_protocol::ItemStartedNotification; @@ -725,6 +726,26 @@ async fn turn_start_file_change_approval_v2() -> Result<()> { ) .await?; + let output_delta_notif = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("item/fileChange/outputDelta"), + ) + .await??; + let output_delta: FileChangeOutputDeltaNotification = serde_json::from_value( + output_delta_notif + .params + .clone() + .expect("item/fileChange/outputDelta params"), + )?; + assert_eq!(output_delta.thread_id, thread.id); + assert_eq!(output_delta.turn_id, turn.id); + assert_eq!(output_delta.item_id, "patch-call"); + assert!( + !output_delta.delta.is_empty(), + "expected delta to be non-empty, got: {}", + output_delta.delta + ); + let completed_file_change = timeout(DEFAULT_READ_TIMEOUT, async { loop { let completed_notif = mcp From 0cc3b502287dd79265920ff08b2e2648411ed161 Mon Sep 17 00:00:00 2001 From: Ali Towaiji <145403626+Towaiji@users.noreply.github.com> Date: Mon, 1 Dec 2025 13:14:36 -0500 Subject: [PATCH 57/61] Fix recent_commits(limit=0) returning 1 commit instead of 0 (#7334) Fixes #7333 This is a small bug fix. This PR fixes an inconsistency in `recent_commits` where `limit == 0` still returns 1 commit due to the use of `limit.max(1)` when constructing the `git log -n` argument. Expected behavior: requesting 0 commits should return an empty list. This PR: - returns an empty `Vec` when `limit == 0` - adds a test for `recent_commits(limit == 0)` that fails before the change and passes afterwards - maintains existing behavior for `limit > 0` This aligns behavior with API expectations and avoids downstream consumers misinterpreting the repository as having commit history when `limit == 0` is used to explicitly request none. Happy to adjust if the current behavior is intentional. --- codex-rs/core/src/git_info.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/codex-rs/core/src/git_info.rs b/codex-rs/core/src/git_info.rs index 34e0afc729..4284ff1263 100644 --- a/codex-rs/core/src/git_info.rs +++ b/codex-rs/core/src/git_info.rs @@ -131,11 +131,15 @@ pub async fn recent_commits(cwd: &Path, limit: usize) -> Vec { } let fmt = "%H%x1f%ct%x1f%s"; // - let n = limit.max(1).to_string(); - let Some(log_out) = - run_git_command_with_timeout(&["log", "-n", &n, &format!("--pretty=format:{fmt}")], cwd) - .await - else { + let limit_arg = (limit > 0).then(|| limit.to_string()); + let mut args: Vec = vec!["log".to_string()]; + if let Some(n) = &limit_arg { + args.push("-n".to_string()); + args.push(n.clone()); + } + args.push(format!("--pretty=format:{fmt}")); + let arg_refs: Vec<&str> = args.iter().map(String::as_str).collect(); + let Some(log_out) = run_git_command_with_timeout(&arg_refs, cwd).await else { return Vec::new(); }; if !log_out.status.success() { From 440c7acd8fd9f69ce98a453a3d1ec0420728f801 Mon Sep 17 00:00:00 2001 From: Albert O'Shea Date: Tue, 2 Dec 2025 05:45:31 +1100 Subject: [PATCH 58/61] fix: nix build missing rmcp output hash (#7436) Output hash for `rmcp-0.9.0` was missing from the nix package, (i.e. `error: No hash was found while vendoring the git dependency rmcp-0.9.0.`) blocking the build. --- codex-rs/default.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/codex-rs/default.nix b/codex-rs/default.nix index 26971f1846..867e57ee2d 100644 --- a/codex-rs/default.nix +++ b/codex-rs/default.nix @@ -22,6 +22,7 @@ rustPlatform.buildRustPackage (_: { cargoLock.outputHashes = { "ratatui-0.29.0" = "sha256-HBvT5c8GsiCxMffNjJGLmHnvG77A6cqEL+1ARurBXho="; "crossterm-0.28.1" = "sha256-6qCtfSMuXACKFb9ATID39XyFDIEMFDmbx6SSmNe+728="; + "rmcp-0.9.0" = "sha256-0iPrpf0Ha/facO3p5e0hUKHBqGp/iS+C+OdS+pRKMOU="; }; meta = with lib; { From 41760f8a093d576453873adb91e81f727f98b91f Mon Sep 17 00:00:00 2001 From: Kaden Gruizenga Date: Mon, 1 Dec 2025 13:46:53 -0500 Subject: [PATCH 59/61] docs: clarify codex max defaults and xhigh availability (#7449) ## Summary Adds the missing `xhigh` reasoning level everywhere it should have been documented, and makes clear it only works with `gpt-5.1-codex-max`. ## Changes * `docs/config.md` * Add `xhigh` to the official list of reasoning levels with a note that `xhigh` is exclusive to Codex Max. * `docs/example-config.md` * Update the example comment adding `xhigh` as a valid option but only for Codex Max. * `docs/faq.md` * Update the model recommendation to `GPT-5.1 Codex Max`. * Mention that users can choose `high` or the newly documented `xhigh` level when using Codex Max. --- docs/config.md | 1 + docs/example-config.md | 2 +- docs/faq.md | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/config.md b/docs/config.md index df4da3b005..a3e6ed266b 100644 --- a/docs/config.md +++ b/docs/config.md @@ -195,6 +195,7 @@ If the selected model is known to support reasoning (for example: `o3`, `o4-mini - `"low"` - `"medium"` (default) - `"high"` +- `"xhigh"` (available only on `gpt-5.1-codex-max`) Note: to minimize reasoning, choose `"minimal"`. diff --git a/docs/example-config.md b/docs/example-config.md index bd220a41e6..cc53efa612 100644 --- a/docs/example-config.md +++ b/docs/example-config.md @@ -37,7 +37,7 @@ model_provider = "openai" # Reasoning & Verbosity (Responses API capable models) ################################################################################ -# Reasoning effort: minimal | low | medium | high (default: medium) +# Reasoning effort: minimal | low | medium | high | xhigh (default: medium; xhigh only on gpt-5.1-codex-max) model_reasoning_effort = "medium" # Reasoning summary: auto | concise | detailed | none (default: auto) diff --git a/docs/faq.md b/docs/faq.md index 72cbdff792..37b9ef39d8 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -8,7 +8,7 @@ In 2021, OpenAI released Codex, an AI system designed to generate code from natu ### Which models are supported? -We recommend using Codex with GPT-5.1 Codex, our best coding model. The default reasoning level is medium, and you can upgrade to high for complex tasks with the `/model` command. +We recommend using Codex with GPT-5.1 Codex Max, our best coding model. The default reasoning level is medium, and you can upgrade to high or xhigh (Codex Max only) for complex tasks with the `/model` command. You can also use older models by using API-based auth and launching codex with the `--model` flag. From c936c68c84f0e389cdaf2e8536c507600ee5771a Mon Sep 17 00:00:00 2001 From: linuxmetel <61399424+linuxmetel@users.noreply.github.com> Date: Tue, 2 Dec 2025 03:58:20 +0900 Subject: [PATCH 60/61] fix: prevent MCP startup failure on missing 'type' field (#7417) Fix the issue #7416 that the codex-cli produce an error "MCP startup failure on missing 'type' field" in the startup. - Cause: serde in `convert_to_rmcp` (`codex-rs/rmcp-client/src/utils.rs`) failed because no `r#type` value was provided - Fix: set a default `r#type` value in the corresponding structs --- codex-rs/mcp-types/generate_mcp_types.py | 27 ++++++++++++++++++++++++ codex-rs/mcp-types/src/lib.rs | 10 +++++++++ 2 files changed, 37 insertions(+) diff --git a/codex-rs/mcp-types/generate_mcp_types.py b/codex-rs/mcp-types/generate_mcp_types.py index 7535b4c760..5aaac81cd7 100755 --- a/codex-rs/mcp-types/generate_mcp_types.py +++ b/codex-rs/mcp-types/generate_mcp_types.py @@ -38,6 +38,13 @@ SERVER_NOTIFICATION_TYPE_NAMES: list[str] = [] # order to compile without warnings. LARGE_ENUMS = {"ServerResult"} +# some types need setting a default value for `r#type` +# ref: [#7417](https://github.com/openai/codex/pull/7417) +default_type_values: dict[str, str] = { + "ToolInputSchema": "object", + "ToolOutputSchema": "object", +} + def main() -> int: parser = argparse.ArgumentParser( @@ -351,6 +358,14 @@ class StructField: out.append(f" pub {self.name}: {self.type_name},\n") +def append_serde_attr(existing: str | None, fragment: str) -> str: + if existing is None: + return f"#[serde({fragment})]" + assert existing.startswith("#[serde(") and existing.endswith(")]"), existing + body = existing[len("#[serde(") : -2] + return f"#[serde({body}, {fragment})]" + + def define_struct( name: str, properties: dict[str, Any], @@ -359,6 +374,14 @@ def define_struct( ) -> list[str]: out: list[str] = [] + type_default_fn: str | None = None + if name in default_type_values: + snake_name = to_snake_case(name) or name + type_default_fn = f"{snake_name}_type_default_str" + out.append(f"fn {type_default_fn}() -> String {{\n") + out.append(f' "{default_type_values[name]}".to_string()\n') + out.append("}\n\n") + fields: list[StructField] = [] for prop_name, prop in properties.items(): if prop_name == "_meta": @@ -380,6 +403,10 @@ def define_struct( if is_optional: prop_type = f"Option<{prop_type}>" rs_prop = rust_prop_name(prop_name, is_optional) + + if prop_name == "type" and type_default_fn: + rs_prop.serde = append_serde_attr(rs_prop.serde, f'default = "{type_default_fn}"') + if prop_type.startswith("&'static str"): fields.append(StructField("const", rs_prop.name, prop_type, rs_prop.serde, rs_prop.ts)) else: diff --git a/codex-rs/mcp-types/src/lib.rs b/codex-rs/mcp-types/src/lib.rs index e390322fc8..7418bea854 100644 --- a/codex-rs/mcp-types/src/lib.rs +++ b/codex-rs/mcp-types/src/lib.rs @@ -1474,6 +1474,10 @@ pub struct Tool { pub title: Option, } +fn tool_output_schema_type_default_str() -> String { + "object".to_string() +} + /// An optional JSON Schema object defining the structure of the tool's output returned in /// the structuredContent field of a CallToolResult. #[derive(Debug, Clone, PartialEq, Deserialize, Serialize, JsonSchema, TS)] @@ -1484,9 +1488,14 @@ pub struct ToolOutputSchema { #[serde(default, skip_serializing_if = "Option::is_none")] #[ts(optional)] pub required: Option>, + #[serde(default = "tool_output_schema_type_default_str")] pub r#type: String, // &'static str = "object" } +fn tool_input_schema_type_default_str() -> String { + "object".to_string() +} + /// A JSON Schema object defining the expected parameters for the tool. #[derive(Debug, Clone, PartialEq, Deserialize, Serialize, JsonSchema, TS)] pub struct ToolInputSchema { @@ -1496,6 +1505,7 @@ pub struct ToolInputSchema { #[serde(default, skip_serializing_if = "Option::is_none")] #[ts(optional)] pub required: Option>, + #[serde(default = "tool_input_schema_type_default_str")] pub r#type: String, // &'static str = "object" } From a3ac5fc0f50c51a701c41a413c04745b095f2d38 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Mon, 1 Dec 2025 13:33:30 -0800 Subject: [PATCH 61/61] chore: update to rmcp@0.10.0 to pick up support for custom client notifications --- codex-rs/Cargo.lock | 18 +++++++++++++----- codex-rs/Cargo.toml | 9 ++++----- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 2320afeecb..ca7e29dda2 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -4464,6 +4464,12 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pastey" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57d6c094ee800037dff99e02cab0eaf3142826586742a270ab3d7a62656bd27a" + [[package]] name = "path-absolutize" version = "3.1.1" @@ -5142,8 +5148,9 @@ dependencies = [ [[package]] name = "rmcp" -version = "0.9.0" -source = "git+https://github.com/bolinfest/rust-sdk?branch=pr556#4d9cc16f4c76c84486344f542ed9a3e9364019ba" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b18323edc657390a6ed4d7a9110b0dec2dc3ed128eb2a123edfbafabdbddc5" dependencies = [ "async-trait", "base64", @@ -5154,7 +5161,7 @@ dependencies = [ "http-body", "http-body-util", "oauth2", - "paste", + "pastey", "pin-project-lite", "process-wrap", "rand 0.9.2", @@ -5176,8 +5183,9 @@ dependencies = [ [[package]] name = "rmcp-macros" -version = "0.9.0" -source = "git+https://github.com/bolinfest/rust-sdk?branch=pr556#4d9cc16f4c76c84486344f542ed9a3e9364019ba" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c75d0a62676bf8c8003c4e3c348e2ceb6a7b3e48323681aaf177fdccdac2ce50" dependencies = [ "darling 0.21.3", "proc-macro2", diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index 44270c4eb4..caaa13068a 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -59,15 +59,15 @@ license = "Apache-2.0" # Internal app_test_support = { path = "app-server/tests/common" } codex-ansi-escape = { path = "ansi-escape" } +codex-api = { path = "codex-api" } codex-app-server = { path = "app-server" } codex-app-server-protocol = { path = "app-server-protocol" } codex-apply-patch = { path = "apply-patch" } codex-arg0 = { path = "arg0" } codex-async-utils = { path = "async-utils" } codex-backend-client = { path = "backend-client" } -codex-api = { path = "codex-api" } -codex-client = { path = "codex-client" } codex-chatgpt = { path = "chatgpt" } +codex-client = { path = "codex-client" } codex-common = { path = "common" } codex-core = { path = "core" } codex-exec = { path = "exec" } @@ -169,10 +169,10 @@ pulldown-cmark = "0.10" rand = "0.9" ratatui = "0.29.0" ratatui-macros = "0.6.0" -regex-lite = "0.1.7" regex = "1.12.2" +regex-lite = "0.1.7" reqwest = "0.12" -rmcp = { version = "0.9.0", default-features = false } +rmcp = { version = "0.10.0", default-features = false } schemars = "0.8.22" seccompiler = "0.5.0" sentry = "0.34.0" @@ -288,7 +288,6 @@ opt-level = 0 # ratatui = { path = "../../ratatui" } crossterm = { git = "https://github.com/nornagon/crossterm", branch = "nornagon/color-query" } ratatui = { git = "https://github.com/nornagon/ratatui", branch = "nornagon-v0.29.0-patch" } -rmcp = { git = "https://github.com/bolinfest/rust-sdk", branch = "pr556" } # Uncomment to debug local changes. # rmcp = { path = "../../rust-sdk/crates/rmcp" }