From 3de879071446a45cc24e09f2343b6421586c4add Mon Sep 17 00:00:00 2001 From: Ahmed Ibrahim Date: Tue, 18 Nov 2025 11:36:23 -0800 Subject: [PATCH 01/18] Add the utility to truncate by tokens (#6746) - This PR is to make it on path for truncating by tokens. This path will be initially used by unified exec and context manager (responsible for MCP calls mainly). - We are exposing new config `calls_output_max_tokens` - Use `tokens` as the main budget unit but truncate based on the model family by Introducing `TruncationPolicy`. - Introduce `truncate_text` as a router for truncation based on the mode. In next PRs: - remove truncate_with_line_bytes_budget - Add the ability to the model to override the token budget. --- codex-rs/core/src/codex.rs | 38 +- codex-rs/core/src/compact.rs | 38 +- codex-rs/core/src/compact_remote.rs | 5 +- codex-rs/core/src/config/mod.rs | 11 + codex-rs/core/src/context_manager/history.rs | 32 +- .../core/src/context_manager/history_tests.rs | 191 ++--- codex-rs/core/src/context_manager/mod.rs | 4 +- codex-rs/core/src/error.rs | 8 +- codex-rs/core/src/model_family.rs | 10 + codex-rs/core/src/state/session.rs | 8 +- codex-rs/core/src/tools/mod.rs | 9 +- codex-rs/core/src/truncate.rs | 705 ++++++++++-------- codex-rs/core/src/unified_exec/mod.rs | 1 + codex-rs/core/src/unified_exec/session.rs | 9 +- .../core/src/unified_exec/session_manager.rs | 15 +- codex-rs/core/tests/suite/truncation.rs | 189 ++++- codex-rs/core/tests/suite/unified_exec.rs | 37 +- codex-rs/core/tests/suite/user_shell_cmd.rs | 2 +- codex-rs/utils/tokenizer/src/lib.rs | 5 + docs/config.md | 1 + docs/example-config.md | 1 + 21 files changed, 770 insertions(+), 549 deletions(-) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 64d06d0571..e308601c55 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -13,6 +13,7 @@ use crate::parse_command::parse_command; use crate::parse_turn_item; use crate::response_processing::process_items; use crate::terminal; +use crate::truncate::TruncationPolicy; use crate::user_notification::UserNotifier; use crate::util::error_or_panic; use async_channel::Receiver; @@ -275,6 +276,7 @@ pub(crate) struct TurnContext { pub(crate) final_output_json_schema: Option, pub(crate) codex_linux_sandbox_exe: Option, pub(crate) tool_call_gate: Arc, + pub(crate) truncation_policy: TruncationPolicy, } impl TurnContext { @@ -401,7 +403,7 @@ impl Session { ); let client = ModelClient::new( - Arc::new(per_turn_config), + Arc::new(per_turn_config.clone()), auth_manager, otel_event_manager, provider, @@ -431,6 +433,7 @@ impl Session { final_output_json_schema: None, codex_linux_sandbox_exe: config.codex_linux_sandbox_exe.clone(), tool_call_gate: Arc::new(ReadinessFlag::new()), + truncation_policy: TruncationPolicy::new(&per_turn_config), } } @@ -678,7 +681,8 @@ impl Session { let reconstructed_history = self.reconstruct_history_from_rollout(&turn_context, &rollout_items); if !reconstructed_history.is_empty() { - self.record_into_history(&reconstructed_history).await; + self.record_into_history(&reconstructed_history, &turn_context) + .await; } // If persisting, persist all rollout items as-is (recorder filters) @@ -935,7 +939,7 @@ impl Session { turn_context: &TurnContext, items: &[ResponseItem], ) { - self.record_into_history(items).await; + self.record_into_history(items, turn_context).await; self.persist_rollout_response_items(items).await; self.send_raw_response_items(turn_context, items).await; } @@ -949,7 +953,10 @@ impl Session { for item in rollout_items { match item { RolloutItem::ResponseItem(response_item) => { - history.record_items(std::iter::once(response_item)); + history.record_items( + std::iter::once(response_item), + turn_context.truncation_policy, + ); } RolloutItem::Compacted(compacted) => { let snapshot = history.get_history(); @@ -973,9 +980,13 @@ impl Session { } /// Append ResponseItems to the in-memory conversation history only. - pub(crate) async fn record_into_history(&self, items: &[ResponseItem]) { + pub(crate) async fn record_into_history( + &self, + items: &[ResponseItem], + turn_context: &TurnContext, + ) { let mut state = self.state.lock().await; - state.record_items(items.iter()); + state.record_items(items.iter(), turn_context.truncation_policy); } pub(crate) async fn replace_history(&self, items: Vec) { @@ -1755,6 +1766,7 @@ async fn spawn_review_thread( final_output_json_schema: None, codex_linux_sandbox_exe: parent_turn_context.codex_linux_sandbox_exe.clone(), tool_call_gate: Arc::new(ReadinessFlag::new()), + truncation_policy: TruncationPolicy::new(&per_turn_config), }; // Seed the child task with the review prompt as the initial user message. @@ -2886,7 +2898,7 @@ mod tests { for item in &initial_context { rollout_items.push(RolloutItem::ResponseItem(item.clone())); } - live_history.record_items(initial_context.iter()); + live_history.record_items(initial_context.iter(), turn_context.truncation_policy); let user1 = ResponseItem::Message { id: None, @@ -2895,7 +2907,7 @@ mod tests { text: "first user".to_string(), }], }; - live_history.record_items(std::iter::once(&user1)); + live_history.record_items(std::iter::once(&user1), turn_context.truncation_policy); rollout_items.push(RolloutItem::ResponseItem(user1.clone())); let assistant1 = ResponseItem::Message { @@ -2905,7 +2917,7 @@ mod tests { text: "assistant reply one".to_string(), }], }; - live_history.record_items(std::iter::once(&assistant1)); + live_history.record_items(std::iter::once(&assistant1), turn_context.truncation_policy); rollout_items.push(RolloutItem::ResponseItem(assistant1.clone())); let summary1 = "summary one"; @@ -2929,7 +2941,7 @@ mod tests { text: "second user".to_string(), }], }; - live_history.record_items(std::iter::once(&user2)); + live_history.record_items(std::iter::once(&user2), turn_context.truncation_policy); rollout_items.push(RolloutItem::ResponseItem(user2.clone())); let assistant2 = ResponseItem::Message { @@ -2939,7 +2951,7 @@ mod tests { text: "assistant reply two".to_string(), }], }; - live_history.record_items(std::iter::once(&assistant2)); + live_history.record_items(std::iter::once(&assistant2), turn_context.truncation_policy); rollout_items.push(RolloutItem::ResponseItem(assistant2.clone())); let summary2 = "summary two"; @@ -2963,7 +2975,7 @@ mod tests { text: "third user".to_string(), }], }; - live_history.record_items(std::iter::once(&user3)); + live_history.record_items(std::iter::once(&user3), turn_context.truncation_policy); rollout_items.push(RolloutItem::ResponseItem(user3.clone())); let assistant3 = ResponseItem::Message { @@ -2973,7 +2985,7 @@ mod tests { text: "assistant reply three".to_string(), }], }; - live_history.record_items(std::iter::once(&assistant3)); + live_history.record_items(std::iter::once(&assistant3), turn_context.truncation_policy); rollout_items.push(RolloutItem::ResponseItem(assistant3.clone())); (rollout_items, live_history.get_history()) diff --git a/codex-rs/core/src/compact.rs b/codex-rs/core/src/compact.rs index 0495c161d5..33d38091f6 100644 --- a/codex-rs/core/src/compact.rs +++ b/codex-rs/core/src/compact.rs @@ -14,7 +14,9 @@ use crate::protocol::EventMsg; use crate::protocol::TaskStartedEvent; use crate::protocol::TurnContextItem; use crate::protocol::WarningEvent; -use crate::truncate::truncate_middle; +use crate::truncate::TruncationPolicy; +use crate::truncate::approx_token_count; +use crate::truncate::truncate_text; use crate::util::backoff; use codex_protocol::items::TurnItem; use codex_protocol::models::ContentItem; @@ -59,7 +61,10 @@ async fn run_compact_task_inner( let initial_input_for_turn: ResponseInputItem = ResponseInputItem::from(input); let mut history = sess.clone_history().await; - history.record_items(&[initial_input_for_turn.into()]); + history.record_items( + &[initial_input_for_turn.into()], + turn_context.truncation_policy, + ); let mut truncated_count = 0usize; @@ -230,7 +235,7 @@ pub(crate) fn build_compacted_history( initial_context, user_messages, summary_text, - COMPACT_USER_MESSAGE_MAX_TOKENS * 4, + COMPACT_USER_MESSAGE_MAX_TOKENS, ) } @@ -238,20 +243,21 @@ fn build_compacted_history_with_limit( mut history: Vec, user_messages: &[String], summary_text: &str, - max_bytes: usize, + max_tokens: usize, ) -> Vec { let mut selected_messages: Vec = Vec::new(); - if max_bytes > 0 { - let mut remaining = max_bytes; + if max_tokens > 0 { + let mut remaining = max_tokens; for message in user_messages.iter().rev() { if remaining == 0 { break; } - if message.len() <= remaining { + let tokens = approx_token_count(message); + if tokens <= remaining { selected_messages.push(message.clone()); - remaining = remaining.saturating_sub(message.len()); + remaining = remaining.saturating_sub(tokens); } else { - let (truncated, _) = truncate_middle(message, remaining); + let truncated = truncate_text(message, TruncationPolicy::Tokens(remaining)); selected_messages.push(truncated); break; } @@ -300,7 +306,8 @@ async fn drain_to_completed( }; match event { Ok(ResponseEvent::OutputItemDone(item)) => { - sess.record_into_history(std::slice::from_ref(&item)).await; + sess.record_into_history(std::slice::from_ref(&item), turn_context) + .await; } Ok(ResponseEvent::RateLimits(snapshot)) => { sess.update_rate_limits(turn_context, snapshot).await; @@ -318,6 +325,7 @@ async fn drain_to_completed( #[cfg(test)] mod tests { + use super::*; use pretty_assertions::assert_eq; @@ -409,16 +417,16 @@ mod tests { } #[test] - fn build_compacted_history_truncates_overlong_user_messages() { + fn build_token_limited_compacted_history_truncates_overlong_user_messages() { // Use a small truncation limit so the test remains fast while still validating // that oversized user content is truncated. - let max_bytes = 128; - let big = "X".repeat(max_bytes + 50); + let max_tokens = 16; + let big = "word ".repeat(200); let history = super::build_compacted_history_with_limit( Vec::new(), std::slice::from_ref(&big), "SUMMARY", - max_bytes, + max_tokens, ); assert_eq!(history.len(), 2); @@ -451,7 +459,7 @@ mod tests { } #[test] - fn build_compacted_history_appends_summary_message() { + fn build_token_limited_compacted_history_appends_summary_message() { let initial_context: Vec = Vec::new(); let user_messages = vec!["first user message".to_string()]; let summary_text = "summary text"; diff --git a/codex-rs/core/src/compact_remote.rs b/codex-rs/core/src/compact_remote.rs index 2c7d57eff2..1726aad6f9 100644 --- a/codex-rs/core/src/compact_remote.rs +++ b/codex-rs/core/src/compact_remote.rs @@ -50,7 +50,10 @@ async fn run_remote_compact_task_inner( let mut history = sess.clone_history().await; if !input.is_empty() { let initial_input_for_turn: ResponseInputItem = ResponseInputItem::from(input); - history.record_items(&[initial_input_for_turn.into()]); + history.record_items( + &[initial_input_for_turn.into()], + turn_context.truncation_policy, + ); } let prompt = Prompt { diff --git a/codex-rs/core/src/config/mod.rs b/codex-rs/core/src/config/mod.rs index 5b57d4dc01..0c00da9a48 100644 --- a/codex-rs/core/src/config/mod.rs +++ b/codex-rs/core/src/config/mod.rs @@ -195,6 +195,9 @@ pub struct Config { /// Additional filenames to try when looking for project-level docs. pub project_doc_fallback_filenames: Vec, + /// Token budget applied when storing tool/function outputs in the context manager. + pub tool_output_token_limit: Option, + /// Directory containing all Codex state (defaults to `~/.codex` but can be /// overridden by the `CODEX_HOME` environment variable). pub codex_home: PathBuf, @@ -636,6 +639,9 @@ pub struct ConfigToml { /// Ordered list of fallback filenames to look for when AGENTS.md is missing. pub project_doc_fallback_filenames: Option>, + /// Token budget applied when storing tool/function outputs in the context manager. + pub tool_output_token_limit: Option, + /// Profile to use from the `profiles` map. pub profile: Option, @@ -1209,6 +1215,7 @@ impl Config { } }) .collect(), + tool_output_token_limit: cfg.tool_output_token_limit, codex_home, history, file_opener: cfg.file_opener.unwrap_or(UriBasedFileOpener::VsCode), @@ -2961,6 +2968,7 @@ model_verbosity = "high" model_providers: fixture.model_provider_map.clone(), project_doc_max_bytes: PROJECT_DOC_MAX_BYTES, project_doc_fallback_filenames: Vec::new(), + tool_output_token_limit: None, codex_home: fixture.codex_home(), history: History::default(), file_opener: UriBasedFileOpener::VsCode, @@ -3032,6 +3040,7 @@ model_verbosity = "high" model_providers: fixture.model_provider_map.clone(), project_doc_max_bytes: PROJECT_DOC_MAX_BYTES, project_doc_fallback_filenames: Vec::new(), + tool_output_token_limit: None, codex_home: fixture.codex_home(), history: History::default(), file_opener: UriBasedFileOpener::VsCode, @@ -3118,6 +3127,7 @@ model_verbosity = "high" model_providers: fixture.model_provider_map.clone(), project_doc_max_bytes: PROJECT_DOC_MAX_BYTES, project_doc_fallback_filenames: Vec::new(), + tool_output_token_limit: None, codex_home: fixture.codex_home(), history: History::default(), file_opener: UriBasedFileOpener::VsCode, @@ -3190,6 +3200,7 @@ model_verbosity = "high" model_providers: fixture.model_provider_map.clone(), project_doc_max_bytes: PROJECT_DOC_MAX_BYTES, project_doc_fallback_filenames: Vec::new(), + tool_output_token_limit: None, codex_home: fixture.codex_home(), history: History::default(), file_opener: UriBasedFileOpener::VsCode, diff --git a/codex-rs/core/src/context_manager/history.rs b/codex-rs/core/src/context_manager/history.rs index 50e3a8bc94..af60ada9fe 100644 --- a/codex-rs/core/src/context_manager/history.rs +++ b/codex-rs/core/src/context_manager/history.rs @@ -1,8 +1,8 @@ use crate::codex::TurnContext; use crate::context_manager::normalize; -use crate::truncate; -use crate::truncate::format_output_for_model_body; -use crate::truncate::globally_truncate_function_output_items; +use crate::truncate::TruncationPolicy; +use crate::truncate::truncate_function_output_items_with_policy; +use crate::truncate::truncate_text; use codex_protocol::models::FunctionCallOutputPayload; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::TokenUsage; @@ -10,12 +10,6 @@ use codex_protocol::protocol::TokenUsageInfo; use codex_utils_tokenizer::Tokenizer; use std::ops::Deref; -const CONTEXT_WINDOW_HARD_LIMIT_FACTOR: f64 = 1.1; -const CONTEXT_WINDOW_HARD_LIMIT_BYTES: usize = - (truncate::MODEL_FORMAT_MAX_BYTES as f64 * CONTEXT_WINDOW_HARD_LIMIT_FACTOR) as usize; -const CONTEXT_WINDOW_HARD_LIMIT_LINES: usize = - (truncate::MODEL_FORMAT_MAX_LINES as f64 * CONTEXT_WINDOW_HARD_LIMIT_FACTOR) as usize; - /// Transcript of conversation history #[derive(Debug, Clone, Default)] pub(crate) struct ContextManager { @@ -50,7 +44,7 @@ impl ContextManager { } /// `items` is ordered from oldest to newest. - pub(crate) fn record_items(&mut self, items: I) + pub(crate) fn record_items(&mut self, items: I, policy: TruncationPolicy) where I: IntoIterator, I::Item: std::ops::Deref, @@ -62,7 +56,7 @@ impl ContextManager { continue; } - let processed = Self::process_item(&item); + let processed = self.process_item(item_ref, policy); self.items.push(processed); } } @@ -150,18 +144,14 @@ impl ContextManager { items.retain(|item| !matches!(item, ResponseItem::GhostSnapshot { .. })); } - fn process_item(item: &ResponseItem) -> ResponseItem { + fn process_item(&self, item: &ResponseItem, policy: TruncationPolicy) -> ResponseItem { match item { ResponseItem::FunctionCallOutput { call_id, output } => { - let truncated = format_output_for_model_body( - output.content.as_str(), - CONTEXT_WINDOW_HARD_LIMIT_BYTES, - CONTEXT_WINDOW_HARD_LIMIT_LINES, - ); + let truncated = truncate_text(output.content.as_str(), policy); let truncated_items = output .content_items .as_ref() - .map(|items| globally_truncate_function_output_items(items)); + .map(|items| truncate_function_output_items_with_policy(items, policy)); ResponseItem::FunctionCallOutput { call_id: call_id.clone(), output: FunctionCallOutputPayload { @@ -172,11 +162,7 @@ impl ContextManager { } } ResponseItem::CustomToolCallOutput { call_id, output } => { - let truncated = format_output_for_model_body( - output, - CONTEXT_WINDOW_HARD_LIMIT_BYTES, - CONTEXT_WINDOW_HARD_LIMIT_LINES, - ); + let truncated = truncate_text(output, policy); ResponseItem::CustomToolCallOutput { call_id: call_id.clone(), output: truncated, diff --git a/codex-rs/core/src/context_manager/history_tests.rs b/codex-rs/core/src/context_manager/history_tests.rs index c81749c2c1..fecd0a7277 100644 --- a/codex-rs/core/src/context_manager/history_tests.rs +++ b/codex-rs/core/src/context_manager/history_tests.rs @@ -1,9 +1,8 @@ use super::*; -use crate::context_manager::MODEL_FORMAT_MAX_LINES; use crate::truncate; +use crate::truncate::TruncationPolicy; use codex_git::GhostCommit; use codex_protocol::models::ContentItem; -use codex_protocol::models::FunctionCallOutputContentItem; use codex_protocol::models::FunctionCallOutputPayload; use codex_protocol::models::LocalShellAction; use codex_protocol::models::LocalShellExecAction; @@ -13,6 +12,9 @@ use codex_protocol::models::ReasoningItemReasoningSummary; use pretty_assertions::assert_eq; use regex_lite::Regex; +const EXEC_FORMAT_MAX_LINES: usize = 256; +const EXEC_FORMAT_MAX_BYTES: usize = 10_000; + fn assistant_msg(text: &str) -> ResponseItem { ResponseItem::Message { id: None, @@ -25,7 +27,9 @@ fn assistant_msg(text: &str) -> ResponseItem { fn create_history_with_items(items: Vec) -> ContextManager { let mut h = ContextManager::new(); - h.record_items(items.iter()); + // Use a generous but fixed token budget; tests only rely on truncation + // behavior, not on a specific model's token limit. + h.record_items(items.iter(), TruncationPolicy::Tokens(10_000)); h } @@ -55,6 +59,7 @@ fn reasoning_msg(text: &str) -> ResponseItem { #[test] fn filters_non_api_messages() { let mut h = ContextManager::default(); + let policy = TruncationPolicy::Tokens(10_000); // System message is not API messages; Other is ignored. let system = ResponseItem::Message { id: None, @@ -64,12 +69,12 @@ fn filters_non_api_messages() { }], }; let reasoning = reasoning_msg("thinking..."); - h.record_items([&system, &reasoning, &ResponseItem::Other]); + h.record_items([&system, &reasoning, &ResponseItem::Other], policy); // User and assistant should be retained. let u = user_msg("hi"); let a = assistant_msg("hello"); - h.record_items([&u, &a]); + h.record_items([&u, &a], policy); let items = h.contents(); assert_eq!( @@ -237,6 +242,9 @@ fn normalization_retains_local_shell_outputs() { #[test] fn record_items_truncates_function_call_output_content() { let mut history = ContextManager::new(); + // Any reasonably small token budget works; the test only cares that + // truncation happens and the marker is present. + let policy = TruncationPolicy::Tokens(1_000); let long_line = "a very long line to trigger truncation\n"; let long_output = long_line.repeat(2_500); let item = ResponseItem::FunctionCallOutput { @@ -248,15 +256,20 @@ fn record_items_truncates_function_call_output_content() { }, }; - history.record_items([&item]); + history.record_items([&item], policy); assert_eq!(history.items.len(), 1); match &history.items[0] { ResponseItem::FunctionCallOutput { output, .. } => { assert_ne!(output.content, long_output); assert!( - output.content.starts_with("Total output lines:"), - "expected truncated summary, got {}", + output.content.contains("tokens truncated"), + "expected token-based truncation marker, got {}", + output.content + ); + assert!( + output.content.contains("tokens truncated"), + "expected truncation marker, got {}", output.content ); } @@ -267,6 +280,7 @@ fn record_items_truncates_function_call_output_content() { #[test] fn record_items_truncates_custom_tool_call_output_content() { let mut history = ContextManager::new(); + let policy = TruncationPolicy::Tokens(1_000); let line = "custom output that is very long\n"; let long_output = line.repeat(2_500); let item = ResponseItem::CustomToolCallOutput { @@ -274,21 +288,48 @@ fn record_items_truncates_custom_tool_call_output_content() { output: long_output.clone(), }; - history.record_items([&item]); + history.record_items([&item], policy); assert_eq!(history.items.len(), 1); match &history.items[0] { ResponseItem::CustomToolCallOutput { output, .. } => { assert_ne!(output, &long_output); assert!( - output.starts_with("Total output lines:"), - "expected truncated summary, got {output}" + output.contains("tokens truncated"), + "expected token-based truncation marker, got {output}" + ); + assert!( + output.contains("tokens truncated") || output.contains("bytes truncated"), + "expected truncation marker, got {output}" ); } other => panic!("unexpected history item: {other:?}"), } } +#[test] +fn record_items_respects_custom_token_limit() { + let mut history = ContextManager::new(); + let policy = TruncationPolicy::Tokens(10); + let long_output = "tokenized content repeated many times ".repeat(200); + let item = ResponseItem::FunctionCallOutput { + call_id: "call-custom-limit".to_string(), + output: FunctionCallOutputPayload { + content: long_output, + success: Some(true), + ..Default::default() + }, + }; + + history.record_items([&item], policy); + + let stored = match &history.items[0] { + ResponseItem::FunctionCallOutput { output, .. } => output, + other => panic!("unexpected history item: {other:?}"), + }; + assert!(stored.content.contains("tokens truncated")); +} + fn assert_truncated_message_matches(message: &str, line: &str, total_lines: usize) { let pattern = truncated_message_pattern(line, total_lines); let regex = Regex::new(&pattern).unwrap_or_else(|err| { @@ -302,23 +343,22 @@ fn assert_truncated_message_matches(message: &str, line: &str, total_lines: usiz .expect("missing body capture") .as_str(); assert!( - body.len() <= truncate::MODEL_FORMAT_MAX_BYTES, + body.len() <= EXEC_FORMAT_MAX_BYTES, "body exceeds byte limit: {} bytes", body.len() ); } fn truncated_message_pattern(line: &str, total_lines: usize) -> String { - let head_lines = MODEL_FORMAT_MAX_LINES / 2; - let tail_lines = MODEL_FORMAT_MAX_LINES - head_lines; + let head_lines = EXEC_FORMAT_MAX_LINES / 2; + let tail_lines = EXEC_FORMAT_MAX_LINES - head_lines; let head_take = head_lines.min(total_lines); let tail_take = tail_lines.min(total_lines.saturating_sub(head_take)); let omitted = total_lines.saturating_sub(head_take + tail_take); let escaped_line = regex_lite::escape(line); if omitted == 0 { return format!( - r"(?s)^Total output lines: {total_lines}\n\n(?P{escaped_line}.*\n\[\.{{3}} output truncated to fit {max_bytes} bytes \.{{3}}]\n\n.*)$", - max_bytes = truncate::MODEL_FORMAT_MAX_BYTES, + r"(?s)^Total output lines: {total_lines}\n\n(?P{escaped_line}.*\n\[\.{{3}} removed \d+ bytes to fit {EXEC_FORMAT_MAX_BYTES} byte limit \.{{3}}]\n\n.*)$", ); } format!( @@ -331,11 +371,7 @@ fn format_exec_output_truncates_large_error() { let line = "very long execution error line that should trigger truncation\n"; let large_error = line.repeat(2_500); // way beyond both byte and line limits - let truncated = truncate::format_output_for_model_body( - &large_error, - truncate::MODEL_FORMAT_MAX_BYTES, - truncate::MODEL_FORMAT_MAX_LINES, - ); + let truncated = truncate::truncate_with_line_bytes_budget(&large_error, EXEC_FORMAT_MAX_BYTES); let total_lines = large_error.lines().count(); assert_truncated_message_matches(&truncated, line, total_lines); @@ -344,17 +380,13 @@ fn format_exec_output_truncates_large_error() { #[test] fn format_exec_output_marks_byte_truncation_without_omitted_lines() { - let long_line = "a".repeat(truncate::MODEL_FORMAT_MAX_BYTES + 50); - let truncated = truncate::format_output_for_model_body( - &long_line, - truncate::MODEL_FORMAT_MAX_BYTES, - truncate::MODEL_FORMAT_MAX_LINES, - ); + let long_line = "a".repeat(EXEC_FORMAT_MAX_BYTES + 50); + let truncated = truncate::truncate_with_line_bytes_budget(&long_line, EXEC_FORMAT_MAX_BYTES); assert_ne!(truncated, long_line); + let removed_bytes = long_line.len().saturating_sub(EXEC_FORMAT_MAX_BYTES); let marker_line = format!( - "[... output truncated to fit {} bytes ...]", - truncate::MODEL_FORMAT_MAX_BYTES + "[... removed {removed_bytes} bytes to fit {EXEC_FORMAT_MAX_BYTES} byte limit ...]" ); assert!( truncated.contains(&marker_line), @@ -371,28 +403,20 @@ fn format_exec_output_returns_original_when_within_limits() { let content = "example output\n".repeat(10); assert_eq!( - truncate::format_output_for_model_body( - &content, - truncate::MODEL_FORMAT_MAX_BYTES, - truncate::MODEL_FORMAT_MAX_LINES - ), + truncate::truncate_with_line_bytes_budget(&content, EXEC_FORMAT_MAX_BYTES), content ); } #[test] fn format_exec_output_reports_omitted_lines_and_keeps_head_and_tail() { - let total_lines = truncate::MODEL_FORMAT_MAX_LINES + 100; + let total_lines = EXEC_FORMAT_MAX_LINES + 100; let content: String = (0..total_lines) .map(|idx| format!("line-{idx}\n")) .collect(); - let truncated = truncate::format_output_for_model_body( - &content, - truncate::MODEL_FORMAT_MAX_BYTES, - truncate::MODEL_FORMAT_MAX_LINES, - ); - let omitted = total_lines - truncate::MODEL_FORMAT_MAX_LINES; + let truncated = truncate::truncate_with_line_bytes_budget(&content, EXEC_FORMAT_MAX_BYTES); + let omitted = total_lines - EXEC_FORMAT_MAX_LINES; let expected_marker = format!("[... omitted {omitted} of {total_lines} lines ...]"); assert!( @@ -413,103 +437,24 @@ fn format_exec_output_reports_omitted_lines_and_keeps_head_and_tail() { #[test] fn format_exec_output_prefers_line_marker_when_both_limits_exceeded() { - let total_lines = truncate::MODEL_FORMAT_MAX_LINES + 42; + let total_lines = EXEC_FORMAT_MAX_LINES + 42; let long_line = "x".repeat(256); let content: String = (0..total_lines) .map(|idx| format!("line-{idx}-{long_line}\n")) .collect(); - let truncated = truncate::format_output_for_model_body( - &content, - truncate::MODEL_FORMAT_MAX_BYTES, - truncate::MODEL_FORMAT_MAX_LINES, - ); + let truncated = truncate::truncate_with_line_bytes_budget(&content, EXEC_FORMAT_MAX_BYTES); assert!( truncated.contains("[... omitted 42 of 298 lines ...]"), "expected omitted marker when line count exceeds limit: {truncated}" ); assert!( - !truncated.contains("output truncated to fit"), + !truncated.contains("byte limit"), "line omission marker should take precedence over byte marker: {truncated}" ); } -#[test] -fn truncates_across_multiple_under_limit_texts_and_reports_omitted() { - // Arrange: several text items, none exceeding per-item limit, but total exceeds budget. - let budget = truncate::MODEL_FORMAT_MAX_BYTES; - let t1_len = (budget / 2).saturating_sub(10); - let t2_len = (budget / 2).saturating_sub(10); - let remaining_after_t1_t2 = budget.saturating_sub(t1_len + t2_len); - let t3_len = 50; // gets truncated to remaining_after_t1_t2 - let t4_len = 5; // omitted - let t5_len = 7; // omitted - - let t1 = "a".repeat(t1_len); - let t2 = "b".repeat(t2_len); - let t3 = "c".repeat(t3_len); - let t4 = "d".repeat(t4_len); - let t5 = "e".repeat(t5_len); - - let item = ResponseItem::FunctionCallOutput { - call_id: "call-omit".to_string(), - output: FunctionCallOutputPayload { - content: "irrelevant".to_string(), - content_items: Some(vec![ - FunctionCallOutputContentItem::InputText { text: t1 }, - FunctionCallOutputContentItem::InputText { text: t2 }, - FunctionCallOutputContentItem::InputImage { - image_url: "img:mid".to_string(), - }, - FunctionCallOutputContentItem::InputText { text: t3 }, - FunctionCallOutputContentItem::InputText { text: t4 }, - FunctionCallOutputContentItem::InputText { text: t5 }, - ]), - success: Some(true), - }, - }; - - let mut history = ContextManager::new(); - history.record_items([&item]); - assert_eq!(history.items.len(), 1); - let json = serde_json::to_value(&history.items[0]).expect("serialize to json"); - - let output = json - .get("output") - .expect("output field") - .as_array() - .expect("array output"); - - // Expect: t1 (full), t2 (full), image, t3 (truncated), summary mentioning 2 omitted. - assert_eq!(output.len(), 5); - - let first = output[0].as_object().expect("first obj"); - assert_eq!(first.get("type").unwrap(), "input_text"); - let first_text = first.get("text").unwrap().as_str().unwrap(); - assert_eq!(first_text.len(), t1_len); - - let second = output[1].as_object().expect("second obj"); - assert_eq!(second.get("type").unwrap(), "input_text"); - let second_text = second.get("text").unwrap().as_str().unwrap(); - assert_eq!(second_text.len(), t2_len); - - assert_eq!( - output[2], - serde_json::json!({"type": "input_image", "image_url": "img:mid"}) - ); - - let fourth = output[3].as_object().expect("fourth obj"); - assert_eq!(fourth.get("type").unwrap(), "input_text"); - let fourth_text = fourth.get("text").unwrap().as_str().unwrap(); - assert_eq!(fourth_text.len(), remaining_after_t1_t2); - - let summary = output[4].as_object().expect("summary obj"); - assert_eq!(summary.get("type").unwrap(), "input_text"); - let summary_text = summary.get("text").unwrap().as_str().unwrap(); - assert!(summary_text.contains("omitted 2 text items")); -} - //TODO(aibrahim): run CI in release mode. #[cfg(not(debug_assertions))] #[test] diff --git a/codex-rs/core/src/context_manager/mod.rs b/codex-rs/core/src/context_manager/mod.rs index ab0d2e8168..d347a7714d 100644 --- a/codex-rs/core/src/context_manager/mod.rs +++ b/codex-rs/core/src/context_manager/mod.rs @@ -1,7 +1,5 @@ mod history; mod normalize; -pub(crate) use crate::truncate::MODEL_FORMAT_MAX_BYTES; -pub(crate) use crate::truncate::MODEL_FORMAT_MAX_LINES; -pub(crate) use crate::truncate::format_output_for_model_body; +pub(crate) use crate::truncate::truncate_with_line_bytes_budget; pub(crate) use history::ContextManager; diff --git a/codex-rs/core/src/error.rs b/codex-rs/core/src/error.rs index 64ba8df848..9a42ec3d1b 100644 --- a/codex-rs/core/src/error.rs +++ b/codex-rs/core/src/error.rs @@ -2,7 +2,8 @@ use crate::codex::ProcessedResponseItem; use crate::exec::ExecToolCallOutput; use crate::token_data::KnownPlan; use crate::token_data::PlanType; -use crate::truncate::truncate_middle; +use crate::truncate::TruncationPolicy; +use crate::truncate::truncate_text; use chrono::DateTime; use chrono::Datelike; use chrono::Local; @@ -461,7 +462,10 @@ pub fn get_error_message_ui(e: &CodexErr) -> String { _ => e.to_string(), }; - truncate_middle(&message, ERROR_MESSAGE_UI_MAX_BYTES).0 + truncate_text( + &message, + TruncationPolicy::Bytes(ERROR_MESSAGE_UI_MAX_BYTES), + ) } #[cfg(test)] diff --git a/codex-rs/core/src/model_family.rs b/codex-rs/core/src/model_family.rs index 150420fecf..0758d20310 100644 --- a/codex-rs/core/src/model_family.rs +++ b/codex-rs/core/src/model_family.rs @@ -4,6 +4,7 @@ use codex_protocol::config_types::Verbosity; use crate::config::types::ReasoningSummaryFormat; use crate::tools::handlers::apply_patch::ApplyPatchToolType; use crate::tools::spec::ConfigShellToolType; +use crate::truncate::TruncationPolicy; /// The `instructions` field in the payload sent to a model should always start /// with this content. @@ -66,6 +67,8 @@ pub struct ModelFamily { /// Preferred shell tool type for this model family when features do not override it. pub shell_type: ConfigShellToolType, + + pub truncation_policy: TruncationPolicy, } macro_rules! model_family { @@ -89,6 +92,7 @@ macro_rules! model_family { shell_type: ConfigShellToolType::Default, default_verbosity: None, default_reasoning_effort: None, + truncation_policy: TruncationPolicy::Bytes(10_000), }; // apply overrides @@ -146,6 +150,7 @@ pub fn find_family_for_model(slug: &str) -> Option { ], supports_parallel_tool_calls: true, support_verbosity: true, + truncation_policy: TruncationPolicy::Tokens(10_000), ) // Internal models. @@ -164,6 +169,7 @@ pub fn find_family_for_model(slug: &str) -> Option { shell_type: if cfg!(windows) { ConfigShellToolType::ShellCommand } else { ConfigShellToolType::Default }, supports_parallel_tool_calls: true, support_verbosity: true, + truncation_policy: TruncationPolicy::Tokens(10_000), ) // Production models. @@ -180,6 +186,7 @@ pub fn find_family_for_model(slug: &str) -> Option { shell_type: if cfg!(windows) { ConfigShellToolType::ShellCommand } else { ConfigShellToolType::Default }, supports_parallel_tool_calls: true, support_verbosity: false, + truncation_policy: TruncationPolicy::Tokens(10_000), ) } else if slug.starts_with("gpt-5.1") { model_family!( @@ -190,6 +197,7 @@ pub fn find_family_for_model(slug: &str) -> Option { default_verbosity: Some(Verbosity::Low), base_instructions: GPT_5_1_INSTRUCTIONS.to_string(), default_reasoning_effort: Some(ReasoningEffort::Medium), + truncation_policy: TruncationPolicy::Bytes(10_000), supports_parallel_tool_calls: true, ) } else if slug.starts_with("gpt-5") { @@ -198,6 +206,7 @@ pub fn find_family_for_model(slug: &str) -> Option { supports_reasoning_summaries: true, needs_special_apply_patch_instructions: true, support_verbosity: true, + truncation_policy: TruncationPolicy::Bytes(10_000), ) } else { None @@ -220,5 +229,6 @@ pub fn derive_default_model_family(model: &str) -> ModelFamily { shell_type: ConfigShellToolType::Default, default_verbosity: None, default_reasoning_effort: None, + truncation_policy: TruncationPolicy::Bytes(10_000), } } diff --git a/codex-rs/core/src/state/session.rs b/codex-rs/core/src/state/session.rs index 5b630d5ce9..2dfa5199f1 100644 --- a/codex-rs/core/src/state/session.rs +++ b/codex-rs/core/src/state/session.rs @@ -7,6 +7,7 @@ use crate::context_manager::ContextManager; use crate::protocol::RateLimitSnapshot; use crate::protocol::TokenUsage; use crate::protocol::TokenUsageInfo; +use crate::truncate::TruncationPolicy; /// Persistent, session-scoped state previously stored directly on `Session`. pub(crate) struct SessionState { @@ -18,20 +19,21 @@ pub(crate) struct SessionState { impl SessionState { /// Create a new session state mirroring previous `State::default()` semantics. pub(crate) fn new(session_configuration: SessionConfiguration) -> Self { + let history = ContextManager::new(); Self { session_configuration, - history: ContextManager::new(), + history, latest_rate_limits: None, } } // History helpers - pub(crate) fn record_items(&mut self, items: I) + pub(crate) fn record_items(&mut self, items: I, policy: TruncationPolicy) where I: IntoIterator, I::Item: std::ops::Deref, { - self.history.record_items(items) + self.history.record_items(items, policy); } pub(crate) fn clone_history(&self) -> ContextManager { diff --git a/codex-rs/core/src/tools/mod.rs b/codex-rs/core/src/tools/mod.rs index c94a7c28d9..99d5f16506 100644 --- a/codex-rs/core/src/tools/mod.rs +++ b/codex-rs/core/src/tools/mod.rs @@ -9,9 +9,7 @@ pub mod runtimes; pub mod sandboxing; pub mod spec; -use crate::context_manager::MODEL_FORMAT_MAX_BYTES; -use crate::context_manager::MODEL_FORMAT_MAX_LINES; -use crate::context_manager::format_output_for_model_body; +use crate::context_manager::truncate_with_line_bytes_budget; use crate::exec::ExecToolCallOutput; pub use router::ToolRouter; use serde::Serialize; @@ -22,6 +20,9 @@ pub(crate) const TELEMETRY_PREVIEW_MAX_LINES: usize = 64; // lines pub(crate) const TELEMETRY_PREVIEW_TRUNCATION_NOTICE: &str = "[... telemetry preview truncated ...]"; +// TODO(aibrahim): migrate shell tool to use truncate text and respect config value +const SHELL_OUTPUT_MAX_BYTES: usize = 10_000; + /// Format the combined exec output for sending back to the model. /// Includes exit code and duration metadata; truncates large bodies safely. pub fn format_exec_output_for_model(exec_output: &ExecToolCallOutput) -> String { @@ -77,5 +78,5 @@ pub fn format_exec_output_str(exec_output: &ExecToolCallOutput) -> String { }; // Truncate for model consumption before serialization. - format_output_for_model_body(&body, MODEL_FORMAT_MAX_BYTES, MODEL_FORMAT_MAX_LINES) + truncate_with_line_bytes_budget(&body, SHELL_OUTPUT_MAX_BYTES) } diff --git a/codex-rs/core/src/truncate.rs b/codex-rs/core/src/truncate.rs index 42d6a967de..bdec0b0d1e 100644 --- a/codex-rs/core/src/truncate.rs +++ b/codex-rs/core/src/truncate.rs @@ -5,45 +5,142 @@ use codex_protocol::models::FunctionCallOutputContentItem; use codex_utils_string::take_bytes_at_char_boundary; use codex_utils_string::take_last_bytes_at_char_boundary; -use codex_utils_tokenizer::Tokenizer; -/// Model-formatting limits: clients get full streams; only content sent to the model is truncated. -pub const MODEL_FORMAT_MAX_BYTES: usize = 10 * 1024; // 10 KiB -pub const MODEL_FORMAT_MAX_LINES: usize = 256; // lines +use crate::config::Config; -/// Globally truncate function output items to fit within `MODEL_FORMAT_MAX_BYTES` -/// by preserving as many text/image items as possible and appending a summary -/// for any omitted text items. -pub(crate) fn globally_truncate_function_output_items( +const APPROX_BYTES_PER_TOKEN: usize = 4; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum TruncationPolicy { + Bytes(usize), + Tokens(usize), +} + +impl TruncationPolicy { + pub fn new(config: &Config) -> Self { + let config_token_limit = config.tool_output_token_limit; + + match config.model_family.truncation_policy { + TruncationPolicy::Bytes(family_bytes) => { + if let Some(token_limit) = config_token_limit { + Self::Bytes(approx_bytes_for_tokens(token_limit)) + } else { + Self::Bytes(family_bytes) + } + } + TruncationPolicy::Tokens(family_tokens) => { + if let Some(token_limit) = config_token_limit { + Self::Tokens(token_limit) + } else { + Self::Tokens(family_tokens) + } + } + } + } + + /// Returns a token budget derived from this policy. + /// + /// - For `Tokens`, this is the explicit token limit. + /// - For `Bytes`, this is an approximate token budget using the global + /// bytes-per-token heuristic. + pub fn token_budget(&self) -> usize { + match self { + TruncationPolicy::Bytes(bytes) => { + usize::try_from(approx_tokens_from_byte_count(*bytes)).unwrap_or(usize::MAX) + } + TruncationPolicy::Tokens(tokens) => *tokens, + } + } + + /// Returns a byte budget derived from this policy. + /// + /// - For `Bytes`, this is the explicit byte limit. + /// - For `Tokens`, this is an approximate byte budget using the global + /// bytes-per-token heuristic. + pub fn byte_budget(&self) -> usize { + match self { + TruncationPolicy::Bytes(bytes) => *bytes, + TruncationPolicy::Tokens(tokens) => approx_bytes_for_tokens(*tokens), + } + } +} + +/// Format a block of exec/tool output for model consumption, truncating by +/// lines and bytes while preserving head and tail segments. +pub(crate) fn truncate_with_line_bytes_budget(content: &str, bytes_budget: usize) -> String { + // TODO(aibrahim): to be removed + let lines_budget = 256; + // Head+tail truncation for the model: show the beginning and end with an elision. + // Clients still receive full streams; only this formatted summary is capped. + let total_lines = content.lines().count(); + if content.len() <= bytes_budget && total_lines <= lines_budget { + return content.to_string(); + } + let output = truncate_formatted_exec_output(content, total_lines, bytes_budget, lines_budget); + format!("Total output lines: {total_lines}\n\n{output}") +} + +pub(crate) fn truncate_text(content: &str, policy: TruncationPolicy) -> String { + match policy { + TruncationPolicy::Bytes(bytes) => truncate_with_byte_estimate( + content, + bytes, + TruncationSource::Policy(TruncationPolicy::Bytes(bytes)), + ), + TruncationPolicy::Tokens(tokens) => { + let (truncated, _) = truncate_with_token_budget( + content, + tokens, + TruncationSource::Policy(TruncationPolicy::Tokens(tokens)), + ); + truncated + } + } +} +/// Globally truncate function output items to fit within the given +/// truncation policy's budget, preserving as many text/image items as +/// possible and appending a summary for any omitted text items. +pub(crate) fn truncate_function_output_items_with_policy( items: &[FunctionCallOutputContentItem], + policy: TruncationPolicy, ) -> Vec { let mut out: Vec = Vec::with_capacity(items.len()); - let mut remaining = MODEL_FORMAT_MAX_BYTES; + let mut remaining_budget = match policy { + TruncationPolicy::Bytes(_) => policy.byte_budget(), + TruncationPolicy::Tokens(_) => policy.token_budget(), + }; let mut omitted_text_items = 0usize; for it in items { match it { FunctionCallOutputContentItem::InputText { text } => { - if remaining == 0 { + if remaining_budget == 0 { omitted_text_items += 1; continue; } - let len = text.len(); - if len <= remaining { + let cost = match policy { + TruncationPolicy::Bytes(_) => text.len(), + TruncationPolicy::Tokens(_) => approx_token_count(text), + }; + + if cost <= remaining_budget { out.push(FunctionCallOutputContentItem::InputText { text: text.clone() }); - remaining -= len; + remaining_budget = remaining_budget.saturating_sub(cost); } else { - let slice = take_bytes_at_char_boundary(text, remaining); - if !slice.is_empty() { - out.push(FunctionCallOutputContentItem::InputText { - text: slice.to_string(), - }); + let snippet_policy = match policy { + TruncationPolicy::Bytes(_) => TruncationPolicy::Bytes(remaining_budget), + TruncationPolicy::Tokens(_) => TruncationPolicy::Tokens(remaining_budget), + }; + let snippet = truncate_text(text, snippet_policy); + if snippet.is_empty() { + omitted_text_items += 1; + } else { + out.push(FunctionCallOutputContentItem::InputText { text: snippet }); } - remaining = 0; + remaining_budget = 0; } } - // todo(aibrahim): handle input images; resize FunctionCallOutputContentItem::InputImage { image_url } => { out.push(FunctionCallOutputContentItem::InputImage { image_url: image_url.clone(), @@ -61,21 +158,81 @@ pub(crate) fn globally_truncate_function_output_items( out } -/// Format a block of exec/tool output for model consumption, truncating by -/// lines and bytes while preserving head and tail segments. -pub(crate) fn format_output_for_model_body( - content: &str, - limit_bytes: usize, - limit_lines: usize, -) -> String { - // Head+tail truncation for the model: show the beginning and end with an elision. - // Clients still receive full streams; only this formatted summary is capped. - let total_lines = content.lines().count(); - if content.len() <= limit_bytes && total_lines <= limit_lines { - return content.to_string(); +/// Truncate the middle of a UTF-8 string to at most `max_tokens` tokens, +/// preserving the beginning and the end. Returns the possibly truncated string +/// and `Some(original_token_count)` if truncation occurred; otherwise returns +/// the original string and `None`. +fn truncate_with_token_budget( + s: &str, + max_tokens: usize, + source: TruncationSource, +) -> (String, Option) { + if s.is_empty() { + return (String::new(), None); } - let output = truncate_formatted_exec_output(content, total_lines, limit_bytes, limit_lines); - format!("Total output lines: {total_lines}\n\n{output}") + + let byte_len = s.len(); + if max_tokens > 0 { + let small_threshold = approx_bytes_for_tokens(max_tokens / 4); + if small_threshold > 0 && byte_len <= small_threshold { + return (s.to_string(), None); + } + } + + let truncated = truncate_with_byte_estimate(s, approx_bytes_for_tokens(max_tokens), source); + let approx_total_usize = approx_token_count(s); + let approx_total = u64::try_from(approx_total_usize).unwrap_or(u64::MAX); + if truncated == s { + (truncated, None) + } else { + (truncated, Some(approx_total)) + } +} + +/// Truncate a string using a byte budget derived from the token budget, without +/// performing any real tokenization. This keeps the logic purely byte-based and +/// uses a bytes placeholder in the truncated output. +fn truncate_with_byte_estimate(s: &str, max_bytes: usize, source: TruncationSource) -> String { + if s.is_empty() { + return String::new(); + } + + if max_bytes == 0 { + // No budget to show content; just report that everything was truncated. + let marker = format_truncation_marker(source, removed_units_for_source(source, s.len())); + return marker; + } + + if s.len() <= max_bytes { + return s.to_string(); + } + + let total_bytes = s.len(); + let removed_bytes = total_bytes.saturating_sub(max_bytes); + let marker = format_truncation_marker(source, removed_units_for_source(source, removed_bytes)); + let marker_len = marker.len(); + + if marker_len >= max_bytes { + let truncated_marker = truncate_on_boundary(&marker, max_bytes); + return truncated_marker.to_string(); + } + + let keep_budget = max_bytes - marker_len; + let (left_budget, right_budget) = split_budget(keep_budget); + let prefix_end = pick_prefix_end(s, left_budget); + let mut suffix_start = pick_suffix_start(s, right_budget); + if suffix_start < prefix_end { + suffix_start = prefix_end; + } + + let mut out = assemble_truncated_output(&s[..prefix_end], &s[suffix_start..], &marker); + + if out.len() > max_bytes { + let boundary = truncate_on_boundary(&out, max_bytes); + out.truncate(boundary.len()); + } + + out } fn truncate_formatted_exec_output( @@ -114,13 +271,17 @@ fn truncate_formatted_exec_output( let truncated_by_bytes = content.len() > limit_bytes; // this is a bit wrong. We are counting metadata lines and not just shell output lines. let marker = if omitted > 0 { - Some(format!( - "\n[... omitted {omitted} of {total_lines} lines ...]\n\n" - )) + let marker_text = format_truncation_marker( + TruncationSource::LineOmission { total_lines }, + u64::try_from(omitted).unwrap_or(u64::MAX), + ); + Some(format!("\n{marker_text}\n\n")) } else if truncated_by_bytes { - Some(format!( - "\n[... output truncated to fit {limit_bytes} bytes ...]\n\n" - )) + let removed_bytes = + u64::try_from(content.len().saturating_sub(limit_bytes)).unwrap_or(u64::MAX); + let marker_text = + format_truncation_marker(TruncationSource::ByteLimit { limit_bytes }, removed_bytes); + Some(format!("\n{marker_text}\n\n")) } else { None }; @@ -147,6 +308,103 @@ fn truncate_formatted_exec_output( result } +#[derive(Clone, Copy)] +pub enum TruncationSource { + Policy(TruncationPolicy), + LineOmission { total_lines: usize }, + ByteLimit { limit_bytes: usize }, +} + +fn format_truncation_marker(source: TruncationSource, removed_count: u64) -> String { + match source { + TruncationSource::Policy(TruncationPolicy::Tokens(_)) => { + format!("[…{removed_count} tokens truncated…]") + } + TruncationSource::Policy(TruncationPolicy::Bytes(_)) => { + format!("[…{removed_count} bytes truncated…]") + } + TruncationSource::LineOmission { total_lines } => { + format!("[... omitted {removed_count} of {total_lines} lines ...]") + } + TruncationSource::ByteLimit { limit_bytes } => { + format!("[... removed {removed_count} bytes to fit {limit_bytes} byte limit ...]") + } + } +} + +fn split_budget(budget: usize) -> (usize, usize) { + let left = budget / 2; + (left, budget - left) +} + +fn removed_units_for_source(source: TruncationSource, removed_bytes: usize) -> u64 { + match source { + TruncationSource::Policy(TruncationPolicy::Tokens(_)) => { + approx_tokens_from_byte_count(removed_bytes) + } + _ => u64::try_from(removed_bytes).unwrap_or(u64::MAX), + } +} + +fn assemble_truncated_output(prefix: &str, suffix: &str, marker: &str) -> String { + let mut out = String::with_capacity(prefix.len() + marker.len() + suffix.len() + 1); + out.push_str(prefix); + out.push_str(marker); + out.push('\n'); + out.push_str(suffix); + out +} + +pub(crate) fn approx_token_count(text: &str) -> usize { + let len = text.len(); + len.saturating_add(APPROX_BYTES_PER_TOKEN.saturating_sub(1)) / APPROX_BYTES_PER_TOKEN +} + +fn approx_bytes_for_tokens(tokens: usize) -> usize { + tokens.saturating_mul(APPROX_BYTES_PER_TOKEN) +} + +fn approx_tokens_from_byte_count(bytes: usize) -> u64 { + let bytes_u64 = bytes as u64; + bytes_u64.saturating_add((APPROX_BYTES_PER_TOKEN as u64).saturating_sub(1)) + / (APPROX_BYTES_PER_TOKEN as u64) +} + +fn truncate_on_boundary(input: &str, max_len: usize) -> &str { + if input.len() <= max_len { + return input; + } + let mut end = max_len; + while end > 0 && !input.is_char_boundary(end) { + end -= 1; + } + &input[..end] +} + +fn pick_prefix_end(s: &str, left_budget: usize) -> usize { + if let Some(head) = s.get(..left_budget) + && let Some(i) = head.rfind('\n') + { + return i + 1; + } + truncate_on_boundary(s, left_budget).len() +} + +fn pick_suffix_start(s: &str, right_budget: usize) -> usize { + let start_tail = s.len().saturating_sub(right_budget); + if let Some(tail) = s.get(start_tail..) + && let Some(i) = tail.find('\n') + { + return start_tail + i + 1; + } + + let mut idx = start_tail.min(s.len()); + while idx < s.len() && !s.is_char_boundary(idx) { + idx += 1; + } + idx +} + fn error_on_double_truncation(content: &str) { if content.contains("Total output lines:") && content.contains("omitted") { tracing::error!( @@ -155,184 +413,31 @@ fn error_on_double_truncation(content: &str) { } } -/// Truncate an output string to a maximum number of “tokens”, where tokens are -/// approximated as individual `char`s. Preserves a prefix and suffix with an -/// elision marker describing how many tokens were omitted. -pub(crate) fn truncate_output_to_tokens( - output: &str, - max_tokens: usize, -) -> (String, Option) { - if max_tokens == 0 { - let total_tokens = output.chars().count(); - let message = format!("…{total_tokens} tokens truncated…"); - return (message, Some(total_tokens)); - } - - let tokens: Vec = output.chars().collect(); - let total_tokens = tokens.len(); - if total_tokens <= max_tokens { - return (output.to_string(), None); - } - - let half = max_tokens / 2; - if half == 0 { - let truncated = total_tokens.saturating_sub(max_tokens); - let message = format!("…{truncated} tokens truncated…"); - return (message, Some(total_tokens)); - } - - let truncated = total_tokens.saturating_sub(half * 2); - let mut truncated_output = String::new(); - truncated_output.extend(&tokens[..half]); - truncated_output.push_str(&format!("…{truncated} tokens truncated…")); - truncated_output.extend(&tokens[total_tokens - half..]); - (truncated_output, Some(total_tokens)) -} - -/// Truncate the middle of a UTF-8 string to at most `max_bytes` bytes, -/// preserving the beginning and the end. Returns the possibly truncated -/// string and `Some(original_token_count)` (counted with the local tokenizer; -/// falls back to a 4-bytes-per-token estimate if the tokenizer cannot load) -/// if truncation occurred; otherwise returns the original string and `None`. -pub(crate) fn truncate_middle(s: &str, max_bytes: usize) -> (String, Option) { - if s.len() <= max_bytes { - return (s.to_string(), None); - } - - // Build a tokenizer for counting (default to o200k_base; fall back to cl100k_base). - // If both fail, fall back to a 4-bytes-per-token estimate. - let tok = Tokenizer::try_default().ok(); - let token_count = |text: &str| -> u64 { - if let Some(ref t) = tok { - t.count(text) as u64 - } else { - (text.len() as u64).div_ceil(4) - } - }; - - let total_tokens = token_count(s); - if max_bytes == 0 { - return ( - format!("…{total_tokens} tokens truncated…"), - Some(total_tokens), - ); - } - - fn truncate_on_boundary(input: &str, max_len: usize) -> &str { - if input.len() <= max_len { - return input; - } - let mut end = max_len; - while end > 0 && !input.is_char_boundary(end) { - end -= 1; - } - &input[..end] - } - - fn pick_prefix_end(s: &str, left_budget: usize) -> usize { - if let Some(head) = s.get(..left_budget) - && let Some(i) = head.rfind('\n') - { - return i + 1; - } - truncate_on_boundary(s, left_budget).len() - } - - fn pick_suffix_start(s: &str, right_budget: usize) -> usize { - let start_tail = s.len().saturating_sub(right_budget); - if let Some(tail) = s.get(start_tail..) - && let Some(i) = tail.find('\n') - { - return start_tail + i + 1; - } - - let mut idx = start_tail.min(s.len()); - while idx < s.len() && !s.is_char_boundary(idx) { - idx += 1; - } - idx - } - - // Iterate to stabilize marker length → keep budget → boundaries. - let mut guess_tokens: u64 = 1; - for _ in 0..4 { - let marker = format!("…{guess_tokens} tokens truncated…"); - let marker_len = marker.len(); - let keep_budget = max_bytes.saturating_sub(marker_len); - if keep_budget == 0 { - return ( - format!("…{total_tokens} tokens truncated…"), - Some(total_tokens), - ); - } - - let left_budget = keep_budget / 2; - let right_budget = keep_budget - left_budget; - let prefix_end = pick_prefix_end(s, left_budget); - let mut suffix_start = pick_suffix_start(s, right_budget); - if suffix_start < prefix_end { - suffix_start = prefix_end; - } - - // Tokens actually removed (middle slice) using the real tokenizer. - let removed_tokens = token_count(&s[prefix_end..suffix_start]); - - // If the number of digits in the token count does not change the marker length, - // we can finalize output. - let final_marker = format!("…{removed_tokens} tokens truncated…"); - if final_marker.len() == marker_len { - let kept_content_bytes = prefix_end + (s.len() - suffix_start); - let mut out = String::with_capacity(final_marker.len() + kept_content_bytes + 1); - out.push_str(&s[..prefix_end]); - out.push_str(&final_marker); - out.push('\n'); - out.push_str(&s[suffix_start..]); - return (out, Some(total_tokens)); - } - - guess_tokens = removed_tokens; - } - - // Fallback build after iterations: compute with the last guess. - let marker = format!("…{guess_tokens} tokens truncated…"); - let marker_len = marker.len(); - let keep_budget = max_bytes.saturating_sub(marker_len); - if keep_budget == 0 { - return ( - format!("…{total_tokens} tokens truncated…"), - Some(total_tokens), - ); - } - - let left_budget = keep_budget / 2; - let right_budget = keep_budget - left_budget; - let prefix_end = pick_prefix_end(s, left_budget); - let mut suffix_start = pick_suffix_start(s, right_budget); - if suffix_start < prefix_end { - suffix_start = prefix_end; - } - - let mut out = String::with_capacity(marker_len + prefix_end + (s.len() - suffix_start) + 1); - out.push_str(&s[..prefix_end]); - out.push_str(&marker); - out.push('\n'); - out.push_str(&s[suffix_start..]); - (out, Some(total_tokens)) -} - #[cfg(test)] mod tests { - use super::MODEL_FORMAT_MAX_BYTES; - use super::MODEL_FORMAT_MAX_LINES; - use super::format_output_for_model_body; - use super::globally_truncate_function_output_items; - use super::truncate_middle; - use super::truncate_output_to_tokens; + use crate::config::OPENAI_DEFAULT_MODEL; + use crate::model_family::derive_default_model_family; + use crate::model_family::find_family_for_model; + + use super::TruncationPolicy; + use super::TruncationSource; + use super::approx_token_count; + use super::truncate_function_output_items_with_policy; + use super::truncate_with_line_bytes_budget; + use super::truncate_with_token_budget; use codex_protocol::models::FunctionCallOutputContentItem; - use codex_utils_tokenizer::Tokenizer; use pretty_assertions::assert_eq; use regex_lite::Regex; + const MODEL_FORMAT_MAX_LINES: usize = 256; + + fn model_format_max_bytes() -> usize { + find_family_for_model(OPENAI_DEFAULT_MODEL) + .unwrap_or_else(|| derive_default_model_family(OPENAI_DEFAULT_MODEL)) + .truncation_policy + .byte_budget() + } + fn truncated_message_pattern(line: &str, total_lines: usize) -> String { let head_lines = MODEL_FORMAT_MAX_LINES / 2; let tail_lines = MODEL_FORMAT_MAX_LINES - head_lines; @@ -342,7 +447,8 @@ mod tests { let escaped_line = regex_lite::escape(line); if omitted == 0 { return format!( - r"(?s)^Total output lines: {total_lines}\n\n(?P{escaped_line}.*\n\[\.{{3}} output truncated to fit {MODEL_FORMAT_MAX_BYTES} bytes \.{{3}}]\n\n.*)$", + r"(?s)^Total output lines: {total_lines}\n\n(?P{escaped_line}.*\n\[\.{{3}} removed \d+ bytes to fit {max_bytes} byte limit \.{{3}}]\n\n.*)$", + max_bytes = model_format_max_bytes(), ); } format!( @@ -351,88 +457,46 @@ mod tests { } #[test] - fn truncate_middle_no_newlines_fallback() { - let tok = Tokenizer::try_default().expect("load tokenizer"); - let s = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ*"; - let max_bytes = 32; - let (out, original) = truncate_middle(s, max_bytes); - assert!(out.starts_with("abc")); - assert!(out.contains("tokens truncated")); - assert!(out.ends_with("XYZ*")); - assert_eq!(original, Some(tok.count(s) as u64)); - } - - #[test] - fn truncate_middle_prefers_newline_boundaries() { - let tok = Tokenizer::try_default().expect("load tokenizer"); - let mut s = String::new(); - for i in 1..=20 { - s.push_str(&format!("{i:03}\n")); - } - assert_eq!(s.len(), 80); - - let max_bytes = 64; - let (out, tokens) = truncate_middle(&s, max_bytes); - assert!(out.starts_with("001\n002\n003\n004\n")); - assert!(out.contains("tokens truncated")); - assert!(out.ends_with("017\n018\n019\n020\n")); - assert_eq!(tokens, Some(tok.count(&s) as u64)); - } - - #[test] - fn truncate_middle_handles_utf8_content() { - let tok = Tokenizer::try_default().expect("load tokenizer"); - let s = "😀😀😀😀😀😀😀😀😀😀\nsecond line with ascii text\n"; - let max_bytes = 32; - let (out, tokens) = truncate_middle(s, max_bytes); - - assert!(out.contains("tokens truncated")); - assert!(!out.contains('\u{fffd}')); - assert_eq!(tokens, Some(tok.count(s) as u64)); - } - - #[test] - fn truncate_middle_prefers_newline_boundaries_2() { - let tok = Tokenizer::try_default().expect("load tokenizer"); - // Build a multi-line string of 20 numbered lines (each "NNN\n"). - let mut s = String::new(); - for i in 1..=20 { - s.push_str(&format!("{i:03}\n")); - } - assert_eq!(s.len(), 80); - - let max_bytes = 64; - let (out, total) = truncate_middle(&s, max_bytes); - assert!(out.starts_with("001\n002\n003\n004\n")); - assert!(out.contains("tokens truncated")); - assert!(out.ends_with("017\n018\n019\n020\n")); - assert_eq!(total, Some(tok.count(&s) as u64)); - } - - #[test] - fn truncate_output_to_tokens_returns_original_when_under_limit() { + fn truncate_middle_returns_original_when_under_limit() { let s = "short output"; - let (truncated, original) = truncate_output_to_tokens(s, 100); - assert_eq!(truncated, s); + let limit = 100; + let source = TruncationSource::Policy(TruncationPolicy::Tokens(limit)); + let (out, original) = truncate_with_token_budget(s, limit, source); + assert_eq!(out, s); assert_eq!(original, None); } #[test] - fn truncate_output_to_tokens_reports_truncation_at_zero_limit() { + fn truncate_middle_reports_truncation_at_zero_limit() { let s = "abcdef"; - let (truncated, original) = truncate_output_to_tokens(s, 0); - assert!(truncated.contains("tokens truncated")); - assert_eq!(original, Some(s.chars().count())); + let source = TruncationSource::Policy(TruncationPolicy::Tokens(0)); + let (out, original) = truncate_with_token_budget(s, 0, source); + assert_eq!(out, "[…2 tokens truncated…]"); + assert_eq!(original, Some(approx_token_count(s) as u64)); } #[test] - fn truncate_output_to_tokens_preserves_prefix_and_suffix() { - let s = "abcdefghijklmnopqrstuvwxyz"; - let max_tokens = 10; - let (truncated, original) = truncate_output_to_tokens(s, max_tokens); - assert!(truncated.starts_with("abcde")); - assert!(truncated.ends_with("vwxyz")); - assert_eq!(original, Some(s.chars().count())); + fn truncate_middle_enforces_token_budget() { + let s = "alpha beta gamma delta epsilon zeta eta theta iota kappa"; + let max_tokens = 12; + let source = TruncationSource::Policy(TruncationPolicy::Tokens(max_tokens)); + let (out, original) = truncate_with_token_budget(s, max_tokens, source); + assert!(out.contains("tokens truncated")); + assert_eq!(original, Some(approx_token_count(s) as u64)); + assert!(out.len() < s.len(), "truncated output should be shorter"); + } + + #[test] + fn truncate_middle_handles_utf8_content() { + let s = "😀😀😀😀😀😀😀😀😀😀\nsecond line with text\n"; + let max_tokens = 8; + let source = TruncationSource::Policy(TruncationPolicy::Tokens(max_tokens)); + let (out, tokens) = truncate_with_token_budget(s, max_tokens, source); + + assert!(out.contains("tokens truncated")); + assert!(!out.contains('\u{fffd}')); + assert_eq!(tokens, Some(approx_token_count(s) as u64)); + assert!(out.len() < s.len(), "UTF-8 content should be shortened"); } #[test] @@ -440,11 +504,7 @@ mod tests { let line = "very long execution error line that should trigger truncation\n"; let large_error = line.repeat(2_500); // way beyond both byte and line limits - let truncated = format_output_for_model_body( - &large_error, - MODEL_FORMAT_MAX_BYTES, - MODEL_FORMAT_MAX_LINES, - ); + let truncated = truncate_with_line_bytes_budget(&large_error, model_format_max_bytes()); let total_lines = large_error.lines().count(); let pattern = truncated_message_pattern(line, total_lines); @@ -459,7 +519,7 @@ mod tests { .expect("missing body capture") .as_str(); assert!( - body.len() <= MODEL_FORMAT_MAX_BYTES, + body.len() <= model_format_max_bytes(), "body exceeds byte limit: {} bytes", body.len() ); @@ -468,16 +528,14 @@ mod tests { #[test] fn format_exec_output_marks_byte_truncation_without_omitted_lines() { - let long_line = "a".repeat(MODEL_FORMAT_MAX_BYTES + 50); - let truncated = format_output_for_model_body( - &long_line, - MODEL_FORMAT_MAX_BYTES, - MODEL_FORMAT_MAX_LINES, - ); + let max_bytes = model_format_max_bytes(); + let long_line = "a".repeat(max_bytes + 50); + let truncated = truncate_with_line_bytes_budget(&long_line, max_bytes); assert_ne!(truncated, long_line); + let removed_bytes = long_line.len().saturating_sub(max_bytes); let marker_line = - format!("[... output truncated to fit {MODEL_FORMAT_MAX_BYTES} bytes ...]"); + format!("[... removed {removed_bytes} bytes to fit {max_bytes} byte limit ...]"); assert!( truncated.contains(&marker_line), "missing byte truncation marker: {truncated}" @@ -493,7 +551,7 @@ mod tests { let content = "example output\n".repeat(10); assert_eq!( - format_output_for_model_body(&content, MODEL_FORMAT_MAX_BYTES, MODEL_FORMAT_MAX_LINES), + truncate_with_line_bytes_budget(&content, model_format_max_bytes()), content ); } @@ -505,8 +563,7 @@ mod tests { .map(|idx| format!("line-{idx}\n")) .collect(); - let truncated = - format_output_for_model_body(&content, MODEL_FORMAT_MAX_BYTES, MODEL_FORMAT_MAX_LINES); + let truncated = truncate_with_line_bytes_budget(&content, model_format_max_bytes()); let omitted = total_lines - MODEL_FORMAT_MAX_LINES; let expected_marker = format!("[... omitted {omitted} of {total_lines} lines ...]"); @@ -535,39 +592,33 @@ mod tests { .map(|idx| format!("line-{idx}-{long_line}\n")) .collect(); - let truncated = - format_output_for_model_body(&content, MODEL_FORMAT_MAX_BYTES, MODEL_FORMAT_MAX_LINES); + let truncated = truncate_with_line_bytes_budget(&content, model_format_max_bytes()); assert!( truncated.contains("[... omitted 42 of 298 lines ...]"), "expected omitted marker when line count exceeds limit: {truncated}" ); assert!( - !truncated.contains("output truncated to fit"), + !truncated.contains("byte limit"), "line omission marker should take precedence over byte marker: {truncated}" ); } #[test] fn truncates_across_multiple_under_limit_texts_and_reports_omitted() { - // Arrange: several text items, none exceeding per-item limit, but total exceeds budget. - let budget = MODEL_FORMAT_MAX_BYTES; - let t1_len = (budget / 2).saturating_sub(10); - let t2_len = (budget / 2).saturating_sub(10); - let remaining_after_t1_t2 = budget.saturating_sub(t1_len + t2_len); - let t3_len = 50; // gets truncated to remaining_after_t1_t2 - let t4_len = 5; // omitted - let t5_len = 7; // omitted - - let t1 = "a".repeat(t1_len); - let t2 = "b".repeat(t2_len); - let t3 = "c".repeat(t3_len); - let t4 = "d".repeat(t4_len); - let t5 = "e".repeat(t5_len); + let chunk = "alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu nu xi omicron pi rho sigma tau upsilon phi chi psi omega.\n"; + let chunk_tokens = approx_token_count(chunk); + assert!(chunk_tokens > 0, "chunk must consume tokens"); + let limit = chunk_tokens * 3; + let t1 = chunk.to_string(); + let t2 = chunk.to_string(); + let t3 = chunk.repeat(10); + let t4 = chunk.to_string(); + let t5 = chunk.to_string(); let items = vec![ - FunctionCallOutputContentItem::InputText { text: t1 }, - FunctionCallOutputContentItem::InputText { text: t2 }, + FunctionCallOutputContentItem::InputText { text: t1.clone() }, + FunctionCallOutputContentItem::InputText { text: t2.clone() }, FunctionCallOutputContentItem::InputImage { image_url: "img:mid".to_string(), }, @@ -576,7 +627,8 @@ mod tests { FunctionCallOutputContentItem::InputText { text: t5 }, ]; - let output = globally_truncate_function_output_items(&items); + let output = + truncate_function_output_items_with_policy(&items, TruncationPolicy::Tokens(limit)); // Expect: t1 (full), t2 (full), image, t3 (truncated), summary mentioning 2 omitted. assert_eq!(output.len(), 5); @@ -585,13 +637,13 @@ mod tests { FunctionCallOutputContentItem::InputText { text } => text, other => panic!("unexpected first item: {other:?}"), }; - assert_eq!(first_text.len(), t1_len); + assert_eq!(first_text, &t1); let second_text = match &output[1] { FunctionCallOutputContentItem::InputText { text } => text, other => panic!("unexpected second item: {other:?}"), }; - assert_eq!(second_text.len(), t2_len); + assert_eq!(second_text, &t2); assert_eq!( output[2], @@ -604,7 +656,10 @@ mod tests { FunctionCallOutputContentItem::InputText { text } => text, other => panic!("unexpected fourth item: {other:?}"), }; - assert_eq!(fourth_text.len(), remaining_after_t1_t2); + assert!( + fourth_text.contains("tokens truncated"), + "expected marker in truncated snippet: {fourth_text}" + ); let summary_text = match &output[4] { FunctionCallOutputContentItem::InputText { text } => text, diff --git a/codex-rs/core/src/unified_exec/mod.rs b/codex-rs/core/src/unified_exec/mod.rs index f77b744497..390401d789 100644 --- a/codex-rs/core/src/unified_exec/mod.rs +++ b/codex-rs/core/src/unified_exec/mod.rs @@ -45,6 +45,7 @@ pub(crate) const MIN_YIELD_TIME_MS: u64 = 250; pub(crate) const MAX_YIELD_TIME_MS: u64 = 30_000; pub(crate) const DEFAULT_MAX_OUTPUT_TOKENS: usize = 10_000; pub(crate) const UNIFIED_EXEC_OUTPUT_MAX_BYTES: usize = 1024 * 1024; // 1 MiB +pub(crate) const UNIFIED_EXEC_OUTPUT_MAX_TOKENS: usize = UNIFIED_EXEC_OUTPUT_MAX_BYTES / 4; pub(crate) struct UnifiedExecContext { pub session: Arc, diff --git a/codex-rs/core/src/unified_exec/session.rs b/codex-rs/core/src/unified_exec/session.rs index bdb935f171..82d6e41370 100644 --- a/codex-rs/core/src/unified_exec/session.rs +++ b/codex-rs/core/src/unified_exec/session.rs @@ -14,11 +14,13 @@ use crate::exec::ExecToolCallOutput; use crate::exec::SandboxType; use crate::exec::StreamOutput; use crate::exec::is_likely_sandbox_denied; -use crate::truncate::truncate_middle; +use crate::truncate::TruncationPolicy; +use crate::truncate::truncate_text; use codex_utils_pty::ExecCommandSession; use codex_utils_pty::SpawnedPty; use super::UNIFIED_EXEC_OUTPUT_MAX_BYTES; +use super::UNIFIED_EXEC_OUTPUT_MAX_TOKENS; use super::UnifiedExecError; #[derive(Debug, Default)] @@ -165,7 +167,10 @@ impl UnifiedExecSession { }; if is_likely_sandbox_denied(self.sandbox_type(), &exec_output) { - let (snippet, _) = truncate_middle(&aggregated_text, UNIFIED_EXEC_OUTPUT_MAX_BYTES); + let snippet = truncate_text( + &aggregated_text, + TruncationPolicy::Tokens(UNIFIED_EXEC_OUTPUT_MAX_TOKENS), + ); let message = if snippet.is_empty() { format!("exit code {exit_code}") } else { diff --git a/codex-rs/core/src/unified_exec/session_manager.rs b/codex-rs/core/src/unified_exec/session_manager.rs index fee46df8b8..57c60f2b84 100644 --- a/codex-rs/core/src/unified_exec/session_manager.rs +++ b/codex-rs/core/src/unified_exec/session_manager.rs @@ -23,6 +23,9 @@ use crate::tools::orchestrator::ToolOrchestrator; use crate::tools::runtimes::unified_exec::UnifiedExecRequest as UnifiedExecToolRequest; use crate::tools::runtimes::unified_exec::UnifiedExecRuntime; use crate::tools::sandboxing::ToolCtx; +use crate::truncate::TruncationPolicy; +use crate::truncate::approx_token_count; +use crate::truncate::truncate_text; use super::ExecCommandRequest; use super::SessionEntry; @@ -36,7 +39,6 @@ use super::generate_chunk_id; use super::resolve_max_tokens; use super::session::OutputBuffer; use super::session::UnifiedExecSession; -use crate::truncate::truncate_output_to_tokens; impl UnifiedExecSessionManager { pub(crate) async fn exec_command( @@ -70,7 +72,7 @@ impl UnifiedExecSessionManager { let wall_time = Instant::now().saturating_duration_since(start); let text = String::from_utf8_lossy(&collected).to_string(); - let (output, original_token_count) = truncate_output_to_tokens(&text, max_tokens); + let output = truncate_text(&text, TruncationPolicy::Tokens(max_tokens)); let chunk_id = generate_chunk_id(); let has_exited = session.has_exited(); let stored_id = self @@ -85,6 +87,8 @@ impl UnifiedExecSessionManager { // Only include a session_id in the response if the process is still alive. let session_id = if has_exited { None } else { Some(stored_id) }; + let original_token_count = approx_token_count(&text); + let response = UnifiedExecResponse { event_call_id: context.call_id.clone(), chunk_id, @@ -92,7 +96,7 @@ impl UnifiedExecSessionManager { output, session_id, exit_code: exit_code.flatten(), - original_token_count, + original_token_count: Some(original_token_count), session_command: Some(request.command.clone()), }; @@ -175,7 +179,8 @@ impl UnifiedExecSessionManager { let wall_time = Instant::now().saturating_duration_since(start); let text = String::from_utf8_lossy(&collected).to_string(); - let (output, original_token_count) = truncate_output_to_tokens(&text, max_tokens); + let output = truncate_text(&text, TruncationPolicy::Tokens(max_tokens)); + let original_token_count = approx_token_count(&text); let chunk_id = generate_chunk_id(); let status = self.refresh_session_state(session_id).await; @@ -199,7 +204,7 @@ impl UnifiedExecSessionManager { output, session_id, exit_code, - original_token_count, + original_token_count: Some(original_token_count), session_command: Some(session_command.clone()), }; diff --git a/codex-rs/core/tests/suite/truncation.rs b/codex-rs/core/tests/suite/truncation.rs index 200d27e887..3cbbc6bd57 100644 --- a/codex-rs/core/tests/suite/truncation.rs +++ b/codex-rs/core/tests/suite/truncation.rs @@ -27,7 +27,6 @@ use core_test_support::skip_if_no_network; use core_test_support::test_codex::test_codex; use core_test_support::wait_for_event; use escargot::CargoBuild; -use regex_lite::Regex; use serde_json::Value; use serde_json::json; use std::collections::HashMap; @@ -48,7 +47,7 @@ async fn truncate_function_error_trims_respond_to_model() -> Result<()> { let test = builder.build(&server).await?; // Construct a very long, non-existent path to force a RespondToModel error with a large message - let long_path = "a".repeat(20_000); + let long_path = "long path text should trigger truncation".repeat(8_000); let call_id = "grep-huge-error"; let args = json!({ "pattern": "alpha", @@ -80,12 +79,16 @@ async fn truncate_function_error_trims_respond_to_model() -> Result<()> { tracing::debug!(output = %output, "truncated function error output"); - // Expect plaintext with byte-truncation marker and no omitted-lines marker + // Expect plaintext with token-based truncation marker and no omitted-lines marker assert!( serde_json::from_str::(&output).is_err(), "expected error output to be plain text", ); - let truncated_pattern = r#"(?s)^Total output lines: 1\s+.*\[\.\.\. output truncated to fit 11264 bytes \.\.\.\]\s*$"#; + assert!( + !output.contains("Total output lines:"), + "error output should not include line-based truncation header: {output}", + ); + let truncated_pattern = r"(?s)^unable to access `.*tokens truncated.*$"; assert_regex_match(truncated_pattern, &output); assert!( !output.contains("omitted"), @@ -269,7 +272,7 @@ async fn mcp_tool_call_output_exceeds_limit_truncated_for_model() -> Result<()> let tool_name = format!("mcp__{server_name}__echo"); // Build a very large message to exceed 10KiB once serialized. - let large_msg = "long-message-with-newlines-".repeat(600); + let large_msg = "long-message-with-newlines-".repeat(6000); let args_json = serde_json::json!({ "message": large_msg }); mount_sse_once( @@ -334,22 +337,19 @@ async fn mcp_tool_call_output_exceeds_limit_truncated_for_model() -> Result<()> .function_call_output_text(call_id) .context("function_call_output present for rmcp call")?; - // Expect plain text with byte-based truncation marker. + // Expect plain text with token-based truncation marker; the original JSON body + // is truncated in the middle of the echo string. assert!( serde_json::from_str::(&output).is_err(), "expected truncated MCP output to be plain text" ); assert!( - output.starts_with("Total output lines: 1\n\n{"), - "expected total line header and JSON head, got: {output}" + !output.contains("Total output lines:"), + "MCP output should not include line-based truncation header: {output}" ); - let byte_marker = Regex::new(r"\[\.\.\. output truncated to fit 11264 bytes \.\.\.\]") - .expect("compile regex"); - assert!( - byte_marker.is_match(&output), - "expected byte truncation marker, got: {output}" - ); + let truncated_pattern = r#"(?s)^\{"echo":\s*"ECHOING: long-message-with-newlines-.*tokens truncated.*long-message-with-newlines-.*$"#; + assert_regex_match(truncated_pattern, &output); Ok(()) } @@ -453,3 +453,164 @@ async fn mcp_image_output_preserves_image_and_no_text_summary() -> Result<()> { Ok(()) } + +// Token-based policy should report token counts even when truncation is byte-estimated. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn token_policy_marker_reports_tokens() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let mut builder = test_codex().with_config(|config| { + config.model = "gpt-5.1-codex".to_string(); // token policy + config.model_family = + find_family_for_model("gpt-5.1-codex").expect("model family for gpt-5.1-codex"); + config.tool_output_token_limit = Some(50); // small budget to force truncation + }); + let fixture = builder.build(&server).await?; + + let call_id = "shell-token-marker"; + let args = json!({ + "command": ["/bin/sh", "-c", "seq 1 150"], + "timeout_ms": 5_000, + }); + + mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-1"), + ev_function_call(call_id, "shell", &serde_json::to_string(&args)?), + ev_completed("resp-1"), + ]), + ) + .await; + let done_mock = mount_sse_once( + &server, + sse(vec![ + ev_assistant_message("msg-1", "done"), + ev_completed("resp-2"), + ]), + ) + .await; + + fixture + .submit_turn_with_policy("run the shell tool", SandboxPolicy::DangerFullAccess) + .await?; + + let output = done_mock + .single_request() + .function_call_output_text(call_id) + .context("shell output present")?; + + assert_regex_match(r"\[\u{2026}127 tokens truncated\u{2026}]", &output); + + Ok(()) +} + +// Byte-based policy should report bytes removed. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn byte_policy_marker_reports_bytes() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let mut builder = test_codex().with_config(|config| { + config.model = "gpt-5.1".to_string(); // byte policy + config.model_family = find_family_for_model("gpt-5.1").expect("model family for gpt-5.1"); + config.tool_output_token_limit = Some(50); // ~200 byte cap + }); + let fixture = builder.build(&server).await?; + + let call_id = "shell-byte-marker"; + let args = json!({ + "command": ["/bin/sh", "-c", "seq 1 150"], + "timeout_ms": 5_000, + }); + + mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-1"), + ev_function_call(call_id, "shell", &serde_json::to_string(&args)?), + ev_completed("resp-1"), + ]), + ) + .await; + let done_mock = mount_sse_once( + &server, + sse(vec![ + ev_assistant_message("msg-1", "done"), + ev_completed("resp-2"), + ]), + ) + .await; + + fixture + .submit_turn_with_policy("run the shell tool", SandboxPolicy::DangerFullAccess) + .await?; + + let output = done_mock + .single_request() + .function_call_output_text(call_id) + .context("shell output present")?; + + assert_regex_match(r"\[\u{2026}505 bytes truncated\u{2026}]", &output); + + Ok(()) +} + +// Overriding config with a large token budget should avoid truncation. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn large_budget_avoids_truncation() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let mut builder = test_codex().with_config(|config| { + config.model = "gpt-5.1-codex".to_string(); + config.model_family = + find_family_for_model("gpt-5.1-codex").expect("model family for gpt-5.1-codex"); + config.tool_output_token_limit = Some(50_000); // ample budget + }); + let fixture = builder.build(&server).await?; + + let call_id = "shell-no-trunc"; + let args = json!({ + "command": ["/bin/sh", "-c", "seq 1 1000"], + "timeout_ms": 5_000, + }); + + mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-1"), + ev_function_call(call_id, "shell", &serde_json::to_string(&args)?), + ev_completed("resp-1"), + ]), + ) + .await; + let done_mock = mount_sse_once( + &server, + sse(vec![ + ev_assistant_message("msg-1", "done"), + ev_completed("resp-2"), + ]), + ) + .await; + + fixture + .submit_turn_with_policy( + "run big output without truncation", + SandboxPolicy::DangerFullAccess, + ) + .await?; + + let output = done_mock + .single_request() + .function_call_output_text(call_id) + .context("shell output present")?; + + assert!( + !output.contains("truncated"), + "output should remain untruncated with ample budget" + ); + + Ok(()) +} diff --git a/codex-rs/core/tests/suite/unified_exec.rs b/codex-rs/core/tests/suite/unified_exec.rs index 970be5277e..23f2c62b9c 100644 --- a/codex-rs/core/tests/suite/unified_exec.rs +++ b/codex-rs/core/tests/suite/unified_exec.rs @@ -26,9 +26,11 @@ use core_test_support::test_codex::TestCodex; use core_test_support::test_codex::test_codex; use core_test_support::wait_for_event; use core_test_support::wait_for_event_match; +use core_test_support::wait_for_event_with_timeout; use regex_lite::Regex; use serde_json::Value; use serde_json::json; +use tokio::time::Duration; fn extract_output_text(item: &Value) -> Option<&str> { item.get("output").and_then(|value| match value { @@ -814,7 +816,7 @@ async fn exec_command_reports_chunk_and_exit_metadata() -> Result<()> { let call_id = "uexec-metadata"; let args = serde_json::json!({ - "cmd": "printf 'abcdefghijklmnopqrstuvwxyz'", + "cmd": "printf 'token one token two token three token four token five token six token seven'", "yield_time_ms": 500, "max_output_tokens": 6, }); @@ -1295,7 +1297,7 @@ async fn unified_exec_streams_after_lagged_output() -> Result<()> { import sys import time -chunk = b'x' * (1 << 20) +chunk = b'long content here to trigger truncation' * (1 << 10) for _ in range(4): sys.stdout.buffer.write(chunk) sys.stdout.flush() @@ -1365,8 +1367,13 @@ PY summary: ReasoningSummary::Auto, }) .await?; - - wait_for_event(&codex, |event| matches!(event, EventMsg::TaskComplete(_))).await; + // This is a worst case scenario for the truncate logic. + wait_for_event_with_timeout( + &codex, + |event| matches!(event, EventMsg::TaskComplete(_)), + Duration::from_secs(10), + ) + .await; let requests = server.received_requests().await.expect("recorded requests"); assert!(!requests.is_empty(), "expected at least one POST request"); @@ -1523,14 +1530,15 @@ async fn unified_exec_formats_large_output_summary() -> Result<()> { } = builder.build(&server).await?; let script = r#"python3 - <<'PY' -for i in range(300): - print(f"line-{i}") +for i in range(10000): + print("token token ") PY "#; let call_id = "uexec-large-output"; let args = serde_json::json!({ "cmd": script, + "max_output_tokens": 100, "yield_time_ms": 500, }); @@ -1577,15 +1585,14 @@ PY let outputs = collect_tool_outputs(&bodies)?; let large_output = outputs.get(call_id).expect("missing large output summary"); - assert_regex_match( - concat!( - r"(?s)", - r"line-0.*?", - r"\[\.{3} omitted \d+ of \d+ lines \.{3}\].*?", - r"line-299", - ), - &large_output.output, - ); + let output_text = large_output.output.replace("\r\n", "\n"); + let truncated_pattern = r#"(?s)^(token token \n){5,}.*\[\u{2026}\d+ tokens truncated\u{2026}]\n(token token \n){5,}$"#; + assert_regex_match(truncated_pattern, &output_text); + + let original_tokens = large_output + .original_token_count + .expect("missing original_token_count for large output summary"); + assert!(original_tokens > 0); Ok(()) } diff --git a/codex-rs/core/tests/suite/user_shell_cmd.rs b/codex-rs/core/tests/suite/user_shell_cmd.rs index 0d42c45c1c..0e9585ba4b 100644 --- a/codex-rs/core/tests/suite/user_shell_cmd.rs +++ b/codex-rs/core/tests/suite/user_shell_cmd.rs @@ -272,7 +272,7 @@ async fn user_shell_command_is_truncated_only_once() -> anyhow::Result<()> { let mut builder = test_codex().with_config(|config| { config.model = "gpt-5.1-codex".to_string(); config.model_family = - find_family_for_model("gpt-5.1-codex").expect("gpt-5.1-codex is a model family"); + find_family_for_model("gpt-5-codex").expect("gpt-5-codex is a model family"); }); let fixture = builder.build(&server).await?; diff --git a/codex-rs/utils/tokenizer/src/lib.rs b/codex-rs/utils/tokenizer/src/lib.rs index 23ab261fa6..1e151393a9 100644 --- a/codex-rs/utils/tokenizer/src/lib.rs +++ b/codex-rs/utils/tokenizer/src/lib.rs @@ -122,6 +122,11 @@ impl Tokenizer { } } +impl fmt::Debug for Tokenizer { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Tokenizer {{ inner: }}") + } +} #[cfg(test)] mod tests { use super::*; diff --git a/docs/config.md b/docs/config.md index 51867773a9..ddfe2ff7c4 100644 --- a/docs/config.md +++ b/docs/config.md @@ -925,6 +925,7 @@ Valid values: | `model_provider` | string | Provider id from `model_providers` (default: `openai`). | | `model_context_window` | number | Context window tokens. | | `model_max_output_tokens` | number | Max output tokens. | +| `tool_output_token_limit` | number | Token budget for stored function/tool outputs in history (default: 2,560 tokens). | | `approval_policy` | `untrusted` \| `on-failure` \| `on-request` \| `never` | When to prompt for approval. | | `sandbox_mode` | `read-only` \| `workspace-write` \| `danger-full-access` | OS sandbox policy. | | `sandbox_workspace_write.writable_roots` | array | Extra writable roots in workspace‑write. | diff --git a/docs/example-config.md b/docs/example-config.md index 21b360ccef..b2da427314 100644 --- a/docs/example-config.md +++ b/docs/example-config.md @@ -33,6 +33,7 @@ model_provider = "openai" # model_context_window = 128000 # tokens; default: auto for model # model_max_output_tokens = 8192 # tokens; default: auto for model # model_auto_compact_token_limit = 0 # disable/override auto; default: model family specific +# tool_output_token_limit = 10000 # tokens stored per tool output; default: 10000 for gpt-5.1-codex ################################################################################ # Reasoning & Verbosity (Responses API capable models) From 4bada5a84d8d29aaec1f2634d118ec1aae3d2179 Mon Sep 17 00:00:00 2001 From: iceweasel-oai Date: Tue, 18 Nov 2025 11:38:18 -0800 Subject: [PATCH 02/18] Prompt to turn on windows sandbox when auto mode selected. (#6618) - stop prompting users to install WSL - prompt users to turn on Windows sandbox when auto mode requested. Screenshot 2025-11-17 110612 Screenshot 2025-11-17 110637 Screenshot 2025-11-17 110719 --- codex-rs/app-server-protocol/src/export.rs | 1 + codex-rs/common/src/approval_presets.rs | 10 +- codex-rs/core/src/config/edit.rs | 9 + codex-rs/core/src/config/mod.rs | 10 + codex-rs/tui/src/app.rs | 83 ++++++- codex-rs/tui/src/app_event.rs | 12 +- codex-rs/tui/src/chatwidget.rs | 190 ++++++++-------- ...get__tests__approvals_selection_popup.snap | 14 +- ...ts__approvals_selection_popup@windows.snap | 16 +- codex-rs/tui/src/chatwidget/tests.rs | 58 ++--- codex-rs/tui/src/lib.rs | 46 +--- codex-rs/tui/src/onboarding/mod.rs | 3 - .../tui/src/onboarding/onboarding_screen.rs | 29 --- codex-rs/tui/src/onboarding/windows.rs | 205 ------------------ codex-rs/windows-sandbox-rs/src/audit.rs | 30 +++ codex-rs/windows-sandbox-rs/src/lib.rs | 10 + 16 files changed, 298 insertions(+), 428 deletions(-) delete mode 100644 codex-rs/tui/src/onboarding/windows.rs diff --git a/codex-rs/app-server-protocol/src/export.rs b/codex-rs/app-server-protocol/src/export.rs index 11296e8e54..68d6249314 100644 --- a/codex-rs/app-server-protocol/src/export.rs +++ b/codex-rs/app-server-protocol/src/export.rs @@ -708,6 +708,7 @@ mod tests { use uuid::Uuid; #[test] + #[ignore = "timing out"] fn generated_ts_has_no_optional_nullable_fields() -> Result<()> { // Assert that there are no types of the form "?: T | null" in the generated TS files. let output_dir = std::env::temp_dir().join(format!("codex_ts_types_{}", Uuid::now_v7())); diff --git a/codex-rs/common/src/approval_presets.rs b/codex-rs/common/src/approval_presets.rs index 6c3bf395ad..1b673d1d96 100644 --- a/codex-rs/common/src/approval_presets.rs +++ b/codex-rs/common/src/approval_presets.rs @@ -24,21 +24,21 @@ pub fn builtin_approval_presets() -> Vec { ApprovalPreset { id: "read-only", label: "Read Only", - description: "Codex can read files and answer questions. Codex requires approval to make edits, run commands, or access network.", + description: "Requires approval to edit files and run commands.", approval: AskForApproval::OnRequest, sandbox: SandboxPolicy::ReadOnly, }, ApprovalPreset { id: "auto", - label: "Auto", - description: "Codex can read files, make edits, and run commands in the workspace. Codex requires approval to work outside the workspace or access network.", + label: "Agent", + description: "Read and edit files, and run commands.", approval: AskForApproval::OnRequest, sandbox: SandboxPolicy::new_workspace_write_policy(), }, ApprovalPreset { id: "full-access", - label: "Full Access", - description: "Codex can read files, make edits, and run commands with network access, without approval. Exercise caution.", + label: "Agent (full access)", + description: "Codex can edit files outside this workspace and run commands with network access. Exercise caution when using.", approval: AskForApproval::Never, sandbox: SandboxPolicy::DangerFullAccess, }, diff --git a/codex-rs/core/src/config/edit.rs b/codex-rs/core/src/config/edit.rs index 2d58ffc604..34659d0598 100644 --- a/codex-rs/core/src/config/edit.rs +++ b/codex-rs/core/src/config/edit.rs @@ -550,6 +550,15 @@ impl ConfigEditsBuilder { self } + /// Enable or disable a feature flag by key under the `[features]` table. + pub fn set_feature_enabled(mut self, key: &str, enabled: bool) -> Self { + self.edits.push(ConfigEdit::SetPath { + segments: vec!["features".to_string(), key.to_string()], + value: value(enabled), + }); + self + } + /// Apply edits on a blocking thread. pub fn apply_blocking(self) -> anyhow::Result<()> { apply_blocking(&self.codex_home, self.profile.as_deref(), &self.edits) diff --git a/codex-rs/core/src/config/mod.rs b/codex-rs/core/src/config/mod.rs index 0c00da9a48..335636debd 100644 --- a/codex-rs/core/src/config/mod.rs +++ b/codex-rs/core/src/config/mod.rs @@ -1320,6 +1320,16 @@ impl Config { Ok(Some(s)) } } + + pub fn set_windows_sandbox_globally(&mut self, value: bool) { + crate::safety::set_windows_sandbox_enabled(value); + if value { + self.features.enable(Feature::WindowsSandbox); + } else { + self.features.disable(Feature::WindowsSandbox); + } + self.forced_auto_mode_downgraded_on_windows = !value; + } } fn default_model() -> String { diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index 9f996007f8..718c33af3c 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -23,8 +23,12 @@ use codex_core::AuthManager; use codex_core::ConversationManager; use codex_core::config::Config; use codex_core::config::edit::ConfigEditsBuilder; +#[cfg(target_os = "windows")] +use codex_core::features::Feature; use codex_core::model_family::find_family_for_model; use codex_core::protocol::FinalOutput; +#[cfg(target_os = "windows")] +use codex_core::protocol::Op; use codex_core::protocol::SessionSource; use codex_core::protocol::TokenUsage; use codex_core::protocol_config_types::ReasoningEffort as ReasoningEffortConfig; @@ -220,7 +224,7 @@ impl App { let enhanced_keys_supported = tui.enhanced_keys_supported(); - let chat_widget = match resume_selection { + let mut chat_widget = match resume_selection { ResumeSelection::StartFresh | ResumeSelection::Exit => { let init = crate::chatwidget::ChatWidgetInit { config: config.clone(), @@ -263,6 +267,8 @@ impl App { } }; + chat_widget.maybe_prompt_windows_sandbox_enable(); + let file_search = FileSearchManager::new(config.cwd.clone(), app_event_tx.clone()); #[cfg(not(debug_assertions))] let upgrade_version = crate::updates::get_upgrade_version(&config); @@ -537,8 +543,71 @@ impl App { AppEvent::OpenFeedbackConsent { category } => { self.chat_widget.open_feedback_consent(category); } - AppEvent::ShowWindowsAutoModeInstructions => { - self.chat_widget.open_windows_auto_mode_instructions(); + AppEvent::OpenWindowsSandboxEnablePrompt { preset } => { + self.chat_widget.open_windows_sandbox_enable_prompt(preset); + } + AppEvent::EnableWindowsSandboxForAuto { preset } => { + #[cfg(target_os = "windows")] + { + let profile = self.active_profile.as_deref(); + let feature_key = Feature::WindowsSandbox.key(); + match ConfigEditsBuilder::new(&self.config.codex_home) + .with_profile(profile) + .set_feature_enabled(feature_key, true) + .apply() + .await + { + Ok(()) => { + self.config.set_windows_sandbox_globally(true); + self.chat_widget.clear_forced_auto_mode_downgrade(); + if let Some((sample_paths, extra_count, failed_scan)) = + self.chat_widget.world_writable_warning_details() + { + self.app_event_tx.send( + AppEvent::OpenWorldWritableWarningConfirmation { + preset: Some(preset.clone()), + sample_paths, + extra_count, + failed_scan, + }, + ); + } else { + self.app_event_tx.send(AppEvent::CodexOp( + Op::OverrideTurnContext { + cwd: None, + approval_policy: Some(preset.approval), + sandbox_policy: Some(preset.sandbox.clone()), + model: None, + effort: None, + summary: None, + }, + )); + self.app_event_tx + .send(AppEvent::UpdateAskForApprovalPolicy(preset.approval)); + self.app_event_tx + .send(AppEvent::UpdateSandboxPolicy(preset.sandbox.clone())); + self.chat_widget.add_info_message( + "Enabled the Windows sandbox feature and switched to Auto mode." + .to_string(), + None, + ); + } + } + Err(err) => { + tracing::error!( + error = %err, + "failed to enable Windows sandbox feature" + ); + self.chat_widget.add_error_message(format!( + "Failed to enable the Windows sandbox feature: {err}" + )); + } + } + } + #[cfg(not(target_os = "windows"))] + { + let _ = preset; + } } AppEvent::PersistModelSelection { model, effort } => { let profile = self.active_profile.as_deref(); @@ -593,6 +662,13 @@ impl App { | codex_core::protocol::SandboxPolicy::ReadOnly ); + self.config.sandbox_policy = policy.clone(); + #[cfg(target_os = "windows")] + if !matches!(policy, codex_core::protocol::SandboxPolicy::ReadOnly) + || codex_core::get_platform_sandbox().is_some() + { + self.config.forced_auto_mode_downgraded_on_windows = false; + } self.chat_widget.set_sandbox_policy(policy); // If sandbox policy becomes workspace-write or read-only, run the Windows world-writable scan. @@ -868,7 +944,6 @@ mod tests { fn make_test_app() -> App { let (chat_widget, app_event_tx, _rx, _op_rx) = make_chatwidget_manual_with_sender(); let config = chat_widget.config_ref().clone(); - let server = Arc::new(ConversationManager::with_auth(CodexAuth::from_api_key( "Test API Key", ))); diff --git a/codex-rs/tui/src/app_event.rs b/codex-rs/tui/src/app_event.rs index 0c3033c5c9..e786372524 100644 --- a/codex-rs/tui/src/app_event.rs +++ b/codex-rs/tui/src/app_event.rs @@ -91,9 +91,17 @@ pub(crate) enum AppEvent { failed_scan: bool, }, - /// Show Windows Subsystem for Linux setup instructions for auto mode. + /// Prompt to enable the Windows sandbox feature before using Auto mode. #[cfg_attr(not(target_os = "windows"), allow(dead_code))] - ShowWindowsAutoModeInstructions, + OpenWindowsSandboxEnablePrompt { + preset: ApprovalPreset, + }, + + /// Enable the Windows sandbox feature and switch to Auto mode. + #[cfg_attr(not(target_os = "windows"), allow(dead_code))] + EnableWindowsSandboxForAuto { + preset: ApprovalPreset, + }, /// Update the current approval policy in the running app and widget. UpdateAskForApprovalPolicy(AskForApproval), diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index cdcc78abad..4e9f496ba3 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -95,8 +95,6 @@ use crate::history_cell::HistoryCell; use crate::history_cell::McpToolCallCell; use crate::history_cell::PlainHistoryCell; use crate::markdown::append_markdown; -#[cfg(target_os = "windows")] -use crate::onboarding::WSL_INSTRUCTIONS; use crate::render::Insets; use crate::render::renderable::ColumnRenderable; use crate::render::renderable::FlexRenderable; @@ -2172,40 +2170,16 @@ impl ChatWidget { let mut items: Vec = Vec::new(); let presets: Vec = builtin_approval_presets(); #[cfg(target_os = "windows")] - let header_renderable: Box = if self - .config - .forced_auto_mode_downgraded_on_windows - { - use ratatui_macros::line; - - let mut header = ColumnRenderable::new(); - header.push(line![ - "Codex forced your settings back to Read Only on this Windows machine.".bold() - ]); - header.push(line![ - "To re-enable Auto mode, run Codex inside Windows Subsystem for Linux (WSL) or enable Full Access manually.".dim() - ]); - Box::new(header) - } else { - Box::new(()) - }; + let forced_windows_read_only = self.config.forced_auto_mode_downgraded_on_windows + && codex_core::get_platform_sandbox().is_none(); #[cfg(not(target_os = "windows"))] - let header_renderable: Box = Box::new(()); + let forced_windows_read_only = false; for preset in presets.into_iter() { let is_current = current_approval == preset.approval && current_sandbox == preset.sandbox; let name = preset.label.to_string(); let description_text = preset.description; - let description = if cfg!(target_os = "windows") - && preset.id == "auto" - && codex_core::get_platform_sandbox().is_none() - { - Some(format!( - "{description_text}\nRequires Windows Subsystem for Linux (WSL). Show installation instructions..." - )) - } else { - Some(description_text.to_string()) - }; + let description = Some(description_text.to_string()); let requires_confirmation = preset.id == "full-access" && !self .config @@ -2223,53 +2197,16 @@ impl ChatWidget { #[cfg(target_os = "windows")] { if codex_core::get_platform_sandbox().is_none() { - vec![Box::new(|tx| { - tx.send(AppEvent::ShowWindowsAutoModeInstructions); + let preset_clone = preset.clone(); + vec![Box::new(move |tx| { + tx.send(AppEvent::OpenWindowsSandboxEnablePrompt { + preset: preset_clone.clone(), + }); })] - } else if !self - .config - .notices - .hide_world_writable_warning - .unwrap_or(false) - && self.windows_world_writable_flagged() + } else if let Some((sample_paths, extra_count, failed_scan)) = + self.world_writable_warning_details() { let preset_clone = preset.clone(); - // Compute sample paths for the warning popup. - let mut env_map: std::collections::HashMap = - std::collections::HashMap::new(); - for (k, v) in std::env::vars() { - env_map.insert(k, v); - } - let (sample_paths, extra_count, failed_scan) = - match codex_windows_sandbox::preflight_audit_everyone_writable( - &self.config.cwd, - &env_map, - Some(self.config.codex_home.as_path()), - ) { - Ok(paths) if !paths.is_empty() => { - fn normalize_windows_path_for_display( - p: &std::path::Path, - ) -> String { - let canon = dunce::canonicalize(p) - .unwrap_or_else(|_| p.to_path_buf()); - canon.display().to_string().replace('/', "\\") - } - let as_strings: Vec = paths - .iter() - .map(|p| normalize_windows_path_for_display(p)) - .collect(); - let samples: Vec = - as_strings.iter().take(3).cloned().collect(); - let extra = if as_strings.len() > samples.len() { - as_strings.len() - samples.len() - } else { - 0 - }; - (samples, extra, false) - } - Err(_) => (Vec::new(), 0, true), - _ => (Vec::new(), 0, false), - }; vec![Box::new(move |tx| { tx.send(AppEvent::OpenWorldWritableWarningConfirmation { preset: Some(preset_clone.clone()), @@ -2300,10 +2237,17 @@ impl ChatWidget { } self.bottom_pane.show_selection_view(SelectionViewParams { - title: Some("Select Approval Mode".to_string()), + title: Some( + if forced_windows_read_only { + "Select approval mode (Codex changed your permissions to Read Only because the Windows sandbox is off)" + .to_string() + } else { + "Select Approval Mode".to_string() + }, + ), footer_hint: Some(standard_popup_hint_line()), items, - header: header_renderable, + header: Box::new(()), ..Default::default() }); } @@ -2328,20 +2272,22 @@ impl ChatWidget { } #[cfg(target_os = "windows")] - fn windows_world_writable_flagged(&self) -> bool { - use std::collections::HashMap; - let mut env_map: HashMap = HashMap::new(); - for (k, v) in std::env::vars() { - env_map.insert(k, v); - } - match codex_windows_sandbox::preflight_audit_everyone_writable( - &self.config.cwd, - &env_map, - Some(self.config.codex_home.as_path()), - ) { - Ok(paths) => !paths.is_empty(), - Err(_) => true, + pub(crate) fn world_writable_warning_details(&self) -> Option<(Vec, usize, bool)> { + if self + .config + .notices + .hide_world_writable_warning + .unwrap_or(false) + { + return None; } + codex_windows_sandbox::world_writable_warning_details(self.config.codex_home.as_path()) + } + + #[cfg(not(target_os = "windows"))] + #[allow(dead_code)] + pub(crate) fn world_writable_warning_details(&self) -> Option<(Vec, usize, bool)> { + None } pub(crate) fn open_full_access_confirmation(&mut self, preset: ApprovalPreset) { @@ -2426,7 +2372,6 @@ impl ChatWidget { SandboxPolicy::ReadOnly => "Read-Only mode", _ => "Auto mode", }; - let title_line = Line::from("Unprotected directories found").bold(); let info_line = if failed_scan { Line::from(vec![ "We couldn't complete the world-writable scan, so protections cannot be verified. " @@ -2443,7 +2388,6 @@ impl ChatWidget { .fg(Color::Red), ]) }; - header_children.push(Box::new(title_line)); header_children.push(Box::new( Paragraph::new(vec![info_line]).wrap(Wrap { trim: false }), )); @@ -2452,8 +2396,9 @@ impl ChatWidget { // Show up to three examples and optionally an "and X more" line. let mut lines: Vec = Vec::new(); lines.push(Line::from("Examples:").bold()); + lines.push(Line::from("")); for p in &sample_paths { - lines.push(Line::from(format!(" - {p}"))); + lines.push(Line::from(format!(" - {p}"))); } if extra_count > 0 { lines.push(Line::from(format!("and {extra_count} more"))); @@ -2521,21 +2466,33 @@ impl ChatWidget { } #[cfg(target_os = "windows")] - pub(crate) fn open_windows_auto_mode_instructions(&mut self) { + pub(crate) fn open_windows_sandbox_enable_prompt(&mut self, preset: ApprovalPreset) { use ratatui_macros::line; let mut header = ColumnRenderable::new(); header.push(line![ - "Auto mode requires Windows Subsystem for Linux (WSL2).".bold() + "Auto mode requires the experimental Windows sandbox.".bold(), + " Turn it on to enable sandboxed commands on Windows." ]); - header.push(line!["Run Codex inside WSL to enable sandboxed commands."]); - header.push(line![""]); - header.push(Paragraph::new(WSL_INSTRUCTIONS).wrap(Wrap { trim: false })); + let preset_clone = preset; let items = vec![SelectionItem { - name: "Back".to_string(), + name: "Turn on Windows sandbox and use Auto mode".to_string(), description: Some( - "Return to the approval mode list. Auto mode stays disabled outside WSL." + "Adds enable_experimental_windows_sandbox = true to config.toml and switches to Auto mode." + .to_string(), + ), + actions: vec![Box::new(move |tx| { + tx.send(AppEvent::EnableWindowsSandboxForAuto { + preset: preset_clone.clone(), + }); + })], + dismiss_on_select: true, + ..Default::default() + }, SelectionItem { + name: "Go Back".to_string(), + description: Some( + "Stay on read-only or full access without enabling the sandbox feature." .to_string(), ), actions: vec![Box::new(|tx| { @@ -2555,7 +2512,31 @@ impl ChatWidget { } #[cfg(not(target_os = "windows"))] - pub(crate) fn open_windows_auto_mode_instructions(&mut self) {} + pub(crate) fn open_windows_sandbox_enable_prompt(&mut self, _preset: ApprovalPreset) {} + + #[cfg(target_os = "windows")] + pub(crate) fn maybe_prompt_windows_sandbox_enable(&mut self) { + if self.config.forced_auto_mode_downgraded_on_windows + && codex_core::get_platform_sandbox().is_none() + && let Some(preset) = builtin_approval_presets() + .into_iter() + .find(|preset| preset.id == "auto") + { + self.open_windows_sandbox_enable_prompt(preset); + } + } + + #[cfg(not(target_os = "windows"))] + pub(crate) fn maybe_prompt_windows_sandbox_enable(&mut self) {} + + #[cfg(target_os = "windows")] + pub(crate) fn clear_forced_auto_mode_downgrade(&mut self) { + self.config.forced_auto_mode_downgraded_on_windows = false; + } + + #[cfg(not(target_os = "windows"))] + #[allow(dead_code)] + pub(crate) fn clear_forced_auto_mode_downgrade(&mut self) {} /// Set the approval policy in the widget's config copy. pub(crate) fn set_approval_policy(&mut self, policy: AskForApproval) { @@ -2564,7 +2545,16 @@ impl ChatWidget { /// Set the sandbox policy in the widget's config copy. pub(crate) fn set_sandbox_policy(&mut self, policy: SandboxPolicy) { + #[cfg(target_os = "windows")] + let should_clear_downgrade = !matches!(policy, SandboxPolicy::ReadOnly) + || codex_core::get_platform_sandbox().is_some(); + self.config.sandbox_policy = policy; + + #[cfg(target_os = "windows")] + if should_clear_downgrade { + self.config.forced_auto_mode_downgraded_on_windows = false; + } } pub(crate) fn set_full_access_warning_acknowledged(&mut self, acknowledged: bool) { diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__approvals_selection_popup.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__approvals_selection_popup.snap index 190594b1b3..6758ec62c5 100644 --- a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__approvals_selection_popup.snap +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__approvals_selection_popup.snap @@ -4,14 +4,10 @@ expression: popup --- Select Approval Mode -› 1. Read Only (current) Codex can read files and answer questions. Codex - requires approval to make edits, run commands, or - access network. - 2. Auto Codex can read files, make edits, and run commands - in the workspace. Codex requires approval to work - outside the workspace or access network. - 3. Full Access Codex can read files, make edits, and run commands - with network access, without approval. Exercise - caution. +› 1. Read Only (current) Requires approval to edit files and run commands. + 2. Agent Read and edit files, and run commands. + 3. Agent (full access) Codex can edit files outside this workspace and run + commands with network access. Exercise caution when + using. Press enter to confirm or esc to go back diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__approvals_selection_popup@windows.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__approvals_selection_popup@windows.snap index 7d16ad57b8..6758ec62c5 100644 --- a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__approvals_selection_popup@windows.snap +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__approvals_selection_popup@windows.snap @@ -4,16 +4,10 @@ expression: popup --- Select Approval Mode -› 1. Read Only (current) Codex can read files and answer questions. Codex - requires approval to make edits, run commands, or - access network. - 2. Auto Codex can read files, make edits, and run commands - in the workspace. Codex requires approval to work - outside the workspace or access network. - Requires Windows Subsystem for Linux (WSL). Show - installation instructions... - 3. Full Access Codex can read files, make edits, and run commands - with network access, without approval. Exercise - caution. +› 1. Read Only (current) Requires approval to edit files and run commands. + 2. Agent Read and edit files, and run commands. + 3. Agent (full access) Codex can edit files outside this workspace and run + commands with network access. Exercise caution when + using. Press enter to confirm or esc to go back diff --git a/codex-rs/tui/src/chatwidget/tests.rs b/codex-rs/tui/src/chatwidget/tests.rs index abd9a61238..013b5c2313 100644 --- a/codex-rs/tui/src/chatwidget/tests.rs +++ b/codex-rs/tui/src/chatwidget/tests.rs @@ -58,6 +58,11 @@ use tempfile::tempdir; use tokio::sync::mpsc::error::TryRecvError; use tokio::sync::mpsc::unbounded_channel; +#[cfg(target_os = "windows")] +fn set_windows_sandbox_enabled(enabled: bool) { + codex_core::set_windows_sandbox_enabled(enabled); +} + fn test_config() -> Config { // Use base defaults to avoid depending on host state. Config::load_from_base_config_with_overrides( @@ -1456,28 +1461,6 @@ fn approvals_selection_popup_snapshot() { assert_snapshot!("approvals_selection_popup", popup); } -#[test] -fn approvals_popup_includes_wsl_note_for_auto_mode() { - let (mut chat, _rx, _op_rx) = make_chatwidget_manual(); - - if cfg!(target_os = "windows") { - chat.config.forced_auto_mode_downgraded_on_windows = true; - } - chat.open_approvals_popup(); - - let popup = render_bottom_popup(&chat, 80); - assert_eq!( - popup.contains("Requires Windows Subsystem for Linux (WSL)"), - cfg!(target_os = "windows"), - "expected auto preset description to mention WSL requirement only on Windows, popup: {popup}" - ); - assert_eq!( - popup.contains("Codex forced your settings back to Read Only on this Windows machine."), - cfg!(target_os = "windows") && chat.config.forced_auto_mode_downgraded_on_windows, - "expected downgrade notice only when auto mode is forced off on Windows, popup: {popup}" - ); -} - #[test] fn full_access_confirmation_popup_snapshot() { let (mut chat, _rx, _op_rx) = make_chatwidget_manual(); @@ -1494,18 +1477,41 @@ fn full_access_confirmation_popup_snapshot() { #[cfg(target_os = "windows")] #[test] -fn windows_auto_mode_instructions_popup_lists_install_steps() { +fn windows_auto_mode_prompt_requests_enabling_sandbox_feature() { let (mut chat, _rx, _op_rx) = make_chatwidget_manual(); - chat.open_windows_auto_mode_instructions(); + let preset = builtin_approval_presets() + .into_iter() + .find(|preset| preset.id == "auto") + .expect("auto preset"); + chat.open_windows_sandbox_enable_prompt(preset); let popup = render_bottom_popup(&chat, 120); assert!( - popup.contains("wsl --install"), - "expected WSL instructions popup to include install command, popup: {popup}" + popup.contains("experimental Windows sandbox"), + "expected auto mode prompt to mention enabling the sandbox feature, popup: {popup}" ); } +#[cfg(target_os = "windows")] +#[test] +fn startup_prompts_for_windows_sandbox_when_auto_requested() { + let (mut chat, _rx, _op_rx) = make_chatwidget_manual(); + + set_windows_sandbox_enabled(false); + chat.config.forced_auto_mode_downgraded_on_windows = true; + + chat.maybe_prompt_windows_sandbox_enable(); + + let popup = render_bottom_popup(&chat, 120); + assert!( + popup.contains("Turn on Windows sandbox and use Auto mode"), + "expected startup prompt to offer enabling the sandbox: {popup}" + ); + + set_windows_sandbox_enabled(true); +} + #[test] fn model_reasoning_selection_popup_snapshot() { let (mut chat, _rx, _op_rx) = make_chatwidget_manual(); diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index ca7cba9c24..ea1dc7523a 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -86,7 +86,7 @@ mod wrapping; #[cfg(test)] pub mod test_backend; -use crate::onboarding::WSL_INSTRUCTIONS; +use crate::onboarding::TrustDirectorySelection; use crate::onboarding::onboarding_screen::OnboardingScreenArgs; use crate::onboarding::onboarding_screen::run_onboarding_app; use crate::tui::Tui; @@ -389,20 +389,13 @@ async fn run_ratatui_app( ); let login_status = get_login_status(&initial_config); let should_show_trust_screen = should_show_trust_screen(&initial_config); - let should_show_windows_wsl_screen = - cfg!(target_os = "windows") && !initial_config.windows_wsl_setup_acknowledged; - let should_show_onboarding = should_show_onboarding( - login_status, - &initial_config, - should_show_trust_screen, - should_show_windows_wsl_screen, - ); + let should_show_onboarding = + should_show_onboarding(login_status, &initial_config, should_show_trust_screen); let config = if should_show_onboarding { let onboarding_result = run_onboarding_app( OnboardingScreenArgs { show_login_screen: should_show_login_screen(login_status, &initial_config), - show_windows_wsl_screen: should_show_windows_wsl_screen, show_trust_screen: should_show_trust_screen, login_status, auth_manager: auth_manager.clone(), @@ -421,21 +414,12 @@ async fn run_ratatui_app( update_action: None, }); } - if onboarding_result.windows_install_selected { - restore(); - session_log::log_session_end(); - let _ = tui.terminal.clear(); - if let Err(err) = writeln!(std::io::stdout(), "{WSL_INSTRUCTIONS}") { - tracing::error!("Failed to write WSL instructions: {err}"); - } - return Ok(AppExitInfo { - token_usage: codex_core::protocol::TokenUsage::default(), - conversation_id: None, - update_action: None, - }); - } - // if the user acknowledged windows or made any trust decision, reload the config accordingly - if should_show_windows_wsl_screen || onboarding_result.directory_trust_decision.is_some() { + // if the user acknowledged windows or made an explicit decision ato trust the directory, reload the config accordingly + if onboarding_result + .directory_trust_decision + .map(|d| d == TrustDirectorySelection::Trust) + .unwrap_or(false) + { load_config_or_exit(cli_kv_overrides, overrides).await } else { initial_config @@ -584,7 +568,7 @@ async fn load_config_or_exit( /// show the trust screen. fn should_show_trust_screen(config: &Config) -> bool { if cfg!(target_os = "windows") && get_platform_sandbox().is_none() { - // If the experimental sandbox is not enabled, Native Windows cannot enforce sandboxed write access without WSL; skip the trust prompt entirely. + // If the experimental sandbox is not enabled, Native Windows cannot enforce sandboxed write access; skip the trust prompt entirely. return false; } if config.did_user_set_custom_approval_policy_or_sandbox_mode { @@ -599,12 +583,7 @@ fn should_show_onboarding( login_status: LoginStatus, config: &Config, show_trust_screen: bool, - show_windows_wsl_screen: bool, ) -> bool { - if show_windows_wsl_screen { - return true; - } - if show_trust_screen { return true; } @@ -628,7 +607,6 @@ mod tests { use codex_core::config::ConfigOverrides; use codex_core::config::ConfigToml; use codex_core::config::ProjectConfig; - use codex_core::set_windows_sandbox_enabled; use serial_test::serial; use tempfile::TempDir; @@ -643,7 +621,7 @@ mod tests { )?; config.did_user_set_custom_approval_policy_or_sandbox_mode = false; config.active_project = ProjectConfig { trust_level: None }; - set_windows_sandbox_enabled(false); + config.set_windows_sandbox_globally(false); let should_show = should_show_trust_screen(&config); if cfg!(target_os = "windows") { @@ -670,7 +648,7 @@ mod tests { )?; config.did_user_set_custom_approval_policy_or_sandbox_mode = false; config.active_project = ProjectConfig { trust_level: None }; - set_windows_sandbox_enabled(true); + config.set_windows_sandbox_globally(true); let should_show = should_show_trust_screen(&config); if cfg!(target_os = "windows") { diff --git a/codex-rs/tui/src/onboarding/mod.rs b/codex-rs/tui/src/onboarding/mod.rs index 6c420dae53..d4cfd6d1f4 100644 --- a/codex-rs/tui/src/onboarding/mod.rs +++ b/codex-rs/tui/src/onboarding/mod.rs @@ -3,6 +3,3 @@ pub mod onboarding_screen; mod trust_directory; pub use trust_directory::TrustDirectorySelection; mod welcome; -mod windows; - -pub(crate) use windows::WSL_INSTRUCTIONS; diff --git a/codex-rs/tui/src/onboarding/onboarding_screen.rs b/codex-rs/tui/src/onboarding/onboarding_screen.rs index 02c7be7ad2..b085f28888 100644 --- a/codex-rs/tui/src/onboarding/onboarding_screen.rs +++ b/codex-rs/tui/src/onboarding/onboarding_screen.rs @@ -20,7 +20,6 @@ use crate::onboarding::auth::SignInState; use crate::onboarding::trust_directory::TrustDirectorySelection; use crate::onboarding::trust_directory::TrustDirectoryWidget; use crate::onboarding::welcome::WelcomeWidget; -use crate::onboarding::windows::WindowsSetupWidget; use crate::tui::FrameRequester; use crate::tui::Tui; use crate::tui::TuiEvent; @@ -30,7 +29,6 @@ use std::sync::RwLock; #[allow(clippy::large_enum_variant)] enum Step { - Windows(WindowsSetupWidget), Welcome(WelcomeWidget), Auth(AuthModeWidget), TrustDirectory(TrustDirectoryWidget), @@ -56,12 +54,10 @@ pub(crate) struct OnboardingScreen { request_frame: FrameRequester, steps: Vec, is_done: bool, - windows_install_selected: bool, should_exit: bool, } pub(crate) struct OnboardingScreenArgs { - pub show_windows_wsl_screen: bool, pub show_trust_screen: bool, pub show_login_screen: bool, pub login_status: LoginStatus, @@ -71,14 +67,12 @@ pub(crate) struct OnboardingScreenArgs { pub(crate) struct OnboardingResult { pub directory_trust_decision: Option, - pub windows_install_selected: bool, pub should_exit: bool, } impl OnboardingScreen { pub(crate) fn new(tui: &mut Tui, args: OnboardingScreenArgs) -> Self { let OnboardingScreenArgs { - show_windows_wsl_screen, show_trust_screen, show_login_screen, login_status, @@ -91,9 +85,6 @@ impl OnboardingScreen { let codex_home = config.codex_home; let cli_auth_credentials_store_mode = config.cli_auth_credentials_store_mode; let mut steps: Vec = Vec::new(); - if show_windows_wsl_screen { - steps.push(Step::Windows(WindowsSetupWidget::new(codex_home.clone()))); - } steps.push(Step::Welcome(WelcomeWidget::new( !matches!(login_status, LoginStatus::NotAuthenticated), tui.frame_requester(), @@ -138,7 +129,6 @@ impl OnboardingScreen { request_frame: tui.frame_requester(), steps, is_done: false, - windows_install_selected: false, should_exit: false, } } @@ -200,10 +190,6 @@ impl OnboardingScreen { .flatten() } - pub fn windows_install_selected(&self) -> bool { - self.windows_install_selected - } - pub fn should_exit(&self) -> bool { self.should_exit } @@ -249,14 +235,6 @@ impl KeyboardHandler for OnboardingScreen { } } }; - if self - .steps - .iter() - .any(|step| matches!(step, Step::Windows(widget) if widget.exit_requested())) - { - self.windows_install_selected = true; - self.is_done = true; - } self.request_frame.schedule_frame(); } @@ -338,7 +316,6 @@ impl WidgetRef for &OnboardingScreen { impl KeyboardHandler for Step { fn handle_key_event(&mut self, key_event: KeyEvent) { match self { - Step::Windows(widget) => widget.handle_key_event(key_event), Step::Welcome(widget) => widget.handle_key_event(key_event), Step::Auth(widget) => widget.handle_key_event(key_event), Step::TrustDirectory(widget) => widget.handle_key_event(key_event), @@ -347,7 +324,6 @@ impl KeyboardHandler for Step { fn handle_paste(&mut self, pasted: String) { match self { - Step::Windows(_) => {} Step::Welcome(_) => {} Step::Auth(widget) => widget.handle_paste(pasted), Step::TrustDirectory(widget) => widget.handle_paste(pasted), @@ -358,7 +334,6 @@ impl KeyboardHandler for Step { impl StepStateProvider for Step { fn get_step_state(&self) -> StepState { match self { - Step::Windows(w) => w.get_step_state(), Step::Welcome(w) => w.get_step_state(), Step::Auth(w) => w.get_step_state(), Step::TrustDirectory(w) => w.get_step_state(), @@ -369,9 +344,6 @@ impl StepStateProvider for Step { impl WidgetRef for Step { fn render_ref(&self, area: Rect, buf: &mut Buffer) { match self { - Step::Windows(widget) => { - widget.render_ref(area, buf); - } Step::Welcome(widget) => { widget.render_ref(area, buf); } @@ -451,7 +423,6 @@ pub(crate) async fn run_onboarding_app( } Ok(OnboardingResult { directory_trust_decision: onboarding_screen.directory_trust_decision(), - windows_install_selected: onboarding_screen.windows_install_selected(), should_exit: onboarding_screen.should_exit(), }) } diff --git a/codex-rs/tui/src/onboarding/windows.rs b/codex-rs/tui/src/onboarding/windows.rs deleted file mode 100644 index 715611b7e1..0000000000 --- a/codex-rs/tui/src/onboarding/windows.rs +++ /dev/null @@ -1,205 +0,0 @@ -use std::path::PathBuf; - -use codex_core::config::edit::ConfigEditsBuilder; -use crossterm::event::KeyCode; -use crossterm::event::KeyEvent; -use crossterm::event::KeyEventKind; -use ratatui::buffer::Buffer; -use ratatui::layout::Rect; -use ratatui::prelude::Widget; -use ratatui::style::Color; -use ratatui::style::Stylize; -use ratatui::text::Line; -use ratatui::widgets::Paragraph; -use ratatui::widgets::WidgetRef; -use ratatui::widgets::Wrap; - -use crate::onboarding::onboarding_screen::KeyboardHandler; -use crate::onboarding::onboarding_screen::StepStateProvider; - -use super::onboarding_screen::StepState; - -pub(crate) const WSL_INSTRUCTIONS: &str = r#"Install WSL2 by opening PowerShell as Administrator and running: - # Install WSL using the default Linux distribution (Ubuntu). - # See https://learn.microsoft.com/en-us/windows/wsl/install for more info - wsl --install - - # Restart your computer, then start a shell inside of Windows Subsystem for Linux - wsl - - # Install Node.js in WSL via nvm - # Documentation: https://learn.microsoft.com/en-us/windows/dev-environment/javascript/nodejs-on-wsl - curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/master/install.sh | bash && export NVM_DIR="$HOME/.nvm" && \. "$NVM_DIR/nvm.sh" - nvm install 22 - - # Install and run Codex in WSL - npm install --global @openai/codex - codex - - # Additional details and instructions for how to install and run Codex in WSL: - https://developers.openai.com/codex/windows"#; - -pub(crate) struct WindowsSetupWidget { - pub codex_home: PathBuf, - pub selection: Option, - pub highlighted: WindowsSetupSelection, - pub error: Option, - exit_requested: bool, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum WindowsSetupSelection { - Continue, - Install, -} - -impl WindowsSetupWidget { - pub fn new(codex_home: PathBuf) -> Self { - Self { - codex_home, - selection: None, - highlighted: WindowsSetupSelection::Install, - error: None, - exit_requested: false, - } - } - - fn handle_continue(&mut self) { - self.highlighted = WindowsSetupSelection::Continue; - match ConfigEditsBuilder::new(&self.codex_home) - .set_windows_wsl_setup_acknowledged(true) - .apply_blocking() - { - Ok(()) => { - self.selection = Some(WindowsSetupSelection::Continue); - self.exit_requested = false; - self.error = None; - } - Err(err) => { - tracing::error!("Failed to persist Windows onboarding acknowledgement: {err:?}"); - self.error = Some(format!("Failed to update config: {err}")); - self.selection = None; - } - } - } - - fn handle_install(&mut self) { - self.highlighted = WindowsSetupSelection::Install; - self.selection = Some(WindowsSetupSelection::Install); - self.exit_requested = true; - } - - pub fn exit_requested(&self) -> bool { - self.exit_requested - } -} - -impl WidgetRef for &WindowsSetupWidget { - fn render_ref(&self, area: Rect, buf: &mut Buffer) { - let mut lines: Vec = vec![ - Line::from(vec![ - "> ".into(), - "To use all Codex features, we recommend running Codex in Windows Subsystem for Linux (WSL2)".bold(), - ]), - Line::from(vec![" ".into(), "WSL allows Codex to run Agent mode in a sandboxed environment with better data protections in place.".into()]), - Line::from(vec![" ".into(), "Learn more: https://developers.openai.com/codex/windows".into()]), - Line::from(""), - ]; - - let create_option = - |idx: usize, option: WindowsSetupSelection, text: &str| -> Line<'static> { - if self.highlighted == option { - Line::from(format!("> {}. {text}", idx + 1)).cyan() - } else { - Line::from(format!(" {}. {}", idx + 1, text)) - } - }; - - lines.push(create_option( - 0, - WindowsSetupSelection::Install, - "Exit and install WSL2", - )); - lines.push(create_option( - 1, - WindowsSetupSelection::Continue, - "Continue anyway", - )); - lines.push("".into()); - - if let Some(error) = &self.error { - lines.push(Line::from(format!(" {error}")).fg(Color::Red)); - lines.push("".into()); - } - - lines.push(Line::from(vec![" Press Enter to continue".dim()])); - - Paragraph::new(lines) - .wrap(Wrap { trim: false }) - .render(area, buf); - } -} - -impl KeyboardHandler for WindowsSetupWidget { - fn handle_key_event(&mut self, key_event: KeyEvent) { - if key_event.kind == KeyEventKind::Release { - return; - } - - match key_event.code { - KeyCode::Up | KeyCode::Char('k') => { - self.highlighted = WindowsSetupSelection::Install; - } - KeyCode::Down | KeyCode::Char('j') => { - self.highlighted = WindowsSetupSelection::Continue; - } - KeyCode::Char('1') => self.handle_install(), - KeyCode::Char('2') => self.handle_continue(), - KeyCode::Enter => match self.highlighted { - WindowsSetupSelection::Install => self.handle_install(), - WindowsSetupSelection::Continue => self.handle_continue(), - }, - _ => {} - } - } -} - -impl StepStateProvider for WindowsSetupWidget { - fn get_step_state(&self) -> StepState { - match self.selection { - Some(WindowsSetupSelection::Continue) => StepState::Hidden, - Some(WindowsSetupSelection::Install) => StepState::Complete, - None => StepState::InProgress, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use tempfile::TempDir; - - #[test] - fn windows_step_hidden_after_continue() { - let temp_dir = TempDir::new().expect("temp dir"); - let mut widget = WindowsSetupWidget::new(temp_dir.path().to_path_buf()); - - assert_eq!(widget.get_step_state(), StepState::InProgress); - - widget.handle_continue(); - - assert_eq!(widget.get_step_state(), StepState::Hidden); - assert!(!widget.exit_requested()); - } - - #[test] - fn windows_step_complete_after_install_selection() { - let temp_dir = TempDir::new().expect("temp dir"); - let mut widget = WindowsSetupWidget::new(temp_dir.path().to_path_buf()); - - widget.handle_install(); - - assert_eq!(widget.get_step_state(), StepState::Complete); - assert!(widget.exit_requested()); - } -} diff --git a/codex-rs/windows-sandbox-rs/src/audit.rs b/codex-rs/windows-sandbox-rs/src/audit.rs index 4cd19dea62..383c652c7c 100644 --- a/codex-rs/windows-sandbox-rs/src/audit.rs +++ b/codex-rs/windows-sandbox-rs/src/audit.rs @@ -1,6 +1,7 @@ use crate::token::world_sid; use crate::winutil::to_wide; use anyhow::Result; +use std::collections::HashMap; use std::collections::HashSet; use std::ffi::c_void; use std::path::Path; @@ -275,6 +276,35 @@ pub fn audit_everyone_writable( ); Ok(Vec::new()) } + +fn normalize_windows_path_for_display(p: impl AsRef) -> String { + let canon = dunce::canonicalize(p.as_ref()).unwrap_or_else(|_| p.as_ref().to_path_buf()); + canon.display().to_string().replace('/', "\\") +} + +pub fn world_writable_warning_details( + codex_home: impl AsRef, +) -> Option<(Vec, usize, bool)> { + let cwd = match std::env::current_dir() { + Ok(cwd) => cwd, + Err(_) => return Some((Vec::new(), 0, true)), + }; + + let env_map: HashMap = std::env::vars().collect(); + match audit_everyone_writable(&cwd, &env_map, Some(codex_home.as_ref())) { + Ok(paths) if paths.is_empty() => None, + Ok(paths) => { + let as_strings: Vec = paths + .iter() + .map(normalize_windows_path_for_display) + .collect(); + let sample_paths: Vec = as_strings.iter().take(3).cloned().collect(); + let extra_count = as_strings.len().saturating_sub(sample_paths.len()); + Some((sample_paths, extra_count, false)) + } + Err(_) => Some((Vec::new(), 0, true)), + } +} // Fast mask-based check: does the DACL contain any ACCESS_ALLOWED ACE for // Everyone that includes generic or specific write bits? Skips inherit-only // ACEs (do not apply to the current object). diff --git a/codex-rs/windows-sandbox-rs/src/lib.rs b/codex-rs/windows-sandbox-rs/src/lib.rs index a2f63c8d45..a868654e7c 100644 --- a/codex-rs/windows-sandbox-rs/src/lib.rs +++ b/codex-rs/windows-sandbox-rs/src/lib.rs @@ -6,6 +6,8 @@ macro_rules! windows_modules { windows_modules!(acl, allow, audit, cap, env, logging, policy, token, winutil); +#[cfg(target_os = "windows")] +pub use audit::world_writable_warning_details; #[cfg(target_os = "windows")] pub use windows_impl::preflight_audit_everyone_writable; #[cfg(target_os = "windows")] @@ -18,6 +20,8 @@ pub use stub::preflight_audit_everyone_writable; #[cfg(not(target_os = "windows"))] pub use stub::run_windows_sandbox_capture; #[cfg(not(target_os = "windows"))] +pub use stub::world_writable_warning_details; +#[cfg(not(target_os = "windows"))] pub use stub::CaptureResult; #[cfg(target_os = "windows")] @@ -455,4 +459,10 @@ mod stub { ) -> Result { bail!("Windows sandbox is only available on Windows") } + + pub fn world_writable_warning_details( + _codex_home: impl AsRef, + ) -> Option<(Vec, usize, bool)> { + None + } } From 29ca89c414b9a2c1ca54b6ca7b972d60fa9a9986 Mon Sep 17 00:00:00 2001 From: Dylan Hurd Date: Tue, 18 Nov 2025 12:46:02 -0800 Subject: [PATCH 03/18] chore(config) enable shell_command (#6843) ## Summary Enables shell_command as default for `gpt-5*` and `codex-*` models. ## Testing - [x] Updated unit tests --- codex-rs/core/src/model_family.rs | 7 +++++-- codex-rs/core/src/tools/spec.rs | 20 ++++---------------- codex-rs/core/tests/suite/model_tools.rs | 16 +++------------- codex-rs/core/tests/suite/prompt_caching.rs | 8 ++++---- 4 files changed, 16 insertions(+), 35 deletions(-) diff --git a/codex-rs/core/src/model_family.rs b/codex-rs/core/src/model_family.rs index 0758d20310..ee827ff10e 100644 --- a/codex-rs/core/src/model_family.rs +++ b/codex-rs/core/src/model_family.rs @@ -149,6 +149,7 @@ pub fn find_family_for_model(slug: &str) -> Option { "test_sync_tool".to_string(), ], supports_parallel_tool_calls: true, + shell_type: ConfigShellToolType::ShellCommand, support_verbosity: true, truncation_policy: TruncationPolicy::Tokens(10_000), ) @@ -166,7 +167,7 @@ pub fn find_family_for_model(slug: &str) -> Option { "list_dir".to_string(), "read_file".to_string(), ], - shell_type: if cfg!(windows) { ConfigShellToolType::ShellCommand } else { ConfigShellToolType::Default }, + shell_type: ConfigShellToolType::ShellCommand, supports_parallel_tool_calls: true, support_verbosity: true, truncation_policy: TruncationPolicy::Tokens(10_000), @@ -183,7 +184,7 @@ pub fn find_family_for_model(slug: &str) -> Option { reasoning_summary_format: ReasoningSummaryFormat::Experimental, base_instructions: GPT_5_CODEX_INSTRUCTIONS.to_string(), apply_patch_tool_type: Some(ApplyPatchToolType::Freeform), - shell_type: if cfg!(windows) { ConfigShellToolType::ShellCommand } else { ConfigShellToolType::Default }, + shell_type: ConfigShellToolType::ShellCommand, supports_parallel_tool_calls: true, support_verbosity: false, truncation_policy: TruncationPolicy::Tokens(10_000), @@ -198,6 +199,7 @@ pub fn find_family_for_model(slug: &str) -> Option { base_instructions: GPT_5_1_INSTRUCTIONS.to_string(), default_reasoning_effort: Some(ReasoningEffort::Medium), truncation_policy: TruncationPolicy::Bytes(10_000), + shell_type: ConfigShellToolType::ShellCommand, supports_parallel_tool_calls: true, ) } else if slug.starts_with("gpt-5") { @@ -205,6 +207,7 @@ pub fn find_family_for_model(slug: &str) -> Option { slug, "gpt-5", supports_reasoning_summaries: true, needs_special_apply_patch_instructions: true, + shell_type: ConfigShellToolType::ShellCommand, support_verbosity: true, truncation_policy: TruncationPolicy::Bytes(10_000), ) diff --git a/codex-rs/core/src/tools/spec.rs b/codex-rs/core/src/tools/spec.rs index 88ddbf5517..32bcd6a241 100644 --- a/codex-rs/core/src/tools/spec.rs +++ b/codex-rs/core/src/tools/spec.rs @@ -1292,11 +1292,7 @@ mod tests { "gpt-5-codex", &Features::with_defaults(), &[ - if cfg!(windows) { - "shell_command" - } else { - "shell" - }, + "shell_command", "list_mcp_resources", "list_mcp_resource_templates", "read_mcp_resource", @@ -1313,11 +1309,7 @@ mod tests { "gpt-5.1-codex", &Features::with_defaults(), &[ - if cfg!(windows) { - "shell_command" - } else { - "shell" - }, + "shell_command", "list_mcp_resources", "list_mcp_resource_templates", "read_mcp_resource", @@ -1392,11 +1384,7 @@ mod tests { "gpt-5.1-codex-mini", &Features::with_defaults(), &[ - if cfg!(windows) { - "shell_command" - } else { - "shell" - }, + "shell_command", "list_mcp_resources", "list_mcp_resource_templates", "read_mcp_resource", @@ -1413,7 +1401,7 @@ mod tests { "gpt-5.1", &Features::with_defaults(), &[ - "shell", + "shell_command", "list_mcp_resources", "list_mcp_resource_templates", "read_mcp_resource", diff --git a/codex-rs/core/tests/suite/model_tools.rs b/codex-rs/core/tests/suite/model_tools.rs index 28593a55b8..3f5845b86f 100644 --- a/codex-rs/core/tests/suite/model_tools.rs +++ b/codex-rs/core/tests/suite/model_tools.rs @@ -119,12 +119,7 @@ async fn model_selects_expected_tools() { assert_eq!( gpt5_codex_tools, vec![ - if cfg!(windows) { - "shell_command" - } else { - "shell" - } - .to_string(), + "shell_command".to_string(), "list_mcp_resources".to_string(), "list_mcp_resource_templates".to_string(), "read_mcp_resource".to_string(), @@ -138,12 +133,7 @@ async fn model_selects_expected_tools() { assert_eq!( gpt51_codex_tools, vec![ - if cfg!(windows) { - "shell_command" - } else { - "shell" - } - .to_string(), + "shell_command".to_string(), "list_mcp_resources".to_string(), "list_mcp_resource_templates".to_string(), "read_mcp_resource".to_string(), @@ -157,7 +147,7 @@ async fn model_selects_expected_tools() { assert_eq!( gpt51_tools, vec![ - "shell".to_string(), + "shell_command".to_string(), "list_mcp_resources".to_string(), "list_mcp_resource_templates".to_string(), "read_mcp_resource".to_string(), diff --git a/codex-rs/core/tests/suite/prompt_caching.rs b/codex-rs/core/tests/suite/prompt_caching.rs index c858ea20fd..d93c2bd79d 100644 --- a/codex-rs/core/tests/suite/prompt_caching.rs +++ b/codex-rs/core/tests/suite/prompt_caching.rs @@ -162,7 +162,7 @@ async fn prompt_tools_are_consistent_across_requests() -> anyhow::Result<()> { ( "gpt-5.1", vec![ - "shell", + "shell_command", "list_mcp_resources", "list_mcp_resource_templates", "read_mcp_resource", @@ -173,7 +173,7 @@ async fn prompt_tools_are_consistent_across_requests() -> anyhow::Result<()> { ( "gpt-5.1", vec![ - "shell", + "shell_command", "list_mcp_resources", "list_mcp_resource_templates", "read_mcp_resource", @@ -185,7 +185,7 @@ async fn prompt_tools_are_consistent_across_requests() -> anyhow::Result<()> { ( "gpt-5.1-codex", vec![ - "shell", + "shell_command", "list_mcp_resources", "list_mcp_resource_templates", "read_mcp_resource", @@ -197,7 +197,7 @@ async fn prompt_tools_are_consistent_across_requests() -> anyhow::Result<()> { ( "gpt-5.1-codex", vec![ - "shell", + "shell_command", "list_mcp_resources", "list_mcp_resource_templates", "read_mcp_resource", From 8ddae8cde36443716e7db1e8fa57ef2b5e77882f Mon Sep 17 00:00:00 2001 From: jif-oai Date: Tue, 18 Nov 2025 21:58:54 +0000 Subject: [PATCH 04/18] feat: review in app server (#6613) --- .../src/protocol/common.rs | 4 + .../app-server-protocol/src/protocol/v2.rs | 39 +++ codex-rs/app-server/README.md | 53 ++++ .../app-server/src/bespoke_event_handling.rs | 50 ++++ .../app-server/src/codex_message_processor.rs | 150 ++++++++++ .../app-server/tests/common/mcp_process.rs | 10 + codex-rs/app-server/tests/suite/v2/mod.rs | 1 + codex-rs/app-server/tests/suite/v2/review.rs | 279 ++++++++++++++++++ codex-rs/core/src/codex.rs | 10 +- codex-rs/core/src/tasks/review.rs | 81 +++-- codex-rs/core/tests/suite/codex_delegate.rs | 3 + codex-rs/core/tests/suite/review.rs | 7 + codex-rs/protocol/src/protocol.rs | 4 +- codex-rs/tui/src/chatwidget.rs | 5 + codex-rs/tui/src/chatwidget/tests.rs | 2 + 15 files changed, 667 insertions(+), 31 deletions(-) create mode 100644 codex-rs/app-server/tests/suite/v2/review.rs diff --git a/codex-rs/app-server-protocol/src/protocol/common.rs b/codex-rs/app-server-protocol/src/protocol/common.rs index db9bed1112..f5408a6225 100644 --- a/codex-rs/app-server-protocol/src/protocol/common.rs +++ b/codex-rs/app-server-protocol/src/protocol/common.rs @@ -129,6 +129,10 @@ client_request_definitions! { params: v2::TurnInterruptParams, response: v2::TurnInterruptResponse, }, + ReviewStart => "review/start" { + params: v2::ReviewStartParams, + response: v2::TurnStartResponse, + }, ModelList => "model/list" { params: v2::ModelListParams, diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index a2b9cee3fe..f4a6d636aa 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -562,6 +562,45 @@ pub struct TurnStartParams { pub summary: Option, } +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ReviewStartParams { + pub thread_id: String, + pub target: ReviewTarget, + + /// When true, also append the final review message to the original thread. + #[serde(default)] + pub append_to_original_thread: bool, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(tag = "type", rename_all = "camelCase")] +#[ts(tag = "type", export_to = "v2/")] +pub enum ReviewTarget { + /// Review the working tree: staged, unstaged, and untracked files. + UncommittedChanges, + + /// Review changes between the current branch and the given base branch. + #[serde(rename_all = "camelCase")] + #[ts(rename_all = "camelCase")] + BaseBranch { branch: String }, + + /// Review the changes introduced by a specific commit. + #[serde(rename_all = "camelCase")] + #[ts(rename_all = "camelCase")] + Commit { + sha: String, + /// Optional human-readable label (e.g., commit subject) for UIs. + title: Option, + }, + + /// Arbitrary instructions, equivalent to the old free-form prompt. + #[serde(rename_all = "camelCase")] + #[ts(rename_all = "camelCase")] + Custom { instructions: String }, +} + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index 5f9b87458f..35653099aa 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -65,6 +65,7 @@ The JSON-RPC API exposes dedicated methods for managing Codex conversations. Thr - `thread/archive` — move a thread’s rollout file into the archived directory; returns `{}` on success. - `turn/start` — add user input to a thread and begin Codex generation; responds with the initial `turn` object and streams `turn/started`, `item/*`, and `turn/completed` notifications. - `turn/interrupt` — request cancellation of an in-flight turn by `(thread_id, turn_id)`; success is an empty `{}` response and the turn finishes with `status: "interrupted"`. +- `review/start` — kick off Codex’s automated reviewer for a thread; responds like `turn/start` and emits a `item/completed` notification with a `codeReview` item when results are ready. ### 1) Start or resume a thread @@ -181,6 +182,58 @@ You can cancel a running Turn with `turn/interrupt`. The server requests cancellations for running subprocesses, then emits a `turn/completed` event with `status: "interrupted"`. Rely on the `turn/completed` to know when Codex-side cleanup is done. +### 6) Request a code review + +Use `review/start` to run Codex’s reviewer on the currently checked-out project. The request takes the thread id plus a `target` describing what should be reviewed: + +- `{"type":"uncommittedChanges"}` — staged, unstaged, and untracked files. +- `{"type":"baseBranch","branch":"main"}` — diff against the provided branch’s upstream (see prompt for the exact `git merge-base`/`git diff` instructions Codex will run). +- `{"type":"commit","sha":"abc1234","title":"Optional subject"}` — review a specific commit. +- `{"type":"custom","instructions":"Free-form reviewer instructions"}` — fallback prompt equivalent to the legacy manual review request. +- `appendToOriginalThread` (bool, default `false`) — when `true`, Codex also records a final assistant-style message with the review summary in the original thread. When `false`, only the `codeReview` item is emitted for the review run and no extra message is added to the original thread. + +Example request/response: + +```json +{ "method": "review/start", "id": 40, "params": { + "threadId": "thr_123", + "appendToOriginalThread": true, + "target": { "type": "commit", "sha": "1234567deadbeef", "title": "Polish tui colors" } +} } +{ "id": 40, "result": { "turn": { + "id": "turn_900", + "status": "inProgress", + "items": [ + { "type": "userMessage", "id": "turn_900", "content": [ { "type": "text", "text": "Review commit 1234567: Polish tui colors" } ] } + ], + "error": null +} } } +``` + +Codex streams the usual `turn/started` notification followed by an `item/started` +with the same `codeReview` item id so clients can show progress: + +```json +{ "method": "item/started", "params": { "item": { + "type": "codeReview", + "id": "turn_900", + "review": "current changes" +} } } +``` + +When the reviewer finishes, the server emits `item/completed` containing the same +`codeReview` item with the final review text: + +```json +{ "method": "item/completed", "params": { "item": { + "type": "codeReview", + "id": "turn_900", + "review": "Looks solid overall...\n\n- Prefer Stylize helpers — app.rs:10-20\n ..." +} } } +``` + +The `review` string is plain text that already bundles the overall explanation plus a bullet list for each structured finding (matching `ThreadItem::CodeReview` in the generated schema). Use this notification to render the reviewer output in your client. + ## Auth endpoints The JSON-RPC auth/account surface exposes request/response methods plus server-initiated notifications (no `id`). Use these to determine auth state, start or cancel logins, logout, and inspect ChatGPT rate limits. diff --git a/codex-rs/app-server/src/bespoke_event_handling.rs b/codex-rs/app-server/src/bespoke_event_handling.rs index 8ed343f03a..ce4f7590e2 100644 --- a/codex-rs/app-server/src/bespoke_event_handling.rs +++ b/codex-rs/app-server/src/bespoke_event_handling.rs @@ -38,7 +38,9 @@ use codex_core::protocol::McpToolCallBeginEvent; use codex_core::protocol::McpToolCallEndEvent; use codex_core::protocol::Op; use codex_core::protocol::ReviewDecision; +use codex_core::review_format::format_review_findings_block; use codex_protocol::ConversationId; +use codex_protocol::protocol::ReviewOutputEvent; use std::convert::TryFrom; use std::sync::Arc; use tokio::sync::oneshot; @@ -189,6 +191,17 @@ pub(crate) async fn apply_bespoke_event_handling( .await; } } + EventMsg::EnteredReviewMode(review_request) => { + let notification = ItemStartedNotification { + item: ThreadItem::CodeReview { + id: event_id.clone(), + review: review_request.user_facing_hint, + }, + }; + outgoing + .send_server_notification(ServerNotification::ItemStarted(notification)) + .await; + } EventMsg::ItemStarted(item_started_event) => { let item: ThreadItem = item_started_event.item.clone().into(); let notification = ItemStartedNotification { item }; @@ -203,6 +216,21 @@ pub(crate) async fn apply_bespoke_event_handling( .send_server_notification(ServerNotification::ItemCompleted(notification)) .await; } + EventMsg::ExitedReviewMode(review_event) => { + let review_text = match review_event.review_output { + Some(output) => render_review_output_text(&output), + None => REVIEW_FALLBACK_MESSAGE.to_string(), + }; + let notification = ItemCompletedNotification { + item: ThreadItem::CodeReview { + id: event_id, + review: review_text, + }, + }; + outgoing + .send_server_notification(ServerNotification::ItemCompleted(notification)) + .await; + } EventMsg::ExecCommandBegin(exec_command_begin_event) => { let item = ThreadItem::CommandExecution { id: exec_command_begin_event.call_id.clone(), @@ -382,6 +410,28 @@ async fn on_exec_approval_response( } } +const REVIEW_FALLBACK_MESSAGE: &str = "Reviewer failed to output a response."; + +fn render_review_output_text(output: &ReviewOutputEvent) -> String { + let mut sections = Vec::new(); + let explanation = output.overall_explanation.trim(); + if !explanation.is_empty() { + sections.push(explanation.to_string()); + } + if !output.findings.is_empty() { + let findings = format_review_findings_block(&output.findings, None); + let trimmed = findings.trim(); + if !trimmed.is_empty() { + sections.push(trimmed.to_string()); + } + } + if sections.is_empty() { + REVIEW_FALLBACK_MESSAGE.to_string() + } else { + sections.join("\n\n") + } +} + async fn on_command_execution_request_approval_response( event_id: String, receiver: oneshot::Receiver, diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index c5fa2a7fa2..0172458074 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -60,6 +60,8 @@ use codex_app_server_protocol::RemoveConversationSubscriptionResponse; use codex_app_server_protocol::RequestId; use codex_app_server_protocol::ResumeConversationParams; use codex_app_server_protocol::ResumeConversationResponse; +use codex_app_server_protocol::ReviewStartParams; +use codex_app_server_protocol::ReviewTarget; use codex_app_server_protocol::SandboxMode; use codex_app_server_protocol::SendUserMessageParams; use codex_app_server_protocol::SendUserMessageResponse; @@ -115,6 +117,7 @@ use codex_core::git_info::git_diff_to_remote; use codex_core::parse_cursor; use codex_core::protocol::EventMsg; use codex_core::protocol::Op; +use codex_core::protocol::ReviewRequest; use codex_core::read_head_for_summary; use codex_feedback::CodexFeedback; use codex_login::ServerOptions as LoginServerOptions; @@ -232,6 +235,91 @@ impl CodexMessageProcessor { } } + fn review_request_from_target( + target: ReviewTarget, + append_to_original_thread: bool, + ) -> Result<(ReviewRequest, String), JSONRPCErrorError> { + fn invalid_request(message: String) -> JSONRPCErrorError { + JSONRPCErrorError { + code: INVALID_REQUEST_ERROR_CODE, + message, + data: None, + } + } + + match target { + // TODO(jif) those messages will be extracted in a follow-up PR. + ReviewTarget::UncommittedChanges => Ok(( + ReviewRequest { + prompt: "Review the current code changes (staged, unstaged, and untracked files) and provide prioritized findings.".to_string(), + user_facing_hint: "current changes".to_string(), + append_to_original_thread, + }, + "Review uncommitted changes".to_string(), + )), + ReviewTarget::BaseBranch { branch } => { + let branch = branch.trim().to_string(); + if branch.is_empty() { + return Err(invalid_request("branch must not be empty".to_string())); + } + let prompt = format!("Review the code changes against the base branch '{branch}'. Start by finding the merge diff between the current branch and {branch}'s upstream e.g. (`git merge-base HEAD \"$(git rev-parse --abbrev-ref \"{branch}@{{upstream}}\")\"`), then run `git diff` against that SHA to see what changes we would merge into the {branch} branch. Provide prioritized, actionable findings."); + let hint = format!("changes against '{branch}'"); + let display = format!("Review changes against base branch '{branch}'"); + Ok(( + ReviewRequest { + prompt, + user_facing_hint: hint, + append_to_original_thread, + }, + display, + )) + } + ReviewTarget::Commit { sha, title } => { + let sha = sha.trim().to_string(); + if sha.is_empty() { + return Err(invalid_request("sha must not be empty".to_string())); + } + let brief_title = title + .map(|t| t.trim().to_string()) + .filter(|t| !t.is_empty()); + let prompt = if let Some(title) = brief_title.clone() { + format!("Review the code changes introduced by commit {sha} (\"{title}\"). Provide prioritized, actionable findings.") + } else { + format!("Review the code changes introduced by commit {sha}. Provide prioritized, actionable findings.") + }; + let short_sha = sha.chars().take(7).collect::(); + let hint = format!("commit {short_sha}"); + let display = if let Some(title) = brief_title { + format!("Review commit {short_sha}: {title}") + } else { + format!("Review commit {short_sha}") + }; + Ok(( + ReviewRequest { + prompt, + user_facing_hint: hint, + append_to_original_thread, + }, + display, + )) + } + ReviewTarget::Custom { instructions } => { + let trimmed = instructions.trim().to_string(); + if trimmed.is_empty() { + return Err(invalid_request("instructions must not be empty".to_string())); + } + Ok(( + ReviewRequest { + prompt: trimmed.clone(), + user_facing_hint: trimmed.clone(), + append_to_original_thread, + }, + trimmed, + )) + } + } + } + pub async fn process_request(&mut self, request: ClientRequest) { match request { ClientRequest::Initialize { .. } => { @@ -263,6 +351,9 @@ impl CodexMessageProcessor { ClientRequest::TurnInterrupt { request_id, params } => { self.turn_interrupt(request_id, params).await; } + ClientRequest::ReviewStart { request_id, params } => { + self.review_start(request_id, params).await; + } ClientRequest::NewConversation { request_id, params } => { // Do not tokio::spawn() to process new_conversation() // asynchronously because we need to ensure the conversation is @@ -2342,6 +2433,65 @@ impl CodexMessageProcessor { } } + async fn review_start(&self, request_id: RequestId, params: ReviewStartParams) { + let ReviewStartParams { + thread_id, + target, + append_to_original_thread, + } = params; + let (_, conversation) = match self.conversation_from_thread_id(&thread_id).await { + Ok(v) => v, + Err(error) => { + self.outgoing.send_error(request_id, error).await; + return; + } + }; + + let (review_request, display_text) = + match Self::review_request_from_target(target, append_to_original_thread) { + Ok(value) => value, + Err(err) => { + self.outgoing.send_error(request_id, err).await; + return; + } + }; + + let turn_id = conversation.submit(Op::Review { review_request }).await; + + match turn_id { + Ok(turn_id) => { + let mut items = Vec::new(); + if !display_text.is_empty() { + items.push(ThreadItem::UserMessage { + id: turn_id.clone(), + content: vec![V2UserInput::Text { text: display_text }], + }); + } + let turn = Turn { + id: turn_id.clone(), + items, + status: TurnStatus::InProgress, + error: None, + }; + let response = TurnStartResponse { turn: turn.clone() }; + self.outgoing.send_response(request_id, response).await; + + let notif = TurnStartedNotification { turn }; + self.outgoing + .send_server_notification(ServerNotification::TurnStarted(notif)) + .await; + } + Err(err) => { + let error = JSONRPCErrorError { + code: INTERNAL_ERROR_CODE, + message: format!("failed to start review: {err}"), + data: None, + }; + self.outgoing.send_error(request_id, error).await; + } + } + } + async fn turn_interrupt(&mut self, request_id: RequestId, params: TurnInterruptParams) { let TurnInterruptParams { thread_id, .. } = params; diff --git a/codex-rs/app-server/tests/common/mcp_process.rs b/codex-rs/app-server/tests/common/mcp_process.rs index 75851eda2c..920a6fa01d 100644 --- a/codex-rs/app-server/tests/common/mcp_process.rs +++ b/codex-rs/app-server/tests/common/mcp_process.rs @@ -35,6 +35,7 @@ use codex_app_server_protocol::NewConversationParams; use codex_app_server_protocol::RemoveConversationListenerParams; use codex_app_server_protocol::RequestId; use codex_app_server_protocol::ResumeConversationParams; +use codex_app_server_protocol::ReviewStartParams; use codex_app_server_protocol::SendUserMessageParams; use codex_app_server_protocol::SendUserTurnParams; use codex_app_server_protocol::ServerRequest; @@ -377,6 +378,15 @@ impl McpProcess { self.send_request("turn/interrupt", params).await } + /// Send a `review/start` JSON-RPC request (v2). + pub async fn send_review_start_request( + &mut self, + params: ReviewStartParams, + ) -> anyhow::Result { + let params = Some(serde_json::to_value(params)?); + self.send_request("review/start", params).await + } + /// Send a `cancelLoginChatGpt` JSON-RPC request. pub async fn send_cancel_login_chat_gpt_request( &mut self, diff --git a/codex-rs/app-server/tests/suite/v2/mod.rs b/codex-rs/app-server/tests/suite/v2/mod.rs index 587afef108..a8594e7ca2 100644 --- a/codex-rs/app-server/tests/suite/v2/mod.rs +++ b/codex-rs/app-server/tests/suite/v2/mod.rs @@ -1,6 +1,7 @@ mod account; mod model_list; mod rate_limits; +mod review; mod thread_archive; mod thread_list; mod thread_resume; diff --git a/codex-rs/app-server/tests/suite/v2/review.rs b/codex-rs/app-server/tests/suite/v2/review.rs new file mode 100644 index 0000000000..194ed6ae96 --- /dev/null +++ b/codex-rs/app-server/tests/suite/v2/review.rs @@ -0,0 +1,279 @@ +use anyhow::Result; +use app_test_support::McpProcess; +use app_test_support::create_final_assistant_message_sse_response; +use app_test_support::create_mock_chat_completions_server_unchecked; +use app_test_support::to_response; +use codex_app_server_protocol::ItemCompletedNotification; +use codex_app_server_protocol::ItemStartedNotification; +use codex_app_server_protocol::JSONRPCError; +use codex_app_server_protocol::JSONRPCNotification; +use codex_app_server_protocol::JSONRPCResponse; +use codex_app_server_protocol::RequestId; +use codex_app_server_protocol::ReviewStartParams; +use codex_app_server_protocol::ReviewTarget; +use codex_app_server_protocol::ThreadItem; +use codex_app_server_protocol::ThreadStartParams; +use codex_app_server_protocol::ThreadStartResponse; +use codex_app_server_protocol::TurnStartResponse; +use codex_app_server_protocol::TurnStatus; +use serde_json::json; +use tempfile::TempDir; +use tokio::time::timeout; + +const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); +const INVALID_REQUEST_ERROR_CODE: i64 = -32600; + +#[tokio::test] +async fn review_start_runs_review_turn_and_emits_code_review_item() -> Result<()> { + let review_payload = json!({ + "findings": [ + { + "title": "Prefer Stylize helpers", + "body": "Use .dim()/.bold() chaining instead of manual Style.", + "confidence_score": 0.9, + "priority": 1, + "code_location": { + "absolute_file_path": "/tmp/file.rs", + "line_range": {"start": 10, "end": 20} + } + } + ], + "overall_correctness": "good", + "overall_explanation": "Looks solid overall with minor polish suggested.", + "overall_confidence_score": 0.75 + }) + .to_string(); + let responses = vec![create_final_assistant_message_sse_response( + &review_payload, + )?]; + let server = create_mock_chat_completions_server_unchecked(responses).await; + + let codex_home = TempDir::new()?; + create_config_toml(codex_home.path(), &server.uri())?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let thread_id = start_default_thread(&mut mcp).await?; + + let review_req = mcp + .send_review_start_request(ReviewStartParams { + thread_id: thread_id.clone(), + append_to_original_thread: true, + target: ReviewTarget::Commit { + sha: "1234567deadbeef".to_string(), + title: Some("Tidy UI colors".to_string()), + }, + }) + .await?; + let review_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(review_req)), + ) + .await??; + let TurnStartResponse { turn } = to_response::(review_resp)?; + let turn_id = turn.id.clone(); + assert_eq!(turn.status, TurnStatus::InProgress); + assert_eq!(turn.items.len(), 1); + match &turn.items[0] { + ThreadItem::UserMessage { content, .. } => { + assert_eq!(content.len(), 1); + assert!(matches!( + &content[0], + codex_app_server_protocol::UserInput::Text { .. } + )); + } + other => panic!("expected user message, got {other:?}"), + } + + let _started: JSONRPCNotification = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("turn/started"), + ) + .await??; + let item_started: JSONRPCNotification = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("item/started"), + ) + .await??; + let started: ItemStartedNotification = + serde_json::from_value(item_started.params.expect("params must be present"))?; + match started.item { + ThreadItem::CodeReview { id, review } => { + assert_eq!(id, turn_id); + assert_eq!(review, "commit 1234567"); + } + other => panic!("expected code review item, got {other:?}"), + } + + let mut review_body: Option = None; + for _ in 0..5 { + let review_notif: JSONRPCNotification = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("item/completed"), + ) + .await??; + let completed: ItemCompletedNotification = + serde_json::from_value(review_notif.params.expect("params must be present"))?; + match completed.item { + ThreadItem::CodeReview { id, review } => { + assert_eq!(id, turn_id); + review_body = Some(review); + break; + } + ThreadItem::UserMessage { .. } => continue, + other => panic!("unexpected item/completed payload: {other:?}"), + } + } + + let review = review_body.expect("did not observe a code review item"); + assert!(review.contains("Prefer Stylize helpers")); + assert!(review.contains("/tmp/file.rs:10-20")); + + Ok(()) +} + +#[tokio::test] +async fn review_start_rejects_empty_base_branch() -> Result<()> { + let server = create_mock_chat_completions_server_unchecked(vec![]).await; + let codex_home = TempDir::new()?; + create_config_toml(codex_home.path(), &server.uri())?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + let thread_id = start_default_thread(&mut mcp).await?; + + let request_id = mcp + .send_review_start_request(ReviewStartParams { + thread_id, + append_to_original_thread: true, + target: ReviewTarget::BaseBranch { + branch: " ".to_string(), + }, + }) + .await?; + let error: JSONRPCError = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_error_message(RequestId::Integer(request_id)), + ) + .await??; + assert_eq!(error.error.code, INVALID_REQUEST_ERROR_CODE); + assert!( + error.error.message.contains("branch must not be empty"), + "unexpected message: {}", + error.error.message + ); + + Ok(()) +} + +#[tokio::test] +async fn review_start_rejects_empty_commit_sha() -> Result<()> { + let server = create_mock_chat_completions_server_unchecked(vec![]).await; + let codex_home = TempDir::new()?; + create_config_toml(codex_home.path(), &server.uri())?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + let thread_id = start_default_thread(&mut mcp).await?; + + let request_id = mcp + .send_review_start_request(ReviewStartParams { + thread_id, + append_to_original_thread: true, + target: ReviewTarget::Commit { + sha: "\t".to_string(), + title: None, + }, + }) + .await?; + let error: JSONRPCError = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_error_message(RequestId::Integer(request_id)), + ) + .await??; + assert_eq!(error.error.code, INVALID_REQUEST_ERROR_CODE); + assert!( + error.error.message.contains("sha must not be empty"), + "unexpected message: {}", + error.error.message + ); + + Ok(()) +} + +#[tokio::test] +async fn review_start_rejects_empty_custom_instructions() -> Result<()> { + let server = create_mock_chat_completions_server_unchecked(vec![]).await; + let codex_home = TempDir::new()?; + create_config_toml(codex_home.path(), &server.uri())?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + let thread_id = start_default_thread(&mut mcp).await?; + + let request_id = mcp + .send_review_start_request(ReviewStartParams { + thread_id, + append_to_original_thread: true, + target: ReviewTarget::Custom { + instructions: "\n\n".to_string(), + }, + }) + .await?; + let error: JSONRPCError = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_error_message(RequestId::Integer(request_id)), + ) + .await??; + assert_eq!(error.error.code, INVALID_REQUEST_ERROR_CODE); + assert!( + error + .error + .message + .contains("instructions must not be empty"), + "unexpected message: {}", + error.error.message + ); + + Ok(()) +} + +async fn start_default_thread(mcp: &mut McpProcess) -> Result { + let thread_req = mcp + .send_thread_start_request(ThreadStartParams { + model: Some("mock-model".to_string()), + ..Default::default() + }) + .await?; + let thread_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(thread_req)), + ) + .await??; + let ThreadStartResponse { thread } = to_response::(thread_resp)?; + Ok(thread.id) +} + +fn create_config_toml(codex_home: &std::path::Path, server_uri: &str) -> std::io::Result<()> { + let config_toml = codex_home.join("config.toml"); + std::fs::write( + config_toml, + format!( + r#" +model = "mock-model" +approval_policy = "never" +sandbox_mode = "read-only" + +model_provider = "mock_provider" + +[model_providers.mock_provider] +name = "Mock provider" +base_url = "{server_uri}/v1" +wire_api = "chat" +request_max_retries = 0 +stream_max_retries = 0 +"# + ), + ) +} diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index e308601c55..19ef0a8c23 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -1774,7 +1774,12 @@ async fn spawn_review_thread( text: review_prompt, }]; let tc = Arc::new(review_turn_context); - sess.spawn_task(tc.clone(), input, ReviewTask).await; + sess.spawn_task( + tc.clone(), + input, + ReviewTask::new(review_request.append_to_original_thread), + ) + .await; // Announce entering review mode so UIs can switch modes. sess.send_event(&tc, EventMsg::EnteredReviewMode(review_request)) @@ -2780,7 +2785,8 @@ mod tests { let input = vec![UserInput::Text { text: "start review".to_string(), }]; - sess.spawn_task(Arc::clone(&tc), input, ReviewTask).await; + sess.spawn_task(Arc::clone(&tc), input, ReviewTask::new(true)) + .await; sess.abort_all_tasks(TurnAbortReason::Interrupted).await; diff --git a/codex-rs/core/src/tasks/review.rs b/codex-rs/core/src/tasks/review.rs index e0bb7d4e93..14a95dba5c 100644 --- a/codex-rs/core/src/tasks/review.rs +++ b/codex-rs/core/src/tasks/review.rs @@ -23,8 +23,18 @@ use codex_protocol::user_input::UserInput; use super::SessionTask; use super::SessionTaskContext; -#[derive(Clone, Copy, Default)] -pub(crate) struct ReviewTask; +#[derive(Clone, Copy)] +pub(crate) struct ReviewTask { + append_to_original_thread: bool, +} + +impl ReviewTask { + pub(crate) fn new(append_to_original_thread: bool) -> Self { + Self { + append_to_original_thread, + } + } +} #[async_trait] impl SessionTask for ReviewTask { @@ -52,13 +62,25 @@ impl SessionTask for ReviewTask { None => None, }; if !cancellation_token.is_cancelled() { - exit_review_mode(session.clone_session(), output.clone(), ctx.clone()).await; + exit_review_mode( + session.clone_session(), + output.clone(), + ctx.clone(), + self.append_to_original_thread, + ) + .await; } None } async fn abort(&self, session: Arc, ctx: Arc) { - exit_review_mode(session.clone_session(), None, ctx).await; + exit_review_mode( + session.clone_session(), + None, + ctx, + self.append_to_original_thread, + ) + .await; } } @@ -175,32 +197,35 @@ pub(crate) async fn exit_review_mode( session: Arc, review_output: Option, ctx: Arc, + append_to_original_thread: bool, ) { - let user_message = if let Some(out) = review_output.clone() { - let mut findings_str = String::new(); - let text = out.overall_explanation.trim(); - if !text.is_empty() { - findings_str.push_str(text); - } - if !out.findings.is_empty() { - let block = format_review_findings_block(&out.findings, None); - findings_str.push_str(&format!("\n{block}")); - } - crate::client_common::REVIEW_EXIT_SUCCESS_TMPL.replace("{results}", &findings_str) - } else { - crate::client_common::REVIEW_EXIT_INTERRUPTED_TMPL.to_string() - }; + if append_to_original_thread { + let user_message = if let Some(out) = review_output.clone() { + let mut findings_str = String::new(); + let text = out.overall_explanation.trim(); + if !text.is_empty() { + findings_str.push_str(text); + } + if !out.findings.is_empty() { + let block = format_review_findings_block(&out.findings, None); + findings_str.push_str(&format!("\n{block}")); + } + crate::client_common::REVIEW_EXIT_SUCCESS_TMPL.replace("{results}", &findings_str) + } else { + crate::client_common::REVIEW_EXIT_INTERRUPTED_TMPL.to_string() + }; - session - .record_conversation_items( - &ctx, - &[ResponseItem::Message { - id: None, - role: "user".to_string(), - content: vec![ContentItem::InputText { text: user_message }], - }], - ) - .await; + session + .record_conversation_items( + &ctx, + &[ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::InputText { text: user_message }], + }], + ) + .await; + } session .send_event( ctx.as_ref(), diff --git a/codex-rs/core/tests/suite/codex_delegate.rs b/codex-rs/core/tests/suite/codex_delegate.rs index c6ece7fe56..6339bfa71a 100644 --- a/codex-rs/core/tests/suite/codex_delegate.rs +++ b/codex-rs/core/tests/suite/codex_delegate.rs @@ -70,6 +70,7 @@ async fn codex_delegate_forwards_exec_approval_and_proceeds_on_approval() { review_request: ReviewRequest { prompt: "Please review".to_string(), user_facing_hint: "review".to_string(), + append_to_original_thread: true, }, }) .await @@ -145,6 +146,7 @@ async fn codex_delegate_forwards_patch_approval_and_proceeds_on_decision() { review_request: ReviewRequest { prompt: "Please review".to_string(), user_facing_hint: "review".to_string(), + append_to_original_thread: true, }, }) .await @@ -199,6 +201,7 @@ async fn codex_delegate_ignores_legacy_deltas() { review_request: ReviewRequest { prompt: "Please review".to_string(), user_facing_hint: "review".to_string(), + append_to_original_thread: true, }, }) .await diff --git a/codex-rs/core/tests/suite/review.rs b/codex-rs/core/tests/suite/review.rs index 9c3f812c29..3904f18f8e 100644 --- a/codex-rs/core/tests/suite/review.rs +++ b/codex-rs/core/tests/suite/review.rs @@ -82,6 +82,7 @@ async fn review_op_emits_lifecycle_and_review_output() { review_request: ReviewRequest { prompt: "Please review my changes".to_string(), user_facing_hint: "my changes".to_string(), + append_to_original_thread: true, }, }) .await @@ -178,6 +179,7 @@ async fn review_op_with_plain_text_emits_review_fallback() { review_request: ReviewRequest { prompt: "Plain text review".to_string(), user_facing_hint: "plain text review".to_string(), + append_to_original_thread: true, }, }) .await @@ -236,6 +238,7 @@ async fn review_filters_agent_message_related_events() { review_request: ReviewRequest { prompt: "Filter streaming events".to_string(), user_facing_hint: "Filter streaming events".to_string(), + append_to_original_thread: true, }, }) .await @@ -320,6 +323,7 @@ async fn review_does_not_emit_agent_message_on_structured_output() { review_request: ReviewRequest { prompt: "check structured".to_string(), user_facing_hint: "check structured".to_string(), + append_to_original_thread: true, }, }) .await @@ -373,6 +377,7 @@ async fn review_uses_custom_review_model_from_config() { review_request: ReviewRequest { prompt: "use custom model".to_string(), user_facing_hint: "use custom model".to_string(), + append_to_original_thread: true, }, }) .await @@ -490,6 +495,7 @@ async fn review_input_isolated_from_parent_history() { review_request: ReviewRequest { prompt: review_prompt.clone(), user_facing_hint: review_prompt.clone(), + append_to_original_thread: true, }, }) .await @@ -602,6 +608,7 @@ async fn review_history_does_not_leak_into_parent_session() { review_request: ReviewRequest { prompt: "Start a review".to_string(), user_facing_hint: "Start a review".to_string(), + append_to_original_thread: true, }, }) .await diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index 7f5e5228d8..934b6ad286 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -1173,11 +1173,13 @@ pub struct GitInfo { pub repository_url: Option, } -/// Review request sent to the review session. #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, JsonSchema, TS)] +/// Review request sent to the review session. pub struct ReviewRequest { pub prompt: String, pub user_facing_hint: String, + #[serde(default)] + pub append_to_original_thread: bool, } /// Structured review result produced by a child review session. diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 4e9f496ba3..c17eea92dc 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -2711,6 +2711,7 @@ impl ChatWidget { review_request: ReviewRequest { prompt: "Review the current code changes (staged, unstaged, and untracked files) and provide prioritized findings.".to_string(), user_facing_hint: "current changes".to_string(), + append_to_original_thread: true, }, })); }, @@ -2767,6 +2768,7 @@ impl ChatWidget { "Review the code changes against the base branch '{branch}'. Start by finding the merge diff between the current branch and {branch}'s upstream e.g. (`git merge-base HEAD \"$(git rev-parse --abbrev-ref \"{branch}@{{upstream}}\")\"`), then run `git diff` against that SHA to see what changes we would merge into the {branch} branch. Provide prioritized, actionable findings." ), user_facing_hint: format!("changes against '{branch}'"), + append_to_original_thread: true, }, })); })], @@ -2807,6 +2809,7 @@ impl ChatWidget { review_request: ReviewRequest { prompt, user_facing_hint: hint, + append_to_original_thread: true, }, })); })], @@ -2841,6 +2844,7 @@ impl ChatWidget { review_request: ReviewRequest { prompt: trimmed.clone(), user_facing_hint: trimmed, + append_to_original_thread: true, }, })); }), @@ -3051,6 +3055,7 @@ pub(crate) fn show_review_commit_picker_with_entries( review_request: ReviewRequest { prompt, user_facing_hint: hint, + append_to_original_thread: true, }, })); })], diff --git a/codex-rs/tui/src/chatwidget/tests.rs b/codex-rs/tui/src/chatwidget/tests.rs index 013b5c2313..d6d8f0248d 100644 --- a/codex-rs/tui/src/chatwidget/tests.rs +++ b/codex-rs/tui/src/chatwidget/tests.rs @@ -145,6 +145,7 @@ fn entered_review_mode_uses_request_hint() { msg: EventMsg::EnteredReviewMode(ReviewRequest { prompt: "Review the latest changes".to_string(), user_facing_hint: "feature branch".to_string(), + append_to_original_thread: true, }), }); @@ -164,6 +165,7 @@ fn entered_review_mode_defaults_to_current_changes_banner() { msg: EventMsg::EnteredReviewMode(ReviewRequest { prompt: "Review the current changes".to_string(), user_facing_hint: "current changes".to_string(), + append_to_original_thread: true, }), }); From f7a921039cc9cee04d45a6e87ff480e968bf5bf7 Mon Sep 17 00:00:00 2001 From: Anton Panasenko Date: Tue, 18 Nov 2025 14:01:01 -0800 Subject: [PATCH 05/18] [codex][otel] support mtls configuration (#6228) fix for https://github.com/openai/codex/issues/6153 supports mTLS configuration and includes TLS features in the library build to enable secure HTTPS connections with custom root certificates. grpc: https://docs.rs/tonic/0.13.1/src/tonic/transport/channel/endpoint.rs.html#63 https: https://docs.rs/reqwest/0.12.23/src/reqwest/async_impl/client.rs.html#516 --- codex-rs/Cargo.lock | 4 + codex-rs/core/src/config/types.rs | 14 +++ codex-rs/core/src/otel_init.rs | 18 ++- codex-rs/otel/Cargo.toml | 10 +- codex-rs/otel/src/config.rs | 9 ++ codex-rs/otel/src/otel_provider.rs | 179 ++++++++++++++++++++++++++++- docs/config.md | 17 +++ docs/example-config.md | 13 +++ 8 files changed, 256 insertions(+), 8 deletions(-) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 0ed45ddb23..31f12774a1 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1366,6 +1366,7 @@ dependencies = [ "codex-app-server-protocol", "codex-protocol", "eventsource-stream", + "http", "opentelemetry", "opentelemetry-otlp", "opentelemetry-semantic-conventions", @@ -5174,6 +5175,7 @@ version = "0.23.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2491382039b29b9b11ff08b76ff6c97cf287671dbb74f0be44bda389fffe9bd1" dependencies = [ + "log", "once_cell", "ring", "rustls-pki-types", @@ -6601,8 +6603,10 @@ dependencies = [ "percent-encoding", "pin-project", "prost", + "rustls-native-certs", "socket2 0.5.10", "tokio", + "tokio-rustls", "tokio-stream", "tower", "tower-layer", diff --git a/codex-rs/core/src/config/types.rs b/codex-rs/core/src/config/types.rs index 9c2cd03d13..c502cfeb19 100644 --- a/codex-rs/core/src/config/types.rs +++ b/codex-rs/core/src/config/types.rs @@ -282,6 +282,14 @@ pub enum OtelHttpProtocol { Json, } +#[derive(Deserialize, Debug, Clone, PartialEq, Default)] +#[serde(rename_all = "kebab-case")] +pub struct OtelTlsConfig { + pub ca_certificate: Option, + pub client_certificate: Option, + pub client_private_key: Option, +} + /// Which OTEL exporter to use. #[derive(Deserialize, Debug, Clone, PartialEq)] #[serde(rename_all = "kebab-case")] @@ -289,12 +297,18 @@ pub enum OtelExporterKind { None, OtlpHttp { endpoint: String, + #[serde(default)] headers: HashMap, protocol: OtelHttpProtocol, + #[serde(default)] + tls: Option, }, OtlpGrpc { endpoint: String, + #[serde(default)] headers: HashMap, + #[serde(default)] + tls: Option, }, } diff --git a/codex-rs/core/src/otel_init.rs b/codex-rs/core/src/otel_init.rs index 5931d7caf9..5900c9b4a6 100644 --- a/codex-rs/core/src/otel_init.rs +++ b/codex-rs/core/src/otel_init.rs @@ -5,6 +5,7 @@ use crate::default_client::originator; use codex_otel::config::OtelExporter; use codex_otel::config::OtelHttpProtocol; use codex_otel::config::OtelSettings; +use codex_otel::config::OtelTlsConfig as OtelTlsSettings; use codex_otel::otel_provider::OtelProvider; use std::error::Error; @@ -21,6 +22,7 @@ pub fn build_provider( endpoint, headers, protocol, + tls, } => { let protocol = match protocol { Protocol::Json => OtelHttpProtocol::Json, @@ -34,14 +36,28 @@ pub fn build_provider( .map(|(k, v)| (k.clone(), v.clone())) .collect(), protocol, + tls: tls.as_ref().map(|config| OtelTlsSettings { + ca_certificate: config.ca_certificate.clone(), + client_certificate: config.client_certificate.clone(), + client_private_key: config.client_private_key.clone(), + }), } } - Kind::OtlpGrpc { endpoint, headers } => OtelExporter::OtlpGrpc { + Kind::OtlpGrpc { + endpoint, + headers, + tls, + } => OtelExporter::OtlpGrpc { endpoint: endpoint.clone(), headers: headers .iter() .map(|(k, v)| (k.clone(), v.clone())) .collect(), + tls: tls.as_ref().map(|config| OtelTlsSettings { + ca_certificate: config.ca_certificate.clone(), + client_certificate: config.client_certificate.clone(), + client_private_key: config.client_private_key.clone(), + }), }, }; diff --git a/codex-rs/otel/Cargo.toml b/codex-rs/otel/Cargo.toml index ea518c2e4f..2ab170c917 100644 --- a/codex-rs/otel/Cargo.toml +++ b/codex-rs/otel/Cargo.toml @@ -27,18 +27,26 @@ opentelemetry-otlp = { workspace = true, features = [ "grpc-tonic", "http-proto", "http-json", + "logs", "reqwest", "reqwest-rustls", + "tls", + "tls-roots", ], optional = true } opentelemetry-semantic-conventions = { workspace = true } opentelemetry_sdk = { workspace = true, features = [ "logs", "rt-tokio", ], optional = true } +http = { workspace = true } reqwest = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } strum_macros = { workspace = true } tokio = { workspace = true } -tonic = { workspace = true, optional = true } +tonic = { workspace = true, optional = true, features = [ + "transport", + "tls-native-roots", + "tls-ring", +] } tracing = { workspace = true } diff --git a/codex-rs/otel/src/config.rs b/codex-rs/otel/src/config.rs index 77063ed09d..b6336b3a5c 100644 --- a/codex-rs/otel/src/config.rs +++ b/codex-rs/otel/src/config.rs @@ -18,16 +18,25 @@ pub enum OtelHttpProtocol { Json, } +#[derive(Clone, Debug, Default)] +pub struct OtelTlsConfig { + pub ca_certificate: Option, + pub client_certificate: Option, + pub client_private_key: Option, +} + #[derive(Clone, Debug)] pub enum OtelExporter { None, OtlpGrpc { endpoint: String, headers: HashMap, + tls: Option, }, OtlpHttp { endpoint: String, headers: HashMap, protocol: OtelHttpProtocol, + tls: Option, }, } diff --git a/codex-rs/otel/src/otel_provider.rs b/codex-rs/otel/src/otel_provider.rs index 222322a2ea..8be2431ea9 100644 --- a/codex-rs/otel/src/otel_provider.rs +++ b/codex-rs/otel/src/otel_provider.rs @@ -1,8 +1,13 @@ use crate::config::OtelExporter; use crate::config::OtelHttpProtocol; use crate::config::OtelSettings; +use crate::config::OtelTlsConfig; +use http::Uri; use opentelemetry::KeyValue; use opentelemetry_otlp::LogExporter; +use opentelemetry_otlp::OTEL_EXPORTER_OTLP_LOGS_TIMEOUT; +use opentelemetry_otlp::OTEL_EXPORTER_OTLP_TIMEOUT; +use opentelemetry_otlp::OTEL_EXPORTER_OTLP_TIMEOUT_DEFAULT; use opentelemetry_otlp::Protocol; use opentelemetry_otlp::WithExportConfig; use opentelemetry_otlp::WithHttpConfig; @@ -10,11 +15,23 @@ use opentelemetry_otlp::WithTonicConfig; use opentelemetry_sdk::Resource; use opentelemetry_sdk::logs::SdkLoggerProvider; use opentelemetry_semantic_conventions as semconv; +use reqwest::Certificate as ReqwestCertificate; +use reqwest::Identity as ReqwestIdentity; use reqwest::header::HeaderMap; use reqwest::header::HeaderName; use reqwest::header::HeaderValue; +use std::env; use std::error::Error; +use std::fs; +use std::io::ErrorKind; +use std::io::{self}; +use std::path::Path; +use std::path::PathBuf; +use std::time::Duration; use tonic::metadata::MetadataMap; +use tonic::transport::Certificate as TonicCertificate; +use tonic::transport::ClientTlsConfig; +use tonic::transport::Identity as TonicIdentity; use tracing::debug; const ENV_ATTRIBUTE: &str = "env"; @@ -47,8 +64,12 @@ impl OtelProvider { debug!("No exporter enabled in OTLP settings."); return Ok(None); } - OtelExporter::OtlpGrpc { endpoint, headers } => { - debug!("Using OTLP Grpc exporter: {}", endpoint); + OtelExporter::OtlpGrpc { + endpoint, + headers, + tls, + } => { + debug!("Using OTLP Grpc exporter: {endpoint}"); let mut header_map = HeaderMap::new(); for (key, value) in headers { @@ -59,10 +80,25 @@ impl OtelProvider { } } + let base_tls_config = ClientTlsConfig::new() + .with_enabled_roots() + .assume_http2(true); + + let tls_config = match tls.as_ref() { + Some(tls) => build_grpc_tls_config( + endpoint, + base_tls_config, + tls, + settings.codex_home.as_path(), + )?, + None => base_tls_config, + }; + let exporter = LogExporter::builder() .with_tonic() .with_endpoint(endpoint) .with_metadata(MetadataMap::from_headers(header_map)) + .with_tls_config(tls_config) .build()?; builder = builder.with_batch_exporter(exporter); @@ -71,20 +107,27 @@ impl OtelProvider { endpoint, headers, protocol, + tls, } => { - debug!("Using OTLP Http exporter: {}", endpoint); + debug!("Using OTLP Http exporter: {endpoint}"); let protocol = match protocol { OtelHttpProtocol::Binary => Protocol::HttpBinary, OtelHttpProtocol::Json => Protocol::HttpJson, }; - let exporter = LogExporter::builder() + let mut exporter_builder = LogExporter::builder() .with_http() .with_endpoint(endpoint) .with_protocol(protocol) - .with_headers(headers.clone()) - .build()?; + .with_headers(headers.clone()); + + if let Some(tls) = tls.as_ref() { + let client = build_http_client(tls, settings.codex_home.as_path())?; + exporter_builder = exporter_builder.with_http_client(client); + } + + let exporter = exporter_builder.build()?; builder = builder.with_batch_exporter(exporter); } @@ -101,3 +144,127 @@ impl Drop for OtelProvider { let _ = self.logger.shutdown(); } } + +fn build_grpc_tls_config( + endpoint: &str, + tls_config: ClientTlsConfig, + tls: &OtelTlsConfig, + codex_home: &Path, +) -> Result> { + let uri: Uri = endpoint.parse()?; + let host = uri.host().ok_or_else(|| { + config_error(format!( + "OTLP gRPC endpoint {endpoint} does not include a host" + )) + })?; + + let mut config = tls_config.domain_name(host.to_owned()); + + if let Some(path) = tls.ca_certificate.as_ref() { + let (pem, _) = read_bytes(codex_home, path)?; + config = config.ca_certificate(TonicCertificate::from_pem(pem)); + } + + match (&tls.client_certificate, &tls.client_private_key) { + (Some(cert_path), Some(key_path)) => { + let (cert_pem, _) = read_bytes(codex_home, cert_path)?; + let (key_pem, _) = read_bytes(codex_home, key_path)?; + config = config.identity(TonicIdentity::from_pem(cert_pem, key_pem)); + } + (Some(_), None) | (None, Some(_)) => { + return Err(config_error( + "client_certificate and client_private_key must both be provided for mTLS", + )); + } + (None, None) => {} + } + + Ok(config) +} + +fn build_http_client( + tls: &OtelTlsConfig, + codex_home: &Path, +) -> Result> { + let mut builder = + reqwest::Client::builder().timeout(resolve_otlp_timeout(OTEL_EXPORTER_OTLP_LOGS_TIMEOUT)); + + if let Some(path) = tls.ca_certificate.as_ref() { + let (pem, location) = read_bytes(codex_home, path)?; + let certificate = ReqwestCertificate::from_pem(pem.as_slice()).map_err(|error| { + config_error(format!( + "failed to parse certificate {}: {error}", + location.display() + )) + })?; + builder = builder.add_root_certificate(certificate); + } + + match (&tls.client_certificate, &tls.client_private_key) { + (Some(cert_path), Some(key_path)) => { + let (mut cert_pem, cert_location) = read_bytes(codex_home, cert_path)?; + let (key_pem, key_location) = read_bytes(codex_home, key_path)?; + cert_pem.extend_from_slice(key_pem.as_slice()); + let identity = ReqwestIdentity::from_pem(cert_pem.as_slice()).map_err(|error| { + config_error(format!( + "failed to parse client identity using {} and {}: {error}", + cert_location.display(), + key_location.display() + )) + })?; + builder = builder.identity(identity); + } + (Some(_), None) | (None, Some(_)) => { + return Err(config_error( + "client_certificate and client_private_key must both be provided for mTLS", + )); + } + (None, None) => {} + } + + builder + .build() + .map_err(|error| Box::new(error) as Box) +} + +fn resolve_otlp_timeout(signal_var: &str) -> Duration { + if let Some(timeout) = read_timeout_env(signal_var) { + return timeout; + } + if let Some(timeout) = read_timeout_env(OTEL_EXPORTER_OTLP_TIMEOUT) { + return timeout; + } + OTEL_EXPORTER_OTLP_TIMEOUT_DEFAULT +} + +fn read_timeout_env(var: &str) -> Option { + let value = env::var(var).ok()?; + let parsed = value.parse::().ok()?; + if parsed < 0 { + return None; + } + Some(Duration::from_millis(parsed as u64)) +} + +fn read_bytes(base: &Path, provided: &PathBuf) -> Result<(Vec, PathBuf), Box> { + let resolved = resolve_config_path(base, provided); + match fs::read(&resolved) { + Ok(bytes) => Ok((bytes, resolved)), + Err(error) => Err(Box::new(io::Error::new( + error.kind(), + format!("failed to read {}: {error}", resolved.display()), + ))), + } +} + +fn resolve_config_path(base: &Path, provided: &PathBuf) -> PathBuf { + if provided.is_absolute() { + provided.clone() + } else { + base.join(provided) + } +} + +fn config_error(message: impl Into) -> Box { + Box::new(io::Error::new(ErrorKind::InvalidData, message.into())) +} diff --git a/docs/config.md b/docs/config.md index ddfe2ff7c4..8e0fa2dac8 100644 --- a/docs/config.md +++ b/docs/config.md @@ -651,6 +651,23 @@ Set `otel.exporter` to control where events go: }} ``` +Both OTLP exporters accept an optional `tls` block so you can trust a custom CA +or enable mutual TLS. Relative paths are resolved against `~/.codex/`: + +```toml +[otel] +exporter = { otlp-http = { + endpoint = "https://otel.example.com/v1/logs", + protocol = "binary", + headers = { "x-otlp-api-key" = "${OTLP_TOKEN}" }, + tls = { + ca-certificate = "certs/otel-ca.pem", + client-certificate = "/etc/codex/certs/client.pem", + client-private-key = "/etc/codex/certs/client-key.pem", + } +}} +``` + If the exporter is `none` nothing is written anywhere; otherwise you must run or point to your own collector. All exporters run on a background batch worker that is flushed on shutdown. diff --git a/docs/example-config.md b/docs/example-config.md index b2da427314..33e9bf4f15 100644 --- a/docs/example-config.md +++ b/docs/example-config.md @@ -369,4 +369,17 @@ exporter = "none" # endpoint = "https://otel.example.com:4317", # headers = { "x-otlp-meta" = "abc123" } # }} + +# Example OTLP exporter with mutual TLS +# [otel] +# exporter = { otlp-http = { +# endpoint = "https://otel.example.com/v1/logs", +# protocol = "binary", +# headers = { "x-otlp-api-key" = "${OTLP_TOKEN}" }, +# tls = { +# ca-certificate = "certs/otel-ca.pem", +# client-certificate = "/etc/codex/certs/client.pem", +# client-private-key = "/etc/codex/certs/client-key.pem", +# } +# }} ``` From 0bf857bc912aa3a62d0e11e34f94b295994b96aa Mon Sep 17 00:00:00 2001 From: simister Date: Tue, 18 Nov 2025 17:06:13 -0500 Subject: [PATCH 06/18] Fix typo in config.md for MCP server (#6845) # External (non-OpenAI) Pull Request Requirements Before opening this Pull Request, please read the dedicated "Contributing" markdown file or your PR may be closed: https://github.com/openai/codex/blob/main/docs/contributing.md If your PR conforms to our contribution guidelines, replace this text with a detailed and high quality description of your changes. Include a link to a bug report or enhancement request. --- docs/config.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/config.md b/docs/config.md index 8e0fa2dac8..a333d365c5 100644 --- a/docs/config.md +++ b/docs/config.md @@ -442,7 +442,7 @@ You can configure Codex to use [MCP servers](https://modelcontextprotocol.io/abo command = "npx" # Optional args = ["-y", "mcp-server"] -# Optional: propagate additional env vars to the MVP server. +# Optional: propagate additional env vars to the MCP server. # A default whitelist of env vars will be propagated to the MCP server. # https://github.com/openai/codex/blob/main/codex-rs/rmcp-client/src/utils.rs#L82 env = { "API_KEY" = "value" } From c56d0c159b7a9218d8721b86c382e82375e08e63 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Tue, 18 Nov 2025 22:18:10 +0000 Subject: [PATCH 07/18] fix: local compaction (#6844) --- codex-rs/core/src/codex.rs | 11 +++++++++-- codex-rs/core/src/compact_remote.rs | 14 +------------- codex-rs/core/src/tasks/compact.rs | 2 +- codex-rs/core/tests/suite/compact.rs | 29 +++++++++++++--------------- 4 files changed, 24 insertions(+), 32 deletions(-) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 19ef0a8c23..5f45fc25b6 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -1431,6 +1431,7 @@ mod handlers { use codex_protocol::protocol::ReviewRequest; use codex_protocol::protocol::TurnAbortReason; + use codex_protocol::user_input::UserInput; use std::sync::Arc; use tracing::info; use tracing::warn; @@ -1639,8 +1640,14 @@ mod handlers { .new_turn_with_sub_id(sub_id, SessionSettingsUpdate::default()) .await; - sess.spawn_task(Arc::clone(&turn_context), vec![], CompactTask) - .await; + sess.spawn_task( + Arc::clone(&turn_context), + vec![UserInput::Text { + text: turn_context.compact_prompt().to_string(), + }], + CompactTask, + ) + .await; } pub async fn shutdown(sess: &Arc, sub_id: String) -> bool { diff --git a/codex-rs/core/src/compact_remote.rs b/codex-rs/core/src/compact_remote.rs index 1726aad6f9..75b81d408c 100644 --- a/codex-rs/core/src/compact_remote.rs +++ b/codex-rs/core/src/compact_remote.rs @@ -10,21 +10,18 @@ use crate::protocol::ErrorEvent; use crate::protocol::EventMsg; use crate::protocol::RolloutItem; use crate::protocol::TaskStartedEvent; -use codex_protocol::models::ResponseInputItem; use codex_protocol::models::ResponseItem; -use codex_protocol::user_input::UserInput; pub(crate) async fn run_remote_compact_task( sess: Arc, turn_context: Arc, - input: Vec, ) -> Option { let start_event = EventMsg::TaskStarted(TaskStartedEvent { model_context_window: turn_context.client.get_model_context_window(), }); sess.send_event(&turn_context, start_event).await; - match run_remote_compact_task_inner(&sess, &turn_context, input).await { + match run_remote_compact_task_inner(&sess, &turn_context).await { Ok(()) => { let event = EventMsg::AgentMessage(AgentMessageEvent { message: "Compact task completed".to_string(), @@ -45,17 +42,8 @@ pub(crate) async fn run_remote_compact_task( async fn run_remote_compact_task_inner( sess: &Arc, turn_context: &Arc, - input: Vec, ) -> CodexResult<()> { let mut history = sess.clone_history().await; - if !input.is_empty() { - let initial_input_for_turn: ResponseInputItem = ResponseInputItem::from(input); - history.record_items( - &[initial_input_for_turn.into()], - turn_context.truncation_policy, - ); - } - let prompt = Prompt { input: history.get_history_for_prompt(), tools: vec![], diff --git a/codex-rs/core/src/tasks/compact.rs b/codex-rs/core/src/tasks/compact.rs index e2e5625b5d..4f161267a8 100644 --- a/codex-rs/core/src/tasks/compact.rs +++ b/codex-rs/core/src/tasks/compact.rs @@ -34,7 +34,7 @@ impl SessionTask for CompactTask { .is_some_and(|auth| auth.mode == AuthMode::ChatGPT) && session.enabled(Feature::RemoteCompaction).await { - crate::compact_remote::run_remote_compact_task(session, ctx, input).await + crate::compact_remote::run_remote_compact_task(session, ctx).await } else { crate::compact::run_compact_task(session, ctx, input).await } diff --git a/codex-rs/core/tests/suite/compact.rs b/codex-rs/core/tests/suite/compact.rs index e8b3813c05..5359f3b1ad 100644 --- a/codex-rs/core/tests/suite/compact.rs +++ b/codex-rs/core/tests/suite/compact.rs @@ -191,22 +191,19 @@ async fn summarize_context_three_requests_and_instructions() { let body2_str = body2.to_string(); let input2 = body2.get("input").and_then(|v| v.as_array()).unwrap(); let has_compact_prompt = body_contains_text(&body2_str, SUMMARIZATION_PROMPT); - if has_compact_prompt { - // The last item is the user message created from the injected input. - let last2 = input2.last().unwrap(); - assert_eq!(last2.get("type").unwrap().as_str().unwrap(), "message"); - assert_eq!(last2.get("role").unwrap().as_str().unwrap(), "user"); - let text2 = last2["content"][0]["text"].as_str().unwrap(); - assert_eq!( - text2, SUMMARIZATION_PROMPT, - "expected summarize trigger, got `{text2}`" - ); - } else { - assert!( - !has_compact_prompt, - "compaction request should not unexpectedly include the summarize trigger" - ); - } + assert!( + has_compact_prompt, + "compaction request should include the summarize trigger" + ); + // The last item is the user message created from the injected input. + let last2 = input2.last().unwrap(); + assert_eq!(last2.get("type").unwrap().as_str().unwrap(), "message"); + assert_eq!(last2.get("role").unwrap().as_str().unwrap(), "user"); + let text2 = last2["content"][0]["text"].as_str().unwrap(); + assert_eq!( + text2, SUMMARIZATION_PROMPT, + "expected summarize trigger, got `{text2}`" + ); // Third request must contain the refreshed instructions, compacted user history, and new user message. let input3 = body3.get("input").and_then(|v| v.as_array()).unwrap(); From ab30453dee68f5b4571843eca0d3d44cdb59e3dd Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Tue, 18 Nov 2025 17:24:32 -0600 Subject: [PATCH 08/18] Improved runtime of `generated_ts_has_no_optional_nullable_fields` test (#6851) The `generated_ts_has_no_optional_nullable_fields` test was occasionally failing on slow CI nodes because of a timeout. This change reduces the work done by the test. It adds some "options" for the `generate_ts` function so it can skip work that's not needed for the test. --- codex-rs/app-server-protocol/src/export.rs | 57 +++++++++++++++++++--- 1 file changed, 49 insertions(+), 8 deletions(-) diff --git a/codex-rs/app-server-protocol/src/export.rs b/codex-rs/app-server-protocol/src/export.rs index 68d6249314..4de66dfb37 100644 --- a/codex-rs/app-server-protocol/src/export.rs +++ b/codex-rs/app-server-protocol/src/export.rs @@ -61,7 +61,32 @@ pub fn generate_types(out_dir: &Path, prettier: Option<&Path>) -> Result<()> { Ok(()) } +#[derive(Clone, Copy, Debug)] +pub struct GenerateTsOptions { + pub generate_indices: bool, + pub ensure_headers: bool, + pub run_prettier: bool, +} + +impl Default for GenerateTsOptions { + fn default() -> Self { + Self { + generate_indices: true, + ensure_headers: true, + run_prettier: true, + } + } +} + pub fn generate_ts(out_dir: &Path, prettier: Option<&Path>) -> Result<()> { + generate_ts_with_options(out_dir, prettier, GenerateTsOptions::default()) +} + +pub fn generate_ts_with_options( + out_dir: &Path, + prettier: Option<&Path>, + options: GenerateTsOptions, +) -> Result<()> { let v2_out_dir = out_dir.join("v2"); ensure_dir(out_dir)?; ensure_dir(&v2_out_dir)?; @@ -74,17 +99,28 @@ pub fn generate_ts(out_dir: &Path, prettier: Option<&Path>) -> Result<()> { export_server_responses(out_dir)?; ServerNotification::export_all_to(out_dir)?; - generate_index_ts(out_dir)?; - generate_index_ts(&v2_out_dir)?; + if options.generate_indices { + generate_index_ts(out_dir)?; + generate_index_ts(&v2_out_dir)?; + } // Ensure our header is present on all TS files (root + subdirs like v2/). - let ts_files = ts_files_in_recursive(out_dir)?; - for file in &ts_files { - prepend_header_if_missing(file)?; + let mut ts_files = Vec::new(); + let should_collect_ts_files = + options.ensure_headers || (options.run_prettier && prettier.is_some()); + if should_collect_ts_files { + ts_files = ts_files_in_recursive(out_dir)?; + } + + if options.ensure_headers { + for file in &ts_files { + prepend_header_if_missing(file)?; + } } // Optionally run Prettier on all generated TS files. - if let Some(prettier_bin) = prettier + if options.run_prettier + && let Some(prettier_bin) = prettier && !ts_files.is_empty() { let status = Command::new(prettier_bin) @@ -708,7 +744,6 @@ mod tests { use uuid::Uuid; #[test] - #[ignore = "timing out"] fn generated_ts_has_no_optional_nullable_fields() -> Result<()> { // Assert that there are no types of the form "?: T | null" in the generated TS files. let output_dir = std::env::temp_dir().join(format!("codex_ts_types_{}", Uuid::now_v7())); @@ -724,7 +759,13 @@ mod tests { let _guard = TempDirGuard(output_dir.clone()); - generate_ts(&output_dir, None)?; + // Avoid doing more work than necessary to keep the test from timing out. + let options = GenerateTsOptions { + generate_indices: false, + ensure_headers: false, + run_prettier: false, + }; + generate_ts_with_options(&output_dir, None, options)?; let mut undefined_offenders = Vec::new(); let mut optional_nullable_offenders = BTreeSet::new(); From b3a824ae3cf7fbbc11873a46281a9a48836f9eb6 Mon Sep 17 00:00:00 2001 From: Owen Lin Date: Tue, 18 Nov 2025 15:25:02 -0800 Subject: [PATCH 09/18] [app-server-test-client] feat: auto approve command (#6852) --- codex-rs/app-server-test-client/src/main.rs | 90 ++++++++++++++++++++- 1 file changed, 86 insertions(+), 4 deletions(-) diff --git a/codex-rs/app-server-test-client/src/main.rs b/codex-rs/app-server-test-client/src/main.rs index a243937b28..8513ec0321 100644 --- a/codex-rs/app-server-test-client/src/main.rs +++ b/codex-rs/app-server-test-client/src/main.rs @@ -17,15 +17,20 @@ use clap::Parser; use clap::Subcommand; use codex_app_server_protocol::AddConversationListenerParams; use codex_app_server_protocol::AddConversationSubscriptionResponse; +use codex_app_server_protocol::ApprovalDecision; use codex_app_server_protocol::AskForApproval; use codex_app_server_protocol::ClientInfo; use codex_app_server_protocol::ClientRequest; +use codex_app_server_protocol::CommandExecutionRequestAcceptSettings; +use codex_app_server_protocol::CommandExecutionRequestApprovalParams; +use codex_app_server_protocol::CommandExecutionRequestApprovalResponse; use codex_app_server_protocol::GetAccountRateLimitsResponse; use codex_app_server_protocol::InitializeParams; use codex_app_server_protocol::InitializeResponse; use codex_app_server_protocol::InputItem; use codex_app_server_protocol::JSONRPCMessage; use codex_app_server_protocol::JSONRPCNotification; +use codex_app_server_protocol::JSONRPCRequest; use codex_app_server_protocol::JSONRPCResponse; use codex_app_server_protocol::LoginChatGptCompleteNotification; use codex_app_server_protocol::LoginChatGptResponse; @@ -36,6 +41,7 @@ use codex_app_server_protocol::SandboxPolicy; use codex_app_server_protocol::SendUserMessageParams; use codex_app_server_protocol::SendUserMessageResponse; use codex_app_server_protocol::ServerNotification; +use codex_app_server_protocol::ServerRequest; use codex_app_server_protocol::ThreadStartParams; use codex_app_server_protocol::ThreadStartResponse; use codex_app_server_protocol::TurnStartParams; @@ -44,6 +50,7 @@ use codex_app_server_protocol::UserInput as V2UserInput; use codex_protocol::ConversationId; use codex_protocol::protocol::Event; use codex_protocol::protocol::EventMsg; +use serde::Serialize; use serde::de::DeserializeOwned; use serde_json::Value; use uuid::Uuid; @@ -603,8 +610,8 @@ impl CodexClient { JSONRPCMessage::Notification(notification) => { self.pending_notifications.push_back(notification); } - JSONRPCMessage::Request(_) => { - bail!("unexpected request from codex app-server"); + JSONRPCMessage::Request(request) => { + self.handle_server_request(request)?; } } } @@ -624,8 +631,8 @@ impl CodexClient { // No outstanding requests, so ignore stray responses/errors for now. continue; } - JSONRPCMessage::Request(_) => { - bail!("unexpected request from codex app-server"); + JSONRPCMessage::Request(request) => { + self.handle_server_request(request)?; } } } @@ -661,6 +668,81 @@ impl CodexClient { fn request_id(&self) -> RequestId { RequestId::String(Uuid::new_v4().to_string()) } + + fn handle_server_request(&mut self, request: JSONRPCRequest) -> Result<()> { + let server_request = ServerRequest::try_from(request) + .context("failed to deserialize ServerRequest from JSONRPCRequest")?; + + match server_request { + ServerRequest::CommandExecutionRequestApproval { request_id, params } => { + self.handle_command_execution_request_approval(request_id, params)?; + } + other => { + bail!("received unsupported server request: {other:?}"); + } + } + + Ok(()) + } + + fn handle_command_execution_request_approval( + &mut self, + request_id: RequestId, + params: CommandExecutionRequestApprovalParams, + ) -> Result<()> { + let CommandExecutionRequestApprovalParams { + thread_id, + turn_id, + item_id, + reason, + risk, + } = params; + + println!( + "\n< commandExecution approval requested for thread {thread_id}, turn {turn_id}, item {item_id}" + ); + if let Some(reason) = reason.as_deref() { + println!("< reason: {reason}"); + } + if let Some(risk) = risk.as_ref() { + println!("< risk assessment: {risk:?}"); + } + + let response = CommandExecutionRequestApprovalResponse { + decision: ApprovalDecision::Accept, + accept_settings: Some(CommandExecutionRequestAcceptSettings { for_session: false }), + }; + self.send_server_request_response(request_id, &response)?; + println!("< approved commandExecution request for item {item_id}"); + Ok(()) + } + + fn send_server_request_response(&mut self, request_id: RequestId, response: &T) -> Result<()> + where + T: Serialize, + { + let message = JSONRPCMessage::Response(JSONRPCResponse { + id: request_id, + result: serde_json::to_value(response)?, + }); + self.write_jsonrpc_message(message) + } + + fn write_jsonrpc_message(&mut self, message: JSONRPCMessage) -> Result<()> { + let payload = serde_json::to_string(&message)?; + let pretty = serde_json::to_string_pretty(&message)?; + print_multiline_with_prefix("> ", &pretty); + + if let Some(stdin) = self.stdin.as_mut() { + writeln!(stdin, "{payload}")?; + stdin + .flush() + .context("failed to flush response to codex app-server")?; + return Ok(()); + } + + bail!("codex app-server stdin closed") + } } fn print_multiline_with_prefix(prefix: &str, payload: &str) { From 9275e933648dc3e47fc0a84aa707fc734da4d2aa Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Tue, 18 Nov 2025 17:27:45 -0600 Subject: [PATCH 10/18] Fix tests so they don't emit an extraneous `config.toml` in the source tree (#6853) This PR fixes the `release_event_does_not_change_selection` test so it doesn't cause an extra `config.toml` to be emitted in the sources when running the tests locally. Prior to this fix, I needed to delete this file every time I ran the tests to prevent it from showing up as an uncommitted source file. --- codex-rs/tui/src/onboarding/trust_directory.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/codex-rs/tui/src/onboarding/trust_directory.rs b/codex-rs/tui/src/onboarding/trust_directory.rs index fd24b8e888..cf5d6e9a1a 100644 --- a/codex-rs/tui/src/onboarding/trust_directory.rs +++ b/codex-rs/tui/src/onboarding/trust_directory.rs @@ -189,13 +189,14 @@ mod tests { use crossterm::event::KeyModifiers; use pretty_assertions::assert_eq; use ratatui::Terminal; - use std::path::PathBuf; + use tempfile::TempDir; #[test] fn release_event_does_not_change_selection() { + let codex_home = TempDir::new().expect("temp home"); let mut widget = TrustDirectoryWidget { - codex_home: PathBuf::from("."), + codex_home: codex_home.path().to_path_buf(), cwd: PathBuf::from("."), is_git_repo: false, selection: None, @@ -217,8 +218,9 @@ mod tests { #[test] fn renders_snapshot_for_git_repo() { + let codex_home = TempDir::new().expect("temp home"); let widget = TrustDirectoryWidget { - codex_home: PathBuf::from("."), + codex_home: codex_home.path().to_path_buf(), cwd: PathBuf::from("/workspace/project"), is_git_repo: true, selection: None, From c1391b9f94d7cbeb4cd1cfbecf2e82a3b0904ca1 Mon Sep 17 00:00:00 2001 From: Jeremy Rose <172423086+nornagon-openai@users.noreply.github.com> Date: Tue, 18 Nov 2025 16:20:19 -0800 Subject: [PATCH 11/18] exec-server (#6630) --- codex-rs/Cargo.lock | 20 + codex-rs/Cargo.toml | 2 + codex-rs/exec-server/Cargo.toml | 47 ++ codex-rs/exec-server/src/main.rs | 11 + codex-rs/exec-server/src/posix.rs | 164 ++++++ .../exec-server/src/posix/escalate_client.rs | 102 ++++ .../src/posix/escalate_protocol.rs | 49 ++ .../exec-server/src/posix/escalate_server.rs | 274 ++++++++++ codex-rs/exec-server/src/posix/mcp.rs | 154 ++++++ codex-rs/exec-server/src/posix/socket.rs | 486 ++++++++++++++++++ 10 files changed, 1309 insertions(+) create mode 100644 codex-rs/exec-server/Cargo.toml create mode 100644 codex-rs/exec-server/src/main.rs create mode 100644 codex-rs/exec-server/src/posix.rs create mode 100644 codex-rs/exec-server/src/posix/escalate_client.rs create mode 100644 codex-rs/exec-server/src/posix/escalate_protocol.rs create mode 100644 codex-rs/exec-server/src/posix/escalate_server.rs create mode 100644 codex-rs/exec-server/src/posix/mcp.rs create mode 100644 codex-rs/exec-server/src/posix/socket.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 31f12774a1..a7ba2a4c68 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1182,6 +1182,26 @@ dependencies = [ "wiremock", ] +[[package]] +name = "codex-exec-server" +version = "0.0.0" +dependencies = [ + "anyhow", + "clap", + "codex-core", + "libc", + "path-absolutize", + "pretty_assertions", + "rmcp", + "serde", + "serde_json", + "socket2 0.6.0", + "tempfile", + "tokio", + "tracing", + "tracing-subscriber", +] + [[package]] name = "codex-execpolicy" version = "0.0.0" diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index b19bf76608..2e88aba713 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -16,6 +16,7 @@ members = [ "common", "core", "exec", + "exec-server", "execpolicy", "execpolicy2", "keyring-store", @@ -176,6 +177,7 @@ sha1 = "0.10.6" sha2 = "0.10" shlex = "1.3.0" similar = "2.7.0" +socket2 = "0.6.0" starlark = "0.13.0" strum = "0.27.2" strum_macros = "0.27.2" diff --git a/codex-rs/exec-server/Cargo.toml b/codex-rs/exec-server/Cargo.toml new file mode 100644 index 0000000000..24ee460f50 --- /dev/null +++ b/codex-rs/exec-server/Cargo.toml @@ -0,0 +1,47 @@ +[package] +edition = "2024" +name = "codex-exec-server" +version = { workspace = true } + +[[bin]] +name = "codex-exec-server" +path = "src/main.rs" + +[lints] +workspace = true + +[dependencies] +anyhow = { workspace = true } +clap = { workspace = true, features = ["derive"] } +codex-core = { workspace = true } +libc = { workspace = true } +path-absolutize = { workspace = true } +rmcp = { workspace = true, default-features = false, features = [ + "auth", + "elicitation", + "base64", + "client", + "macros", + "schemars", + "server", + "transport-child-process", + "transport-streamable-http-client-reqwest", + "transport-streamable-http-server", + "transport-io", +] } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +socket2 = { workspace = true } +tokio = { workspace = true, features = [ + "io-std", + "macros", + "process", + "rt-multi-thread", + "signal", +] } +tracing = { workspace = true } +tracing-subscriber = { workspace = true, features = ["env-filter", "fmt"] } + +[dev-dependencies] +pretty_assertions = { workspace = true } +tempfile = { workspace = true } diff --git a/codex-rs/exec-server/src/main.rs b/codex-rs/exec-server/src/main.rs new file mode 100644 index 0000000000..23a18b2525 --- /dev/null +++ b/codex-rs/exec-server/src/main.rs @@ -0,0 +1,11 @@ +#[cfg(target_os = "windows")] +fn main() { + eprintln!("codex-exec-server is not implemented on Windows targets"); + std::process::exit(1); +} + +#[cfg(not(target_os = "windows"))] +mod posix; + +#[cfg(not(target_os = "windows"))] +pub use posix::main; diff --git a/codex-rs/exec-server/src/posix.rs b/codex-rs/exec-server/src/posix.rs new file mode 100644 index 0000000000..179fc7a6fc --- /dev/null +++ b/codex-rs/exec-server/src/posix.rs @@ -0,0 +1,164 @@ +//! This is an MCP that implements an alternative `shell` tool with fine-grained privilege +//! escalation based on a per-exec() policy. +//! +//! We spawn Bash process inside a sandbox. The Bash we spawn is patched to allow us to intercept +//! every exec() call it makes by invoking a wrapper program and passing in the arguments it would +//! have passed to exec(). The Bash process (and its descendants) inherit a communication socket +//! from us, and we give its fd number in the CODEX_ESCALATE_SOCKET environment variable. +//! +//! When we intercept an exec() call, we send a message over the socket back to the main +//! MCP process. The MCP process can then decide whether to allow the exec() call to proceed +//! or to escalate privileges and run the requested command with elevated permissions. In the +//! latter case, we send a message back to the child requesting that it forward its open FDs to us. +//! We then execute the requested command on its behalf, patching in the forwarded FDs. +//! +//! +//! ### The privilege escalation flow +//! +//! Child MCP Bash Escalate Helper +//! | +//! o----->o +//! | | +//! | o--(exec)-->o +//! | | | +//! |o<-(EscalateReq)--o +//! || | | +//! |o--(Escalate)---->o +//! || | | +//! |o<---------(fds)--o +//! || | | +//! o<-----o | | +//! | || | | +//! x----->o | | +//! || | | +//! |x--(exit code)--->o +//! | | | +//! | o<--(exit)--x +//! | | +//! o<-----x +//! +//! ### The non-escalation flow +//! +//! MCP Bash Escalate Helper Child +//! | +//! o----->o +//! | | +//! | o--(exec)-->o +//! | | | +//! |o<-(EscalateReq)--o +//! || | | +//! |o-(Run)---------->o +//! | | | +//! | | x--(exec)-->o +//! | | | +//! | o<--------------(exit)--x +//! | | +//! o<-----x +//! +use std::path::Path; + +use clap::Parser; +use clap::Subcommand; +use tracing_subscriber::EnvFilter; +use tracing_subscriber::{self}; + +use crate::posix::escalate_protocol::EscalateAction; +use crate::posix::escalate_server::EscalateServer; + +mod escalate_client; +mod escalate_protocol; +mod escalate_server; +mod mcp; +mod socket; + +fn dummy_exec_policy(file: &Path, argv: &[String], _workdir: &Path) -> EscalateAction { + // TODO: execpolicy + if file == Path::new("/opt/homebrew/bin/gh") + && let [_, arg1, arg2, ..] = argv + && arg1 == "issue" + && arg2 == "list" + { + return EscalateAction::Escalate; + } + EscalateAction::Run +} + +#[derive(Parser)] +#[command(version)] +pub struct Cli { + #[command(subcommand)] + subcommand: Option, +} + +#[derive(Subcommand)] +enum Commands { + Escalate(EscalateArgs), + ShellExec(ShellExecArgs), +} + +/// Invoked from within the sandbox to (potentially) escalate permissions. +#[derive(Parser, Debug)] +struct EscalateArgs { + file: String, + + #[arg(trailing_var_arg = true)] + argv: Vec, +} + +impl EscalateArgs { + /// This is the escalate client. It talks to the escalate server to determine whether to exec() + /// the command directly or to proxy to the escalate server. + async fn run(self) -> anyhow::Result { + let EscalateArgs { file, argv } = self; + escalate_client::run(file, argv).await + } +} + +/// Debugging command to emulate an MCP "shell" tool call. +#[derive(Parser, Debug)] +struct ShellExecArgs { + command: String, +} + +#[tokio::main] +pub async fn main() -> anyhow::Result<()> { + let cli = Cli::parse(); + tracing_subscriber::fmt() + .with_env_filter(EnvFilter::from_default_env()) + .with_writer(std::io::stderr) + .with_ansi(false) + .init(); + + match cli.subcommand { + Some(Commands::Escalate(args)) => { + std::process::exit(args.run().await?); + } + Some(Commands::ShellExec(args)) => { + let bash_path = mcp::get_bash_path()?; + let escalate_server = EscalateServer::new(bash_path, dummy_exec_policy); + let result = escalate_server + .exec( + args.command.clone(), + std::env::vars().collect(), + std::env::current_dir()?, + None, + ) + .await?; + println!("{result:?}"); + std::process::exit(result.exit_code); + } + None => { + let bash_path = mcp::get_bash_path()?; + + tracing::info!("Starting MCP server"); + let service = mcp::serve(bash_path, dummy_exec_policy) + .await + .inspect_err(|e| { + tracing::error!("serving error: {:?}", e); + })?; + + service.waiting().await?; + Ok(()) + } + } +} diff --git a/codex-rs/exec-server/src/posix/escalate_client.rs b/codex-rs/exec-server/src/posix/escalate_client.rs new file mode 100644 index 0000000000..2add1dade2 --- /dev/null +++ b/codex-rs/exec-server/src/posix/escalate_client.rs @@ -0,0 +1,102 @@ +use std::io; +use std::os::fd::AsRawFd; +use std::os::fd::FromRawFd as _; +use std::os::fd::OwnedFd; + +use anyhow::Context as _; + +use crate::posix::escalate_protocol::BASH_EXEC_WRAPPER_ENV_VAR; +use crate::posix::escalate_protocol::ESCALATE_SOCKET_ENV_VAR; +use crate::posix::escalate_protocol::EscalateAction; +use crate::posix::escalate_protocol::EscalateRequest; +use crate::posix::escalate_protocol::EscalateResponse; +use crate::posix::escalate_protocol::SuperExecMessage; +use crate::posix::escalate_protocol::SuperExecResult; +use crate::posix::socket::AsyncDatagramSocket; +use crate::posix::socket::AsyncSocket; + +fn get_escalate_client() -> anyhow::Result { + // TODO: we should defensively require only calling this once, since AsyncSocket will take ownership of the fd. + let client_fd = std::env::var(ESCALATE_SOCKET_ENV_VAR)?.parse::()?; + if client_fd < 0 { + return Err(anyhow::anyhow!( + "{ESCALATE_SOCKET_ENV_VAR} is not a valid file descriptor: {client_fd}" + )); + } + Ok(unsafe { AsyncDatagramSocket::from_raw_fd(client_fd) }?) +} + +pub(crate) async fn run(file: String, argv: Vec) -> anyhow::Result { + let handshake_client = get_escalate_client()?; + let (server, client) = AsyncSocket::pair()?; + const HANDSHAKE_MESSAGE: [u8; 1] = [0]; + handshake_client + .send_with_fds(&HANDSHAKE_MESSAGE, &[server.into_inner().into()]) + .await + .context("failed to send handshake datagram")?; + let env = std::env::vars() + .filter(|(k, _)| { + !matches!( + k.as_str(), + ESCALATE_SOCKET_ENV_VAR | BASH_EXEC_WRAPPER_ENV_VAR + ) + }) + .collect(); + client + .send(EscalateRequest { + file: file.clone().into(), + argv: argv.clone(), + workdir: std::env::current_dir()?, + env, + }) + .await + .context("failed to send EscalateRequest")?; + let message = client.receive::().await?; + match message.action { + EscalateAction::Escalate => { + // TODO: maybe we should send ALL open FDs (except the escalate client)? + let fds_to_send = [ + unsafe { OwnedFd::from_raw_fd(io::stdin().as_raw_fd()) }, + unsafe { OwnedFd::from_raw_fd(io::stdout().as_raw_fd()) }, + unsafe { OwnedFd::from_raw_fd(io::stderr().as_raw_fd()) }, + ]; + + // TODO: also forward signals over the super-exec socket + + client + .send_with_fds( + SuperExecMessage { + fds: fds_to_send.iter().map(AsRawFd::as_raw_fd).collect(), + }, + &fds_to_send, + ) + .await + .context("failed to send SuperExecMessage")?; + let SuperExecResult { exit_code } = client.receive::().await?; + Ok(exit_code) + } + EscalateAction::Run => { + // We avoid std::process::Command here because we want to be as transparent as + // possible. std::os::unix::process::CommandExt has .exec() but it does some funky + // stuff with signal masks and dup2() on its standard FDs, which we don't want. + use std::ffi::CString; + let file = CString::new(file).context("NUL in file")?; + + let argv_cstrs: Vec = argv + .iter() + .map(|s| CString::new(s.as_str()).context("NUL in argv")) + .collect::, _>>()?; + + let mut argv: Vec<*const libc::c_char> = + argv_cstrs.iter().map(|s| s.as_ptr()).collect(); + argv.push(std::ptr::null()); + + let err = unsafe { + libc::execv(file.as_ptr(), argv.as_ptr()); + std::io::Error::last_os_error() + }; + + Err(err.into()) + } + } +} diff --git a/codex-rs/exec-server/src/posix/escalate_protocol.rs b/codex-rs/exec-server/src/posix/escalate_protocol.rs new file mode 100644 index 0000000000..09d33fc982 --- /dev/null +++ b/codex-rs/exec-server/src/posix/escalate_protocol.rs @@ -0,0 +1,49 @@ +use std::collections::HashMap; +use std::os::fd::RawFd; +use std::path::PathBuf; + +use serde::Deserialize; +use serde::Serialize; + +/// 'exec-server escalate' reads this to find the inherited FD for the escalate socket. +pub(super) const ESCALATE_SOCKET_ENV_VAR: &str = "CODEX_ESCALATE_SOCKET"; + +/// The patched bash uses this to wrap exec() calls. +pub(super) const BASH_EXEC_WRAPPER_ENV_VAR: &str = "BASH_EXEC_WRAPPER"; + +/// The client sends this to the server to request an exec() call. +#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Eq)] +pub(super) struct EscalateRequest { + /// The absolute path to the executable to run, i.e. the first arg to exec. + pub(super) file: PathBuf, + /// The argv, including the program name (argv[0]). + pub(super) argv: Vec, + pub(super) workdir: PathBuf, + pub(super) env: HashMap, +} + +/// The server sends this to the client to respond to an exec() request. +#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Eq)] +pub(super) struct EscalateResponse { + pub(super) action: EscalateAction, +} + +#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Eq)] +pub(super) enum EscalateAction { + /// The command should be run directly by the client. + Run, + /// The command should be escalated to the server for execution. + Escalate, +} + +/// The client sends this to the server to forward its open FDs. +#[derive(Clone, Serialize, Deserialize, Debug)] +pub(super) struct SuperExecMessage { + pub(super) fds: Vec, +} + +/// The server responds when the exec()'d command has exited. +#[derive(Clone, Serialize, Deserialize, Debug)] +pub(super) struct SuperExecResult { + pub(super) exit_code: i32, +} diff --git a/codex-rs/exec-server/src/posix/escalate_server.rs b/codex-rs/exec-server/src/posix/escalate_server.rs new file mode 100644 index 0000000000..6d058ff0f0 --- /dev/null +++ b/codex-rs/exec-server/src/posix/escalate_server.rs @@ -0,0 +1,274 @@ +use std::collections::HashMap; +use std::os::fd::AsRawFd; +use std::path::Path; +use std::path::PathBuf; +use std::process::Stdio; +use std::time::Duration; + +use anyhow::Context as _; +use path_absolutize::Absolutize as _; + +use codex_core::exec::SandboxType; +use codex_core::exec::process_exec_tool_call; +use codex_core::get_platform_sandbox; +use codex_core::protocol::SandboxPolicy; +use tokio::process::Command; + +use crate::posix::escalate_protocol::BASH_EXEC_WRAPPER_ENV_VAR; +use crate::posix::escalate_protocol::ESCALATE_SOCKET_ENV_VAR; +use crate::posix::escalate_protocol::EscalateAction; +use crate::posix::escalate_protocol::EscalateRequest; +use crate::posix::escalate_protocol::EscalateResponse; +use crate::posix::escalate_protocol::SuperExecMessage; +use crate::posix::escalate_protocol::SuperExecResult; +use crate::posix::socket::AsyncDatagramSocket; +use crate::posix::socket::AsyncSocket; + +/// This is the policy which decides how to handle an exec() call. +/// +/// `file` is the absolute, canonical path to the executable to run, i.e. the first arg to exec. +/// `argv` is the argv, including the program name (`argv[0]`). +/// `workdir` is the absolute, canonical path to the working directory in which to execute the +/// command. +pub(crate) type ExecPolicy = fn(file: &Path, argv: &[String], workdir: &Path) -> EscalateAction; + +pub(crate) struct EscalateServer { + bash_path: PathBuf, + policy: ExecPolicy, +} + +impl EscalateServer { + pub fn new(bash_path: PathBuf, policy: ExecPolicy) -> Self { + Self { bash_path, policy } + } + + pub async fn exec( + &self, + command: String, + env: HashMap, + workdir: PathBuf, + timeout_ms: Option, + ) -> anyhow::Result { + let (escalate_server, escalate_client) = AsyncDatagramSocket::pair()?; + let client_socket = escalate_client.into_inner(); + client_socket.set_cloexec(false)?; + + let escalate_task = tokio::spawn(escalate_task(escalate_server, self.policy)); + let mut env = env.clone(); + env.insert( + ESCALATE_SOCKET_ENV_VAR.to_string(), + client_socket.as_raw_fd().to_string(), + ); + env.insert( + BASH_EXEC_WRAPPER_ENV_VAR.to_string(), + format!("{} escalate", std::env::current_exe()?.to_string_lossy()), + ); + let result = process_exec_tool_call( + codex_core::exec::ExecParams { + command: vec![ + self.bash_path.to_string_lossy().to_string(), + "-c".to_string(), + command, + ], + cwd: PathBuf::from(&workdir), + timeout_ms, + env, + with_escalated_permissions: None, + justification: None, + arg0: None, + }, + get_platform_sandbox().unwrap_or(SandboxType::None), + // TODO: use the sandbox policy and cwd from the calling client + &SandboxPolicy::ReadOnly, + &PathBuf::from("/__NONEXISTENT__"), // This is ignored for ReadOnly + &None, + None, + ) + .await?; + escalate_task.abort(); + let result = ExecResult { + exit_code: result.exit_code, + output: result.aggregated_output.text, + duration: result.duration, + timed_out: result.timed_out, + }; + Ok(result) + } +} + +async fn escalate_task(socket: AsyncDatagramSocket, policy: ExecPolicy) -> anyhow::Result<()> { + loop { + let (_, mut fds) = socket.receive_with_fds().await?; + if fds.len() != 1 { + tracing::error!("expected 1 fd in datagram handshake, got {}", fds.len()); + continue; + } + let stream_socket = AsyncSocket::from_fd(fds.remove(0))?; + tokio::spawn(async move { + if let Err(err) = handle_escalate_session_with_policy(stream_socket, policy).await { + tracing::error!("escalate session failed: {err:?}"); + } + }); + } +} + +#[derive(Debug)] +pub(crate) struct ExecResult { + pub(crate) exit_code: i32, + pub(crate) output: String, + pub(crate) duration: Duration, + pub(crate) timed_out: bool, +} + +async fn handle_escalate_session_with_policy( + socket: AsyncSocket, + policy: ExecPolicy, +) -> anyhow::Result<()> { + let EscalateRequest { + file, + argv, + workdir, + env, + } = socket.receive::().await?; + let file = PathBuf::from(&file).absolutize()?.into_owned(); + let workdir = PathBuf::from(&workdir).absolutize()?.into_owned(); + let action = policy(file.as_path(), &argv, &workdir); + tracing::debug!("decided {action:?} for {file:?} {argv:?} {workdir:?}"); + match action { + EscalateAction::Run => { + socket + .send(EscalateResponse { + action: EscalateAction::Run, + }) + .await?; + } + EscalateAction::Escalate => { + socket + .send(EscalateResponse { + action: EscalateAction::Escalate, + }) + .await?; + let (msg, fds) = socket + .receive_with_fds::() + .await + .context("failed to receive SuperExecMessage")?; + if fds.len() != msg.fds.len() { + return Err(anyhow::anyhow!( + "mismatched number of fds in SuperExecMessage: {} in the message, {} from the control message", + msg.fds.len(), + fds.len() + )); + } + + if msg + .fds + .iter() + .any(|src_fd| fds.iter().any(|dst_fd| dst_fd.as_raw_fd() == *src_fd)) + { + return Err(anyhow::anyhow!( + "overlapping fds not yet supported in SuperExecMessage" + )); + } + + let mut command = Command::new(file); + command + .args(&argv[1..]) + .arg0(argv[0].clone()) + .envs(&env) + .current_dir(&workdir) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + unsafe { + command.pre_exec(move || { + for (dst_fd, src_fd) in msg.fds.iter().zip(&fds) { + libc::dup2(src_fd.as_raw_fd(), *dst_fd); + } + Ok(()) + }); + } + let mut child = command.spawn()?; + let exit_status = child.wait().await?; + socket + .send(SuperExecResult { + exit_code: exit_status.code().unwrap_or(127), + }) + .await?; + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + use std::collections::HashMap; + use std::path::PathBuf; + + #[tokio::test] + async fn handle_escalate_session_respects_run_in_sandbox_decision() -> anyhow::Result<()> { + let (server, client) = AsyncSocket::pair()?; + let server_task = tokio::spawn(handle_escalate_session_with_policy( + server, + |_file, _argv, _workdir| EscalateAction::Run, + )); + + client + .send(EscalateRequest { + file: PathBuf::from("/bin/echo"), + argv: vec!["echo".to_string()], + workdir: PathBuf::from("/tmp"), + env: HashMap::new(), + }) + .await?; + + let response = client.receive::().await?; + assert_eq!( + EscalateResponse { + action: EscalateAction::Run, + }, + response + ); + server_task.await? + } + + #[tokio::test] + async fn handle_escalate_session_executes_escalated_command() -> anyhow::Result<()> { + let (server, client) = AsyncSocket::pair()?; + let server_task = tokio::spawn(handle_escalate_session_with_policy( + server, + |_file, _argv, _workdir| EscalateAction::Escalate, + )); + + client + .send(EscalateRequest { + file: PathBuf::from("/bin/sh"), + argv: vec![ + "sh".to_string(), + "-c".to_string(), + r#"if [ "$KEY" = VALUE ]; then exit 42; else exit 1; fi"#.to_string(), + ], + workdir: std::env::current_dir()?, + env: HashMap::from([("KEY".to_string(), "VALUE".to_string())]), + }) + .await?; + + let response = client.receive::().await?; + assert_eq!( + EscalateResponse { + action: EscalateAction::Escalate, + }, + response + ); + + client + .send_with_fds(SuperExecMessage { fds: Vec::new() }, &[]) + .await?; + + let result = client.receive::().await?; + assert_eq!(42, result.exit_code); + + server_task.await? + } +} diff --git a/codex-rs/exec-server/src/posix/mcp.rs b/codex-rs/exec-server/src/posix/mcp.rs new file mode 100644 index 0000000000..f4e2b19d4f --- /dev/null +++ b/codex-rs/exec-server/src/posix/mcp.rs @@ -0,0 +1,154 @@ +use std::path::PathBuf; +use std::time::Duration; + +use anyhow::Context as _; +use anyhow::Result; +use rmcp::ErrorData as McpError; +use rmcp::RoleServer; +use rmcp::ServerHandler; +use rmcp::ServiceExt; +use rmcp::handler::server::router::tool::ToolRouter; +use rmcp::handler::server::wrapper::Parameters; +use rmcp::model::*; +use rmcp::schemars; +use rmcp::service::RequestContext; +use rmcp::service::RunningService; +use rmcp::tool; +use rmcp::tool_handler; +use rmcp::tool_router; +use rmcp::transport::stdio; + +use crate::posix::escalate_server; +use crate::posix::escalate_server::EscalateServer; +use crate::posix::escalate_server::ExecPolicy; + +/// Path to our patched bash. +const CODEX_BASH_PATH_ENV_VAR: &str = "CODEX_BASH_PATH"; + +pub(crate) fn get_bash_path() -> Result { + std::env::var(CODEX_BASH_PATH_ENV_VAR) + .map(PathBuf::from) + .context(format!("{CODEX_BASH_PATH_ENV_VAR} must be set")) +} + +#[derive(Debug, serde::Deserialize, schemars::JsonSchema)] +pub struct ExecParams { + /// The bash string to execute. + pub command: String, + /// The working directory to execute the command in. Must be an absolute path. + pub workdir: String, + /// The timeout for the command in milliseconds. + pub timeout_ms: Option, +} + +#[derive(Debug, serde::Serialize, schemars::JsonSchema)] +pub struct ExecResult { + pub exit_code: i32, + pub output: String, + pub duration: Duration, + pub timed_out: bool, +} + +impl From for ExecResult { + fn from(result: escalate_server::ExecResult) -> Self { + Self { + exit_code: result.exit_code, + output: result.output, + duration: result.duration, + timed_out: result.timed_out, + } + } +} + +#[derive(Clone)] +pub struct ExecTool { + tool_router: ToolRouter, + bash_path: PathBuf, + policy: ExecPolicy, +} + +#[tool_router] +impl ExecTool { + pub fn new(bash_path: PathBuf, policy: ExecPolicy) -> Self { + Self { + tool_router: Self::tool_router(), + bash_path, + policy, + } + } + + /// Runs a shell command and returns its output. You MUST provide the workdir as an absolute path. + #[tool] + async fn shell( + &self, + _context: RequestContext, + Parameters(params): Parameters, + ) -> Result { + let escalate_server = EscalateServer::new(self.bash_path.clone(), self.policy); + let result = escalate_server + .exec( + params.command, + // TODO: use ShellEnvironmentPolicy + std::env::vars().collect(), + PathBuf::from(¶ms.workdir), + params.timeout_ms, + ) + .await + .map_err(|e| McpError::internal_error(e.to_string(), None))?; + Ok(CallToolResult::success(vec![Content::json( + ExecResult::from(result), + )?])) + } + + #[allow(dead_code)] + async fn prompt( + &self, + command: String, + workdir: String, + context: RequestContext, + ) -> Result { + context + .peer + .create_elicitation(CreateElicitationRequestParam { + message: format!("Allow Codex to run `{command:?}` in `{workdir:?}`?"), + #[allow(clippy::expect_used)] + requested_schema: ElicitationSchema::builder() + .property("dummy", PrimitiveSchema::String(StringSchema::new())) + .build() + .expect("failed to build elicitation schema"), + }) + .await + .map_err(|e| McpError::internal_error(e.to_string(), None)) + } +} + +#[tool_handler] +impl ServerHandler for ExecTool { + fn get_info(&self) -> ServerInfo { + ServerInfo { + protocol_version: ProtocolVersion::V_2025_06_18, + capabilities: ServerCapabilities::builder().enable_tools().build(), + server_info: Implementation::from_build_env(), + instructions: Some( + "This server provides a tool to execute shell commands and return their output." + .to_string(), + ), + } + } + + async fn initialize( + &self, + _request: InitializeRequestParam, + _context: RequestContext, + ) -> Result { + Ok(self.get_info()) + } +} + +pub(crate) async fn serve( + bash_path: PathBuf, + policy: ExecPolicy, +) -> Result, rmcp::service::ServerInitializeError> { + let tool = ExecTool::new(bash_path, policy); + tool.serve(stdio()).await +} diff --git a/codex-rs/exec-server/src/posix/socket.rs b/codex-rs/exec-server/src/posix/socket.rs new file mode 100644 index 0000000000..92c93dcc7d --- /dev/null +++ b/codex-rs/exec-server/src/posix/socket.rs @@ -0,0 +1,486 @@ +use libc::c_uint; +use serde::Deserialize; +use serde::Serialize; +use socket2::Domain; +use socket2::MaybeUninitSlice; +use socket2::MsgHdr; +use socket2::MsgHdrMut; +use socket2::Socket; +use socket2::Type; +use std::io::IoSlice; +use std::mem::MaybeUninit; +use std::os::fd::AsRawFd; +use std::os::fd::FromRawFd; +use std::os::fd::OwnedFd; +use std::os::fd::RawFd; +use tokio::io::Interest; +use tokio::io::unix::AsyncFd; + +const MAX_FDS_PER_MESSAGE: usize = 16; +const LENGTH_PREFIX_SIZE: usize = size_of::(); +const MAX_DATAGRAM_SIZE: usize = 8192; + +/// Converts a slice of MaybeUninit to a slice of T. +/// +/// The caller guarantees that every element of `buf` is initialized. +fn assume_init(buf: &[MaybeUninit]) -> &[T] { + unsafe { std::slice::from_raw_parts(buf.as_ptr().cast(), buf.len()) } +} + +fn assume_init_slice(buf: &[MaybeUninit; N]) -> &[T; N] { + unsafe { &*(buf as *const [MaybeUninit; N] as *const [T; N]) } +} + +fn assume_init_vec(mut buf: Vec>) -> Vec { + unsafe { + let ptr = buf.as_mut_ptr() as *mut T; + let len = buf.len(); + let cap = buf.capacity(); + std::mem::forget(buf); + Vec::from_raw_parts(ptr, len, cap) + } +} + +fn control_space_for_fds(count: usize) -> usize { + unsafe { libc::CMSG_SPACE((count * size_of::()) as _) as usize } +} + +/// Extracts the FDs from a SCM_RIGHTS control message. +fn extract_fds(control: &[u8]) -> Vec { + let mut fds = Vec::new(); + let mut hdr: libc::msghdr = unsafe { std::mem::zeroed() }; + hdr.msg_control = control.as_ptr() as *mut libc::c_void; + hdr.msg_controllen = control.len() as _; + let hdr = hdr; // drop mut + + let mut cmsg = unsafe { libc::CMSG_FIRSTHDR(&hdr) as *const libc::cmsghdr }; + while !cmsg.is_null() { + let level = unsafe { (*cmsg).cmsg_level }; + let ty = unsafe { (*cmsg).cmsg_type }; + if level == libc::SOL_SOCKET && ty == libc::SCM_RIGHTS { + let data_ptr = unsafe { libc::CMSG_DATA(cmsg).cast::() }; + let fd_count: usize = { + let cmsg_data_len = + unsafe { (*cmsg).cmsg_len as usize } - unsafe { libc::CMSG_LEN(0) as usize }; + cmsg_data_len / size_of::() + }; + for i in 0..fd_count { + let fd = unsafe { data_ptr.add(i).read() }; + fds.push(unsafe { OwnedFd::from_raw_fd(fd) }); + } + } + cmsg = unsafe { libc::CMSG_NXTHDR(&hdr, cmsg) }; + } + fds +} + +/// Read a frame from a SOCK_STREAM socket. +/// +/// A frame is a message length prefix followed by a payload. FDs may be included in the control +/// message when receiving the frame header. +async fn read_frame(async_socket: &AsyncFd) -> std::io::Result<(Vec, Vec)> { + let (message_len, fds) = read_frame_header(async_socket).await?; + let payload = read_frame_payload(async_socket, message_len).await?; + Ok((payload, fds)) +} + +/// Read the frame header (i.e. length) and any FDs from a SOCK_STREAM socket. +async fn read_frame_header( + async_socket: &AsyncFd, +) -> std::io::Result<(usize, Vec)> { + let mut header = [MaybeUninit::::uninit(); LENGTH_PREFIX_SIZE]; + let mut filled = 0; + let mut control = vec![MaybeUninit::::uninit(); control_space_for_fds(MAX_FDS_PER_MESSAGE)]; + let mut captured_control = false; + + while filled < LENGTH_PREFIX_SIZE { + let mut guard = async_socket.readable().await?; + // The first read should come with a control message containing any FDs. + let result = if !captured_control { + guard.try_io(|inner| { + let mut bufs = [MaybeUninitSlice::new(&mut header[filled..])]; + let (read, control_len) = { + let mut msg = MsgHdrMut::new() + .with_buffers(&mut bufs) + .with_control(&mut control); + let read = inner.get_ref().recvmsg(&mut msg, 0)?; + (read, msg.control_len()) + }; + control.truncate(control_len); + captured_control = true; + Ok(read) + }) + } else { + guard.try_io(|inner| inner.get_ref().recv(&mut header[filled..])) + }; + let Ok(result) = result else { + // Would block, try again. + continue; + }; + + let read = result?; + if read == 0 { + return Err(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + "socket closed while receiving frame header", + )); + } + + filled += read; + assert!(filled <= LENGTH_PREFIX_SIZE); + if filled == LENGTH_PREFIX_SIZE { + let len_bytes = assume_init_slice(&header); + let payload_len = u32::from_le_bytes(*len_bytes) as usize; + let fds = extract_fds(assume_init(&control)); + return Ok((payload_len, fds)); + } + } + unreachable!("header loop always returns") +} + +/// Read `message_len` bytes from a SOCK_STREAM socket. +async fn read_frame_payload( + async_socket: &AsyncFd, + message_len: usize, +) -> std::io::Result> { + if message_len == 0 { + return Ok(Vec::new()); + } + let mut payload = vec![MaybeUninit::::uninit(); message_len]; + let mut filled = 0; + while filled < message_len { + let mut guard = async_socket.readable().await?; + let result = guard.try_io(|inner| inner.get_ref().recv(&mut payload[filled..])); + let Ok(result) = result else { + // Would block, try again. + continue; + }; + let read = result?; + if read == 0 { + return Err(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + "socket closed while receiving frame payload", + )); + } + filled += read; + assert!(filled <= message_len); + if filled == message_len { + return Ok(assume_init_vec(payload)); + } + } + unreachable!("loop exits only after returning payload") +} + +fn send_message_bytes(socket: &Socket, data: &[u8], fds: &[OwnedFd]) -> std::io::Result<()> { + if fds.len() > MAX_FDS_PER_MESSAGE { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("too many fds: {}", fds.len()), + )); + } + let mut frame = Vec::with_capacity(LENGTH_PREFIX_SIZE + data.len()); + frame.extend_from_slice(&encode_length(data.len())?); + frame.extend_from_slice(data); + + let mut control = vec![0u8; control_space_for_fds(fds.len())]; + unsafe { + let cmsg = control.as_mut_ptr().cast::(); + (*cmsg).cmsg_len = libc::CMSG_LEN(size_of::() as c_uint * fds.len() as c_uint) as _; + (*cmsg).cmsg_level = libc::SOL_SOCKET; + (*cmsg).cmsg_type = libc::SCM_RIGHTS; + let data_ptr = libc::CMSG_DATA(cmsg).cast::(); + for (i, fd) in fds.iter().enumerate() { + data_ptr.add(i).write(fd.as_raw_fd()); + } + } + + let payload = [IoSlice::new(&frame)]; + let msg = MsgHdr::new().with_buffers(&payload).with_control(&control); + let mut sent = socket.sendmsg(&msg, 0)?; + while sent < frame.len() { + let bytes = socket.send(&frame[sent..])?; + if bytes == 0 { + return Err(std::io::Error::new( + std::io::ErrorKind::WriteZero, + "socket closed while sending frame payload", + )); + } + sent += bytes; + } + Ok(()) +} + +fn encode_length(len: usize) -> std::io::Result<[u8; LENGTH_PREFIX_SIZE]> { + let len_u32 = u32::try_from(len).map_err(|_| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("message too large: {len}"), + ) + })?; + Ok(len_u32.to_le_bytes()) +} + +pub(crate) fn send_json_message( + socket: &Socket, + msg: T, + fds: &[OwnedFd], +) -> std::io::Result<()> { + let data = serde_json::to_vec(&msg)?; + send_message_bytes(socket, &data, fds) +} + +fn send_datagram_bytes(socket: &Socket, data: &[u8], fds: &[OwnedFd]) -> std::io::Result<()> { + if fds.len() > MAX_FDS_PER_MESSAGE { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("too many fds: {}", fds.len()), + )); + } + let mut control = vec![0u8; control_space_for_fds(fds.len())]; + if !fds.is_empty() { + unsafe { + let cmsg = control.as_mut_ptr().cast::(); + (*cmsg).cmsg_len = + libc::CMSG_LEN(size_of::() as c_uint * fds.len() as c_uint) as _; + (*cmsg).cmsg_level = libc::SOL_SOCKET; + (*cmsg).cmsg_type = libc::SCM_RIGHTS; + let data_ptr = libc::CMSG_DATA(cmsg).cast::(); + for (i, fd) in fds.iter().enumerate() { + data_ptr.add(i).write(fd.as_raw_fd()); + } + } + } + let payload = [IoSlice::new(data)]; + let msg = MsgHdr::new().with_buffers(&payload).with_control(&control); + let written = socket.sendmsg(&msg, 0)?; + if written != data.len() { + return Err(std::io::Error::new( + std::io::ErrorKind::WriteZero, + format!( + "short datagram write: wrote {written} bytes out of {}", + data.len() + ), + )); + } + Ok(()) +} + +fn receive_datagram_bytes(socket: &Socket) -> std::io::Result<(Vec, Vec)> { + let mut buffer = vec![MaybeUninit::::uninit(); MAX_DATAGRAM_SIZE]; + let mut control = vec![MaybeUninit::::uninit(); control_space_for_fds(MAX_FDS_PER_MESSAGE)]; + let (read, control_len) = { + let mut bufs = [MaybeUninitSlice::new(&mut buffer)]; + let mut msg = MsgHdrMut::new() + .with_buffers(&mut bufs) + .with_control(&mut control); + let read = socket.recvmsg(&mut msg, 0)?; + (read, msg.control_len()) + }; + let data = assume_init(&buffer[..read]).to_vec(); + let fds = extract_fds(assume_init(&control[..control_len])); + Ok((data, fds)) +} + +pub(crate) struct AsyncSocket { + inner: AsyncFd, +} + +impl AsyncSocket { + fn new(socket: Socket) -> std::io::Result { + socket.set_nonblocking(true)?; + let async_socket = AsyncFd::new(socket)?; + Ok(AsyncSocket { + inner: async_socket, + }) + } + + pub fn from_fd(fd: OwnedFd) -> std::io::Result { + AsyncSocket::new(Socket::from(fd)) + } + + pub fn pair() -> std::io::Result<(AsyncSocket, AsyncSocket)> { + let (server, client) = Socket::pair(Domain::UNIX, Type::STREAM, None)?; + Ok((AsyncSocket::new(server)?, AsyncSocket::new(client)?)) + } + + pub async fn send_with_fds( + &self, + msg: T, + fds: &[OwnedFd], + ) -> std::io::Result<()> { + self.inner + .async_io(Interest::WRITABLE, |socket| { + send_json_message(socket, &msg, fds) + }) + .await + } + + pub async fn receive_with_fds Deserialize<'de>>( + &self, + ) -> std::io::Result<(T, Vec)> { + let (payload, fds) = read_frame(&self.inner).await?; + let message: T = serde_json::from_slice(&payload)?; + Ok((message, fds)) + } + + pub async fn send(&self, msg: T) -> std::io::Result<()> + where + T: Serialize, + { + self.send_with_fds(&msg, &[]).await + } + + pub async fn receive Deserialize<'de>>(&self) -> std::io::Result { + let (msg, fds) = self.receive_with_fds().await?; + if !fds.is_empty() { + tracing::warn!("unexpected fds in receive: {}", fds.len()); + } + Ok(msg) + } + + pub fn into_inner(self) -> Socket { + self.inner.into_inner() + } +} + +pub(crate) struct AsyncDatagramSocket { + inner: AsyncFd, +} + +impl AsyncDatagramSocket { + fn new(socket: Socket) -> std::io::Result { + socket.set_nonblocking(true)?; + Ok(Self { + inner: AsyncFd::new(socket)?, + }) + } + + pub unsafe fn from_raw_fd(fd: RawFd) -> std::io::Result { + Self::new(unsafe { Socket::from_raw_fd(fd) }) + } + + pub fn pair() -> std::io::Result<(Self, Self)> { + let (server, client) = Socket::pair(Domain::UNIX, Type::DGRAM, None)?; + Ok((Self::new(server)?, Self::new(client)?)) + } + + pub async fn send_with_fds(&self, data: &[u8], fds: &[OwnedFd]) -> std::io::Result<()> { + self.inner + .async_io(Interest::WRITABLE, |socket| { + send_datagram_bytes(socket, data, fds) + }) + .await + } + + pub async fn receive_with_fds(&self) -> std::io::Result<(Vec, Vec)> { + self.inner + .async_io(Interest::READABLE, receive_datagram_bytes) + .await + } + + pub fn into_inner(self) -> Socket { + self.inner.into_inner() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + use serde::Deserialize; + use serde::Serialize; + use std::os::fd::AsFd; + use std::os::fd::AsRawFd; + use tempfile::NamedTempFile; + + #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)] + struct TestPayload { + id: i32, + label: String, + } + + fn fd_list(count: usize) -> std::io::Result> { + let file = NamedTempFile::new()?; + let mut fds = Vec::new(); + for _ in 0..count { + fds.push(file.as_fd().try_clone_to_owned()?); + } + Ok(fds) + } + + #[tokio::test] + async fn async_socket_round_trips_payload_and_fds() -> std::io::Result<()> { + let (server, client) = AsyncSocket::pair()?; + let payload = TestPayload { + id: 7, + label: "round-trip".to_string(), + }; + let send_fds = fd_list(1)?; + + let receive_task = + tokio::spawn(async move { server.receive_with_fds::().await }); + client.send_with_fds(payload.clone(), &send_fds).await?; + drop(send_fds); + + let (received_payload, received_fds) = receive_task.await.unwrap()?; + assert_eq!(payload, received_payload); + assert_eq!(1, received_fds.len()); + let fd_status = unsafe { libc::fcntl(received_fds[0].as_raw_fd(), libc::F_GETFD) }; + assert!( + fd_status >= 0, + "expected received file descriptor to be valid, but got {fd_status}", + ); + Ok(()) + } + + #[tokio::test] + async fn async_datagram_sockets_round_trip_messages() -> std::io::Result<()> { + let (server, client) = AsyncDatagramSocket::pair()?; + let data = b"datagram payload".to_vec(); + let send_fds = fd_list(1)?; + let receive_task = tokio::spawn(async move { server.receive_with_fds().await }); + + client.send_with_fds(&data, &send_fds).await?; + drop(send_fds); + + let (received_bytes, received_fds) = receive_task.await.unwrap()?; + assert_eq!(data, received_bytes); + assert_eq!(1, received_fds.len()); + Ok(()) + } + + #[test] + fn send_message_bytes_rejects_excessive_fd_counts() -> std::io::Result<()> { + let (socket, _peer) = Socket::pair(Domain::UNIX, Type::STREAM, None)?; + let fds = fd_list(MAX_FDS_PER_MESSAGE + 1)?; + let err = send_message_bytes(&socket, b"hello", &fds).unwrap_err(); + assert_eq!(std::io::ErrorKind::InvalidInput, err.kind()); + Ok(()) + } + + #[test] + fn send_datagram_bytes_rejects_excessive_fd_counts() -> std::io::Result<()> { + let (socket, _peer) = Socket::pair(Domain::UNIX, Type::DGRAM, None)?; + let fds = fd_list(MAX_FDS_PER_MESSAGE + 1)?; + let err = send_datagram_bytes(&socket, b"hi", &fds).unwrap_err(); + assert_eq!(std::io::ErrorKind::InvalidInput, err.kind()); + Ok(()) + } + + #[test] + fn encode_length_errors_for_oversized_messages() { + let err = encode_length(usize::MAX).unwrap_err(); + assert_eq!(std::io::ErrorKind::InvalidInput, err.kind()); + } + + #[tokio::test] + async fn receive_fails_when_peer_closes_before_header() { + let (server, client) = AsyncSocket::pair().expect("failed to create socket pair"); + drop(client); + let err = server + .receive::() + .await + .expect_err("expected read failure"); + assert_eq!(std::io::ErrorKind::UnexpectedEof, err.kind()); + } +} From 4fb714fb46ac729f38ba8219fba59b491717c100 Mon Sep 17 00:00:00 2001 From: zhao-oai Date: Tue, 18 Nov 2025 16:25:50 -0800 Subject: [PATCH 12/18] updating codex backend models (#6855) --- .../src/models/credit_status_details.rs | 49 +++++++++++++++++++ .../src/models/mod.rs | 3 ++ .../src/models/rate_limit_status_payload.rs | 16 +++++- .../src/models/rate_limit_window_snapshot.rs | 1 - 4 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 codex-rs/codex-backend-openapi-models/src/models/credit_status_details.rs diff --git a/codex-rs/codex-backend-openapi-models/src/models/credit_status_details.rs b/codex-rs/codex-backend-openapi-models/src/models/credit_status_details.rs new file mode 100644 index 0000000000..d444a7a8d2 --- /dev/null +++ b/codex-rs/codex-backend-openapi-models/src/models/credit_status_details.rs @@ -0,0 +1,49 @@ +/* + * codex-backend + * + * codex-backend + * + * The version of the OpenAPI document: 0.0.1 + * + * Generated by: https://openapi-generator.tech + */ +use serde::Deserialize; +use serde::Serialize; + +#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] +pub struct CreditStatusDetails { + #[serde(rename = "unlimited")] + pub unlimited: bool, + #[serde( + rename = "balance", + default, + with = "::serde_with::rust::double_option", + skip_serializing_if = "Option::is_none" + )] + pub balance: Option>, + #[serde( + rename = "approx_local_messages", + default, + with = "::serde_with::rust::double_option", + skip_serializing_if = "Option::is_none" + )] + pub approx_local_messages: Option>>, + #[serde( + rename = "approx_cloud_messages", + default, + with = "::serde_with::rust::double_option", + skip_serializing_if = "Option::is_none" + )] + pub approx_cloud_messages: Option>>, +} + +impl CreditStatusDetails { + pub fn new(unlimited: bool) -> CreditStatusDetails { + CreditStatusDetails { + unlimited, + balance: None, + approx_local_messages: None, + approx_cloud_messages: None, + } + } +} diff --git a/codex-rs/codex-backend-openapi-models/src/models/mod.rs b/codex-rs/codex-backend-openapi-models/src/models/mod.rs index 96348d72c2..d767154925 100644 --- a/codex-rs/codex-backend-openapi-models/src/models/mod.rs +++ b/codex-rs/codex-backend-openapi-models/src/models/mod.rs @@ -32,3 +32,6 @@ pub use self::rate_limit_status_details::RateLimitStatusDetails; pub mod rate_limit_window_snapshot; pub use self::rate_limit_window_snapshot::RateLimitWindowSnapshot; + +pub mod credit_status_details; +pub use self::credit_status_details::CreditStatusDetails; diff --git a/codex-rs/codex-backend-openapi-models/src/models/rate_limit_status_payload.rs b/codex-rs/codex-backend-openapi-models/src/models/rate_limit_status_payload.rs index d2af76f4d7..0f5caf52ff 100644 --- a/codex-rs/codex-backend-openapi-models/src/models/rate_limit_status_payload.rs +++ b/codex-rs/codex-backend-openapi-models/src/models/rate_limit_status_payload.rs @@ -23,6 +23,13 @@ pub struct RateLimitStatusPayload { skip_serializing_if = "Option::is_none" )] pub rate_limit: Option>>, + #[serde( + rename = "credits", + default, + with = "::serde_with::rust::double_option", + skip_serializing_if = "Option::is_none" + )] + pub credits: Option>>, } impl RateLimitStatusPayload { @@ -30,12 +37,15 @@ impl RateLimitStatusPayload { RateLimitStatusPayload { plan_type, rate_limit: None, + credits: None, } } } #[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] pub enum PlanType { + #[serde(rename = "guest")] + Guest, #[serde(rename = "free")] Free, #[serde(rename = "go")] @@ -44,6 +54,8 @@ pub enum PlanType { Plus, #[serde(rename = "pro")] Pro, + #[serde(rename = "free_workspace")] + FreeWorkspace, #[serde(rename = "team")] Team, #[serde(rename = "business")] @@ -52,6 +64,8 @@ pub enum PlanType { Education, #[serde(rename = "quorum")] Quorum, + #[serde(rename = "k12")] + K12, #[serde(rename = "enterprise")] Enterprise, #[serde(rename = "edu")] @@ -60,6 +74,6 @@ pub enum PlanType { impl Default for PlanType { fn default() -> PlanType { - Self::Free + Self::Guest } } diff --git a/codex-rs/codex-backend-openapi-models/src/models/rate_limit_window_snapshot.rs b/codex-rs/codex-backend-openapi-models/src/models/rate_limit_window_snapshot.rs index 4fc04f4be6..b2a6c0c228 100644 --- a/codex-rs/codex-backend-openapi-models/src/models/rate_limit_window_snapshot.rs +++ b/codex-rs/codex-backend-openapi-models/src/models/rate_limit_window_snapshot.rs @@ -7,7 +7,6 @@ * * Generated by: https://openapi-generator.tech */ - use serde::Deserialize; use serde::Serialize; From cf57320b9fea784228a26bf11f3747404b204d63 Mon Sep 17 00:00:00 2001 From: iceweasel-oai Date: Tue, 18 Nov 2025 16:35:00 -0800 Subject: [PATCH 13/18] windows sandbox: support multiple workspace roots (#6854) The Windows sandbox did not previously support multiple workspace roots via config. Now it does --- codex-rs/Cargo.lock | 1 + codex-rs/cli/src/debug_sandbox.rs | 8 +-- codex-rs/core/src/exec.rs | 17 ++--- codex-rs/windows-sandbox-rs/Cargo.toml | 3 + .../windows-sandbox-rs/sandbox_smoketests.py | 46 +++++++++++- codex-rs/windows-sandbox-rs/src/allow.rs | 72 +++++++++++++++---- codex-rs/windows-sandbox-rs/src/lib.rs | 19 +++-- codex-rs/windows-sandbox-rs/src/policy.rs | 41 +++-------- 8 files changed, 141 insertions(+), 66 deletions(-) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index a7ba2a4c68..c756c30ab9 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1627,6 +1627,7 @@ name = "codex-windows-sandbox" version = "0.1.0" dependencies = [ "anyhow", + "codex-protocol", "dirs-next", "dunce", "rand 0.8.5", diff --git a/codex-rs/cli/src/debug_sandbox.rs b/codex-rs/cli/src/debug_sandbox.rs index df4c2e97c9..26fecd55c6 100644 --- a/codex-rs/cli/src/debug_sandbox.rs +++ b/codex-rs/cli/src/debug_sandbox.rs @@ -138,11 +138,7 @@ async fn run_command_under_sandbox( { use codex_windows_sandbox::run_windows_sandbox_capture; - let policy_str = match &config.sandbox_policy { - codex_core::protocol::SandboxPolicy::DangerFullAccess => "workspace-write", - codex_core::protocol::SandboxPolicy::ReadOnly => "read-only", - codex_core::protocol::SandboxPolicy::WorkspaceWrite { .. } => "workspace-write", - }; + let policy_str = serde_json::to_string(&config.sandbox_policy)?; let sandbox_cwd = sandbox_policy_cwd.clone(); let cwd_clone = cwd.clone(); @@ -153,7 +149,7 @@ async fn run_command_under_sandbox( // Preflight audit is invoked elsewhere at the appropriate times. let res = tokio::task::spawn_blocking(move || { run_windows_sandbox_capture( - policy_str, + policy_str.as_str(), &sandbox_cwd, base_dir.as_path(), command_vec, diff --git a/codex-rs/core/src/exec.rs b/codex-rs/core/src/exec.rs index b44a40bdb2..583c2233a2 100644 --- a/codex-rs/core/src/exec.rs +++ b/codex-rs/core/src/exec.rs @@ -182,12 +182,11 @@ async fn exec_windows_sandbox( .. } = params; - let policy_str = match sandbox_policy { - SandboxPolicy::DangerFullAccess => "workspace-write", - SandboxPolicy::ReadOnly => "read-only", - SandboxPolicy::WorkspaceWrite { .. } => "workspace-write", - }; - + let policy_str = serde_json::to_string(sandbox_policy).map_err(|err| { + CodexErr::Io(io::Error::other(format!( + "failed to serialize Windows sandbox policy: {err}" + ))) + })?; let sandbox_cwd = cwd.clone(); let codex_home = find_codex_home().map_err(|err| { CodexErr::Io(io::Error::other(format!( @@ -196,7 +195,7 @@ async fn exec_windows_sandbox( })?; let spawn_res = tokio::task::spawn_blocking(move || { run_windows_sandbox_capture( - policy_str, + policy_str.as_str(), &sandbox_cwd, codex_home.as_ref(), command, @@ -444,7 +443,9 @@ async fn exec( stdout_stream: Option, ) -> Result { #[cfg(target_os = "windows")] - if sandbox == SandboxType::WindowsRestrictedToken { + if sandbox == SandboxType::WindowsRestrictedToken + && !matches!(sandbox_policy, SandboxPolicy::DangerFullAccess) + { return exec_windows_sandbox(params, sandbox_policy).await; } let timeout = params.timeout_duration(); diff --git a/codex-rs/windows-sandbox-rs/Cargo.toml b/codex-rs/windows-sandbox-rs/Cargo.toml index ba39bbe0fc..7664ca719a 100644 --- a/codex-rs/windows-sandbox-rs/Cargo.toml +++ b/codex-rs/windows-sandbox-rs/Cargo.toml @@ -12,6 +12,9 @@ anyhow = "1.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" dunce = "1.0" +[dependencies.codex-protocol] +package = "codex-protocol" +path = "../protocol" [dependencies.rand] version = "0.8" default-features = false diff --git a/codex-rs/windows-sandbox-rs/sandbox_smoketests.py b/codex-rs/windows-sandbox-rs/sandbox_smoketests.py index 63bf349096..fb8f90121b 100644 --- a/codex-rs/windows-sandbox-rs/sandbox_smoketests.py +++ b/codex-rs/windows-sandbox-rs/sandbox_smoketests.py @@ -50,6 +50,7 @@ TIMEOUT_SEC = 20 WS_ROOT = Path(os.environ["USERPROFILE"]) / "sbx_ws_tests" OUTSIDE = Path(os.environ["USERPROFILE"]) / "sbx_ws_outside" # outside CWD for deny checks +EXTRA_ROOT = Path(os.environ["USERPROFILE"]) / "WorkspaceRoot" # additional writable root ENV_BASE = {} # extend if needed @@ -57,7 +58,13 @@ class CaseResult: def __init__(self, name: str, ok: bool, detail: str = ""): self.name, self.ok, self.detail = name, ok, detail -def run_sbx(policy: str, cmd_argv: List[str], cwd: Path, env_extra: Optional[dict] = None) -> Tuple[int, str, str]: +def run_sbx( + policy: str, + cmd_argv: List[str], + cwd: Path, + env_extra: Optional[dict] = None, + additional_root: Optional[Path] = None, +) -> Tuple[int, str, str]: env = os.environ.copy() env.update(ENV_BASE) if env_extra: @@ -68,7 +75,15 @@ def run_sbx(policy: str, cmd_argv: List[str], cwd: Path, env_extra: Optional[dic raise ValueError(f"unknown policy: {policy}") policy_flags: List[str] = ["--full-auto"] if policy == "workspace-write" else [] - argv = [*CODEX_CMD, "sandbox", "windows", *policy_flags, "--", *cmd_argv] + overrides: List[str] = [] + if policy == "workspace-write" and additional_root is not None: + # Use config override to inject an additional writable root. + overrides = [ + "-c", + f'sandbox_workspace_write.writable_roots=["{additional_root.as_posix()}"]', + ] + + argv = [*CODEX_CMD, "sandbox", "windows", *policy_flags, *overrides, "--", *cmd_argv] print(cmd_argv) cp = subprocess.run(argv, cwd=str(cwd), env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -134,6 +149,7 @@ def main() -> int: results: List[CaseResult] = [] make_dir_clean(WS_ROOT) OUTSIDE.mkdir(exist_ok=True) + EXTRA_ROOT.mkdir(exist_ok=True) # Environment probe: some hosts allow TEMP writes even under read-only # tokens due to ACLs and restricted SID semantics. Detect and adapt tests. probe_rc, _, _ = run_sbx( @@ -165,6 +181,32 @@ def main() -> int: rc, out, err = run_sbx("workspace-write", ["cmd", "/c", f"echo nope > {outside_file}"], WS_ROOT) add("WS: write outside workspace denied", rc != 0 and assert_not_exists(outside_file), f"rc={rc}") + # 3b. WS: allow write in additional workspace root + extra_target = EXTRA_ROOT / "extra_ok.txt" + remove_if_exists(extra_target) + rc, out, err = run_sbx( + "workspace-write", + ["cmd", "/c", f"echo extra > {extra_target}"], + WS_ROOT, + additional_root=EXTRA_ROOT, + ) + add("WS: write in additional root allowed", rc == 0 and assert_exists(extra_target), f"rc={rc}, err={err}") + + # 3c. RO: deny write in additional workspace root + ro_extra_target = EXTRA_ROOT / "extra_ro.txt" + remove_if_exists(ro_extra_target) + rc, out, err = run_sbx( + "read-only", + ["cmd", "/c", f"echo nope > {ro_extra_target}"], + WS_ROOT, + additional_root=EXTRA_ROOT, + ) + add( + "RO: write in additional root denied", + rc != 0 and assert_not_exists(ro_extra_target), + f"rc={rc}", + ) + # 4. WS: allow TEMP write rc, out, err = run_sbx("workspace-write", ["cmd", "/c", "echo tempok > %TEMP%\\ws_temp_ok.txt"], WS_ROOT) add("WS: TEMP write allowed", rc == 0, f"rc={rc}") diff --git a/codex-rs/windows-sandbox-rs/src/allow.rs b/codex-rs/windows-sandbox-rs/src/allow.rs index 01254fb58c..de62159994 100644 --- a/codex-rs/windows-sandbox-rs/src/allow.rs +++ b/codex-rs/windows-sandbox-rs/src/allow.rs @@ -1,37 +1,83 @@ -use crate::policy::SandboxMode; use crate::policy::SandboxPolicy; +use dunce::canonicalize; use std::collections::HashMap; use std::path::Path; use std::path::PathBuf; pub fn compute_allow_paths( policy: &SandboxPolicy, - _policy_cwd: &Path, + policy_cwd: &Path, command_cwd: &Path, env_map: &HashMap, ) -> Vec { let mut allow: Vec = Vec::new(); let mut seen = std::collections::HashSet::new(); - if matches!(policy.0, SandboxMode::WorkspaceWrite) { - let abs = command_cwd.to_path_buf(); - if seen.insert(abs.to_string_lossy().to_string()) && abs.exists() { - allow.push(abs); + + let mut add_path = |p: PathBuf| { + if seen.insert(p.to_string_lossy().to_string()) && p.exists() { + allow.push(p); + } + }; + + if matches!(policy, SandboxPolicy::WorkspaceWrite { .. }) { + add_path(command_cwd.to_path_buf()); + if let SandboxPolicy::WorkspaceWrite { writable_roots, .. } = policy { + for root in writable_roots { + let candidate = if root.is_absolute() { + root.clone() + } else { + policy_cwd.join(root) + }; + let canonical = canonicalize(&candidate).unwrap_or(candidate); + add_path(canonical); + } } } - if !matches!(policy.0, SandboxMode::ReadOnly) { + if !matches!(policy, SandboxPolicy::ReadOnly) { for key in ["TEMP", "TMP"] { if let Some(v) = env_map.get(key) { let abs = PathBuf::from(v); - if seen.insert(abs.to_string_lossy().to_string()) && abs.exists() { - allow.push(abs); - } + add_path(abs); } else if let Ok(v) = std::env::var(key) { let abs = PathBuf::from(v); - if seen.insert(abs.to_string_lossy().to_string()) && abs.exists() { - allow.push(abs); - } + add_path(abs); } } } allow } + +#[cfg(test)] +mod tests { + use super::compute_allow_paths; + use codex_protocol::protocol::SandboxPolicy; + use std::collections::HashMap; + use std::fs; + use std::path::PathBuf; + + #[test] + fn includes_additional_writable_roots() { + let command_cwd = PathBuf::from(r"C:\Workspace"); + let extra_root = PathBuf::from(r"C:\AdditionalWritableRoot"); + let _ = fs::create_dir_all(&command_cwd); + let _ = fs::create_dir_all(&extra_root); + + let policy = SandboxPolicy::WorkspaceWrite { + writable_roots: vec![extra_root.clone()], + network_access: false, + exclude_tmpdir_env_var: false, + exclude_slash_tmp: false, + }; + + let allow = compute_allow_paths(&policy, &command_cwd, &command_cwd, &HashMap::new()); + + assert!( + allow.iter().any(|p| p == &command_cwd), + "command cwd should be allowed" + ); + assert!( + allow.iter().any(|p| p == &extra_root), + "additional writable root should be allowed" + ); + } +} diff --git a/codex-rs/windows-sandbox-rs/src/lib.rs b/codex-rs/windows-sandbox-rs/src/lib.rs index a868654e7c..9d5e9aeae2 100644 --- a/codex-rs/windows-sandbox-rs/src/lib.rs +++ b/codex-rs/windows-sandbox-rs/src/lib.rs @@ -40,7 +40,7 @@ mod windows_impl { use super::logging::log_failure; use super::logging::log_start; use super::logging::log_success; - use super::policy::SandboxMode; + use super::policy::parse_policy; use super::policy::SandboxPolicy; use super::token::convert_string_sid_to_sid; use super::winutil::format_last_error; @@ -194,7 +194,7 @@ mod windows_impl { mut env_map: HashMap, timeout_ms: Option, ) -> Result { - let policy = SandboxPolicy::parse(policy_json_or_preset)?; + let policy = parse_policy(policy_json_or_preset)?; normalize_null_device_env(&mut env_map); ensure_non_interactive_pager(&mut env_map); apply_no_network_to_env(&mut env_map)?; @@ -206,27 +206,32 @@ mod windows_impl { let logs_base_dir = Some(codex_home); log_start(&command, logs_base_dir); let cap_sid_path = cap_sid_file(codex_home); + let is_workspace_write = matches!(&policy, SandboxPolicy::WorkspaceWrite { .. }); + let (h_token, psid_to_use): (HANDLE, *mut c_void) = unsafe { - match &policy.0 { - SandboxMode::ReadOnly => { + match &policy { + SandboxPolicy::ReadOnly => { let caps = load_or_create_cap_sids(codex_home); ensure_dir(&cap_sid_path)?; fs::write(&cap_sid_path, serde_json::to_string(&caps)?)?; let psid = convert_string_sid_to_sid(&caps.readonly).unwrap(); super::token::create_readonly_token_with_cap(psid)? } - SandboxMode::WorkspaceWrite => { + SandboxPolicy::WorkspaceWrite { .. } => { let caps = load_or_create_cap_sids(codex_home); ensure_dir(&cap_sid_path)?; fs::write(&cap_sid_path, serde_json::to_string(&caps)?)?; let psid = convert_string_sid_to_sid(&caps.workspace).unwrap(); super::token::create_workspace_write_token_with_cap(psid)? } + SandboxPolicy::DangerFullAccess => { + anyhow::bail!("DangerFullAccess is not supported for sandboxing") + } } }; unsafe { - if matches!(policy.0, SandboxMode::WorkspaceWrite) { + if is_workspace_write { if let Ok(base) = super::token::get_current_token_for_restriction() { if let Ok(bytes) = super::token::get_logon_sid_bytes(base) { let mut tmp = bytes.clone(); @@ -238,7 +243,7 @@ mod windows_impl { } } - let persist_aces = matches!(policy.0, SandboxMode::WorkspaceWrite); + let persist_aces = is_workspace_write; let allow = compute_allow_paths(&policy, sandbox_policy_cwd, ¤t_dir, &env_map); let mut guards: Vec<(PathBuf, *mut c_void)> = Vec::new(); unsafe { diff --git a/codex-rs/windows-sandbox-rs/src/policy.rs b/codex-rs/windows-sandbox-rs/src/policy.rs index 32cfa79f98..4c62c71df3 100644 --- a/codex-rs/windows-sandbox-rs/src/policy.rs +++ b/codex-rs/windows-sandbox-rs/src/policy.rs @@ -1,36 +1,17 @@ use anyhow::Result; -use serde::Deserialize; -use serde::Serialize; +pub use codex_protocol::protocol::SandboxPolicy; -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct PolicyJson { - pub mode: String, - #[serde(default)] - pub workspace_roots: Vec, -} - -#[derive(Clone, Debug)] -pub enum SandboxMode { - ReadOnly, - WorkspaceWrite, -} - -#[derive(Clone, Debug)] -pub struct SandboxPolicy(pub SandboxMode); - -impl SandboxPolicy { - pub fn parse(value: &str) -> Result { - match value { - "read-only" => Ok(SandboxPolicy(SandboxMode::ReadOnly)), - "workspace-write" => Ok(SandboxPolicy(SandboxMode::WorkspaceWrite)), - other => { - let pj: PolicyJson = serde_json::from_str(other)?; - Ok(match pj.mode.as_str() { - "read-only" => SandboxPolicy(SandboxMode::ReadOnly), - "workspace-write" => SandboxPolicy(SandboxMode::WorkspaceWrite), - _ => SandboxPolicy(SandboxMode::ReadOnly), - }) +pub fn parse_policy(value: &str) -> Result { + match value { + "read-only" => Ok(SandboxPolicy::ReadOnly), + "workspace-write" => Ok(SandboxPolicy::new_workspace_write_policy()), + "danger-full-access" => anyhow::bail!("DangerFullAccess is not supported for sandboxing"), + other => { + let parsed: SandboxPolicy = serde_json::from_str(other)?; + if matches!(parsed, SandboxPolicy::DangerFullAccess) { + anyhow::bail!("DangerFullAccess is not supported for sandboxing"); } + Ok(parsed) } } } From b952bd264927d6824a9d5dafcbaceb111cfe8548 Mon Sep 17 00:00:00 2001 From: iceweasel-oai Date: Tue, 18 Nov 2025 16:43:34 -0800 Subject: [PATCH 14/18] smoketest for browser vuln, rough draft of Windows security doc (#6822) --- .../windows-sandbox-rs/sandbox_smoketests.py | 23 ++++++++++++++----- docs/windows_sandbox_security.md | 20 ++++++++++++++++ 2 files changed, 37 insertions(+), 6 deletions(-) create mode 100644 docs/windows_sandbox_security.md diff --git a/codex-rs/windows-sandbox-rs/sandbox_smoketests.py b/codex-rs/windows-sandbox-rs/sandbox_smoketests.py index fb8f90121b..0fc049d4ed 100644 --- a/codex-rs/windows-sandbox-rs/sandbox_smoketests.py +++ b/codex-rs/windows-sandbox-rs/sandbox_smoketests.py @@ -477,12 +477,23 @@ def main() -> int: rc, out, err = run_sbx("workspace-write", ["cmd", "/c", f"echo leak > {outside_after_timeout}"], WS_ROOT) add("WS: post-timeout outside write still denied", rc != 0 and assert_not_exists(outside_after_timeout), f"rc={rc}") - # 41. WS: additional protected path variation (.ssh) - ssh_var = WS_ROOT / ".SsH" / "config" - remove_if_exists(ssh_var.parent) - ssh_var.parent.mkdir(exist_ok=True) - rc, out, err = run_sbx("workspace-write", ["cmd", "/c", "echo key > .SsH\\config"], WS_ROOT) - add("WS: protected path variation (.ssh) denied", rc != 0 and assert_not_exists(ssh_var), f"rc={rc}") + # 41. RO: Start-Process https blocked (KNOWN FAIL until GUI escape fixed) + rc, out, err = run_sbx( + "read-only", + [ + "powershell", + "-NoLogo", + "-NoProfile", + "-Command", + "Start-Process 'https://codex-invalid.local/smoke'", + ], + WS_ROOT, + ) + add( + "RO: Start-Process https denied (KNOWN FAIL)", + rc != 0, + f"rc={rc}, stdout={out}, stderr={err}", + ) return summarize(results) diff --git a/docs/windows_sandbox_security.md b/docs/windows_sandbox_security.md new file mode 100644 index 0000000000..3dab0449d1 --- /dev/null +++ b/docs/windows_sandbox_security.md @@ -0,0 +1,20 @@ +# Windows Sandbox Status + +## Implementation Overview + +When commands run via `codex sandbox windows …` (or when the CLI/TUI calls into the same crate in-process for sandboxed turns), the launcher configures a restricted Windows token and an allowlist policy scoped to the declared workspace roots. Writes are blocked everywhere except inside those roots (plus `%TEMP%` when workspace-write mode is requested), and common escape vectors such as alternate data streams, UNC paths, and device handles are denied proactively. The CLI also injects stub executables (for example, wrapping `ssh`) ahead of the host PATH so we can intercept dangerous tools before they ever leave the sandbox. + +## Known Security Limitations + +Running `python windows-sandbox-rs/sandbox_smoketests.py` with full filesystem and network access currently results in **37/42** passing cases. The list below focuses on the four high-value failures numbered #32 and higher in the smoketests (earlier tests are less security-focused). + +| Test | Purpose | +| --------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| ADS write denied (#32) | Ensures alternate data streams cannot be written inside the workspace, preventing tools from hiding payloads in ADS. The sandbox currently allows the write (process returns `rc=0`). | +| protected path case-variation denied (#33) | Confirms that protected directories such as `.git` remain blocked even when attackers use case tricks like `.GiT`. The current allowlist treats `.GiT` as distinct, so the write succeeds. | +| PATH stub bypass denied (#35) | Verifies that a workspace-provided `ssh.bat` shim placed ahead of the host PATH runs instead of the real `ssh`. The sandbox exits early before emitting the shim's `stubbed` output, so we cannot prove the interception works. | +| Start-Process https denied (KNOWN FAIL) (#41) | Validates that read-only runs cannot launch the host's default browser via `Start-Process 'https://...'`. Today the command succeeds (exit code 0) because Explorer handles the ShellExecute request outside the sandbox. The failure is captured by `windows-sandbox-rs/sandbox_smoketests.py` (last case). | + +## Want to Help? + +If you are a security-minded Windows user, help us get these tests passing! Improved implementations that make these smoke tests pass meaningfully reduce Codex's escape surface. After iterating, rerun `python windows-sandbox-rs/sandbox_smoketests.py` to validate the fixes and help us drive the suite toward 42/42. From 526eb3ff82b7404a32acf2671977eaa888a3d021 Mon Sep 17 00:00:00 2001 From: Jeremy Rose <172423086+nornagon-openai@users.noreply.github.com> Date: Tue, 18 Nov 2025 16:47:37 -0800 Subject: [PATCH 15/18] tui: add branch to 'codex resume', filter by cwd (#6232) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit By default, show only sessions that shared a cwd with the current cwd. `--all` shows all sessions in all cwds. Also, show the branch name from the rollout metadata. Screenshot 2025-11-04 at 3 30 47 PM --- codex-rs/cli/src/main.rs | 28 ++- codex-rs/tui/src/cli.rs | 4 + codex-rs/tui/src/lib.rs | 1 + codex-rs/tui/src/resume_picker.rs | 235 ++++++++++++++---- ...me_picker__tests__resume_picker_table.snap | 8 +- docs/getting-started.md | 1 + 6 files changed, 227 insertions(+), 50 deletions(-) diff --git a/codex-rs/cli/src/main.rs b/codex-rs/cli/src/main.rs index 6a3b24aa98..bb60870de5 100644 --- a/codex-rs/cli/src/main.rs +++ b/codex-rs/cli/src/main.rs @@ -134,6 +134,10 @@ struct ResumeCommand { #[arg(long = "last", default_value_t = false, conflicts_with = "session_id")] last: bool, + /// Show all sessions (disables cwd filtering and shows CWD column). + #[arg(long = "all", default_value_t = false)] + all: bool, + #[clap(flatten)] config_overrides: TuiCli, } @@ -448,6 +452,7 @@ async fn cli_main(codex_linux_sandbox_exe: Option) -> anyhow::Result<() Some(Subcommand::Resume(ResumeCommand { session_id, last, + all, config_overrides, })) => { interactive = finalize_resume_interactive( @@ -455,6 +460,7 @@ async fn cli_main(codex_linux_sandbox_exe: Option) -> anyhow::Result<() root_config_overrides.clone(), session_id, last, + all, config_overrides, ); let exit_info = codex_tui::run_main(interactive, codex_linux_sandbox_exe).await?; @@ -611,6 +617,7 @@ fn finalize_resume_interactive( root_config_overrides: CliConfigOverrides, session_id: Option, last: bool, + show_all: bool, resume_cli: TuiCli, ) -> TuiCli { // Start with the parsed interactive CLI so resume shares the same @@ -619,6 +626,7 @@ fn finalize_resume_interactive( interactive.resume_picker = resume_session_id.is_none() && !last; interactive.resume_last = last; interactive.resume_session_id = resume_session_id; + interactive.resume_show_all = show_all; // Merge resume-scoped flags and overrides with highest precedence. merge_resume_cli_flags(&mut interactive, resume_cli); @@ -702,13 +710,21 @@ mod tests { let Subcommand::Resume(ResumeCommand { session_id, last, + all, config_overrides: resume_cli, }) = subcommand.expect("resume present") else { unreachable!() }; - finalize_resume_interactive(interactive, root_overrides, session_id, last, resume_cli) + finalize_resume_interactive( + interactive, + root_overrides, + session_id, + last, + all, + resume_cli, + ) } fn sample_exit_info(conversation: Option<&str>) -> AppExitInfo { @@ -775,6 +791,7 @@ mod tests { assert!(interactive.resume_picker); assert!(!interactive.resume_last); assert_eq!(interactive.resume_session_id, None); + assert!(!interactive.resume_show_all); } #[test] @@ -783,6 +800,7 @@ mod tests { assert!(!interactive.resume_picker); assert!(interactive.resume_last); assert_eq!(interactive.resume_session_id, None); + assert!(!interactive.resume_show_all); } #[test] @@ -791,6 +809,14 @@ mod tests { assert!(!interactive.resume_picker); assert!(!interactive.resume_last); assert_eq!(interactive.resume_session_id.as_deref(), Some("1234")); + assert!(!interactive.resume_show_all); + } + + #[test] + fn resume_all_flag_sets_show_all() { + let interactive = finalize_from_args(["codex", "resume", "--all"].as_ref()); + assert!(interactive.resume_picker); + assert!(interactive.resume_show_all); } #[test] diff --git a/codex-rs/tui/src/cli.rs b/codex-rs/tui/src/cli.rs index e7a0c945bf..2b19b4c064 100644 --- a/codex-rs/tui/src/cli.rs +++ b/codex-rs/tui/src/cli.rs @@ -28,6 +28,10 @@ pub struct Cli { #[clap(skip)] pub resume_session_id: Option, + /// Internal: show all sessions (disables cwd filtering and shows CWD column). + #[clap(skip)] + pub resume_show_all: bool, + /// Model the agent should use. #[arg(long, short = 'm')] pub model: Option, diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index ea1dc7523a..33bd18c437 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -474,6 +474,7 @@ async fn run_ratatui_app( &mut tui, &config.codex_home, &config.model_provider_id, + cli.resume_show_all, ) .await? { diff --git a/codex-rs/tui/src/resume_picker.rs b/codex-rs/tui/src/resume_picker.rs index 83e3a1aef4..6a21496d34 100644 --- a/codex-rs/tui/src/resume_picker.rs +++ b/codex-rs/tui/src/resume_picker.rs @@ -26,12 +26,14 @@ use tokio_stream::StreamExt; use tokio_stream::wrappers::UnboundedReceiverStream; use unicode_width::UnicodeWidthStr; +use crate::diff_render::display_path_for; use crate::key_hint; use crate::text_formatting::truncate_text; use crate::tui::FrameRequester; use crate::tui::Tui; use crate::tui::TuiEvent; use codex_protocol::models::ResponseItem; +use codex_protocol::protocol::SessionMetaLine; const PAGE_SIZE: usize = 25; const LOAD_NEAR_THRESHOLD: usize = 5; @@ -69,11 +71,17 @@ pub async fn run_resume_picker( tui: &mut Tui, codex_home: &Path, default_provider: &str, + show_all: bool, ) -> Result { let alt = AltScreenGuard::enter(tui); let (bg_tx, bg_rx) = mpsc::unbounded_channel(); let default_provider = default_provider.to_string(); + let filter_cwd = if show_all { + None + } else { + std::env::current_dir().ok() + }; let loader_tx = bg_tx.clone(); let page_loader: PageLoader = Arc::new(move |request: PageLoadRequest| { @@ -102,6 +110,8 @@ pub async fn run_resume_picker( alt.tui.frame_requester(), page_loader, default_provider.clone(), + show_all, + filter_cwd, ); state.load_initial_page().await?; state.request_frame(); @@ -177,6 +187,8 @@ struct PickerState { page_loader: PageLoader, view_rows: Option, default_provider: String, + show_all: bool, + filter_cwd: Option, } struct PaginationState { @@ -234,6 +246,8 @@ struct Row { preview: String, created_at: Option>, updated_at: Option>, + cwd: Option, + git_branch: Option, } impl PickerState { @@ -242,6 +256,8 @@ impl PickerState { requester: FrameRequester, page_loader: PageLoader, default_provider: String, + show_all: bool, + filter_cwd: Option, ) -> Self { Self { codex_home, @@ -264,6 +280,8 @@ impl PickerState { page_loader, view_rows: None, default_provider, + show_all, + filter_cwd, } } @@ -418,13 +436,15 @@ impl PickerState { } fn apply_filter(&mut self) { + let base_iter = self + .all_rows + .iter() + .filter(|row| self.row_matches_filter(row)); if self.query.is_empty() { - self.filtered_rows = self.all_rows.clone(); + self.filtered_rows = base_iter.cloned().collect(); } else { let q = self.query.to_lowercase(); - self.filtered_rows = self - .all_rows - .iter() + self.filtered_rows = base_iter .filter(|r| r.preview.to_lowercase().contains(&q)) .cloned() .collect(); @@ -439,6 +459,19 @@ impl PickerState { self.request_frame(); } + fn row_matches_filter(&self, row: &Row) -> bool { + if self.show_all { + return true; + } + let Some(filter_cwd) = self.filter_cwd.as_ref() else { + return true; + }; + let Some(row_cwd) = row.cwd.as_ref() else { + return false; + }; + paths_match(row_cwd, filter_cwd) + } + fn set_query(&mut self, new_query: String) { if self.query == new_query { return; @@ -606,6 +639,7 @@ fn head_to_row(item: &ConversationItem) -> Row { .and_then(parse_timestamp_str) .or(created_at); + let (cwd, git_branch) = extract_session_meta_from_head(&item.head); let preview = preview_from_head(&item.head) .map(|s| s.trim().to_string()) .filter(|s| !s.is_empty()) @@ -616,9 +650,29 @@ fn head_to_row(item: &ConversationItem) -> Row { preview, created_at, updated_at, + cwd, + git_branch, } } +fn extract_session_meta_from_head(head: &[serde_json::Value]) -> (Option, Option) { + for value in head { + if let Ok(meta_line) = serde_json::from_value::(value.clone()) { + let cwd = Some(meta_line.meta.cwd); + let git_branch = meta_line.git.and_then(|git| git.branch); + return (cwd, git_branch); + } + } + (None, None) +} + +fn paths_match(a: &Path, b: &Path) -> bool { + if let (Ok(ca), Ok(cb)) = (a.canonicalize(), b.canonicalize()) { + return ca == cb; + } + a == b +} + fn parse_timestamp_str(ts: &str) -> Option> { chrono::DateTime::parse_from_rfc3339(ts) .map(|dt| dt.with_timezone(&Utc)) @@ -670,7 +724,7 @@ fn draw_picker(tui: &mut Tui, state: &PickerState) -> std::io::Result<()> { }; frame.render_widget_ref(Line::from(q), search); - let metrics = calculate_column_metrics(&state.filtered_rows); + let metrics = calculate_column_metrics(&state.filtered_rows, state.show_all); // Column headers and list render_column_headers(frame, columns, &metrics); @@ -720,10 +774,11 @@ fn render_list( let labels = &metrics.labels; let mut y = area.y; - let max_created_width = metrics.max_created_width; let max_updated_width = metrics.max_updated_width; + let max_branch_width = metrics.max_branch_width; + let max_cwd_width = metrics.max_cwd_width; - for (idx, (row, (created_label, updated_label))) in rows[start..end] + for (idx, (row, (updated_label, branch_label, cwd_label))) in rows[start..end] .iter() .zip(labels[start..end].iter()) .enumerate() @@ -731,38 +786,69 @@ fn render_list( let is_sel = start + idx == state.selected; let marker = if is_sel { "> ".bold() } else { " ".into() }; let marker_width = 2usize; - let created_span = if max_created_width == 0 { - None - } else { - Some(Span::from(format!("{created_label: 0 { - preview_width = preview_width.saturating_sub(max_created_width + 2); - } if max_updated_width > 0 { preview_width = preview_width.saturating_sub(max_updated_width + 2); } - let add_leading_gap = max_created_width == 0 && max_updated_width == 0; + if max_branch_width > 0 { + preview_width = preview_width.saturating_sub(max_branch_width + 2); + } + if max_cwd_width > 0 { + preview_width = preview_width.saturating_sub(max_cwd_width + 2); + } + let add_leading_gap = max_updated_width == 0 && max_branch_width == 0 && max_cwd_width == 0; if add_leading_gap { preview_width = preview_width.saturating_sub(2); } let preview = truncate_text(&row.preview, preview_width); let mut spans: Vec = vec![marker]; - if let Some(created) = created_span { - spans.push(created); - spans.push(" ".into()); - } if let Some(updated) = updated_span { spans.push(updated); spans.push(" ".into()); } + if let Some(branch) = branch_span { + spans.push(branch); + spans.push(" ".into()); + } + if let Some(cwd) = cwd_span { + spans.push(cwd); + spans.push(" ".into()); + } if add_leading_gap { spans.push(" ".into()); } @@ -844,12 +930,6 @@ fn human_time_ago(ts: DateTime) -> String { } } -fn format_created_label(row: &Row) -> String { - row.created_at - .map(human_time_ago) - .unwrap_or_else(|| "-".to_string()) -} - fn format_updated_label(row: &Row) -> String { match (row.updated_at, row.created_at) { (Some(updated), _) => human_time_ago(updated), @@ -868,15 +948,6 @@ fn render_column_headers( } let mut spans: Vec = vec![" ".into()]; - if metrics.max_created_width > 0 { - let label = format!( - "{text: 0 { let label = format!( "{text: 0 { + let label = format!( + "{text: 0 { + let label = format!( + "{text:, + max_branch_width: usize, + max_cwd_width: usize, + labels: Vec<(String, String, String)>, } -fn calculate_column_metrics(rows: &[Row]) -> ColumnMetrics { - let mut labels: Vec<(String, String)> = Vec::with_capacity(rows.len()); - let mut max_created_width = UnicodeWidthStr::width("Created"); +fn calculate_column_metrics(rows: &[Row], include_cwd: bool) -> ColumnMetrics { + fn right_elide(s: &str, max: usize) -> String { + if s.chars().count() <= max { + return s.to_string(); + } + if max <= 1 { + return "…".to_string(); + } + let tail_len = max - 1; + let tail: String = s + .chars() + .rev() + .take(tail_len) + .collect::() + .chars() + .rev() + .collect(); + format!("…{tail}") + } + + let mut labels: Vec<(String, String, String)> = Vec::with_capacity(rows.len()); let mut max_updated_width = UnicodeWidthStr::width("Updated"); + let mut max_branch_width = UnicodeWidthStr::width("Branch"); + let mut max_cwd_width = if include_cwd { + UnicodeWidthStr::width("CWD") + } else { + 0 + }; for row in rows { - let created = format_created_label(row); let updated = format_updated_label(row); - max_created_width = max_created_width.max(UnicodeWidthStr::width(created.as_str())); + let branch_raw = row.git_branch.clone().unwrap_or_default(); + let branch = right_elide(&branch_raw, 24); + let cwd = if include_cwd { + let cwd_raw = row + .cwd + .as_ref() + .map(|p| display_path_for(p, std::path::Path::new("/"))) + .unwrap_or_default(); + right_elide(&cwd_raw, 24) + } else { + String::new() + }; max_updated_width = max_updated_width.max(UnicodeWidthStr::width(updated.as_str())); - labels.push((created, updated)); + max_branch_width = max_branch_width.max(UnicodeWidthStr::width(branch.as_str())); + max_cwd_width = max_cwd_width.max(UnicodeWidthStr::width(cwd.as_str())); + labels.push((updated, branch, cwd)); } ColumnMetrics { - max_created_width, max_updated_width, + max_branch_width, + max_cwd_width, labels, } } @@ -1088,6 +1215,8 @@ mod tests { FrameRequester::test_dummy(), loader, String::from("openai"), + true, + None, ); let now = Utc::now(); @@ -1097,18 +1226,24 @@ mod tests { preview: String::from("Fix resume picker timestamps"), created_at: Some(now - Duration::minutes(16)), updated_at: Some(now - Duration::seconds(42)), + cwd: None, + git_branch: None, }, Row { path: PathBuf::from("/tmp/b.jsonl"), preview: String::from("Investigate lazy pagination cap"), created_at: Some(now - Duration::hours(1)), updated_at: Some(now - Duration::minutes(35)), + cwd: None, + git_branch: None, }, Row { path: PathBuf::from("/tmp/c.jsonl"), preview: String::from("Explain the codebase"), created_at: Some(now - Duration::hours(2)), updated_at: Some(now - Duration::hours(2)), + cwd: None, + git_branch: None, }, ]; state.all_rows = rows.clone(); @@ -1118,7 +1253,7 @@ mod tests { state.scroll_top = 0; state.update_view_rows(3); - let metrics = calculate_column_metrics(&state.filtered_rows); + let metrics = calculate_column_metrics(&state.filtered_rows, state.show_all); let width: u16 = 80; let height: u16 = 6; @@ -1148,6 +1283,8 @@ mod tests { FrameRequester::test_dummy(), loader, String::from("openai"), + true, + None, ); state.reset_pagination(); @@ -1214,6 +1351,8 @@ mod tests { FrameRequester::test_dummy(), loader, String::from("openai"), + true, + None, ); state.reset_pagination(); state.ingest_page(page( @@ -1243,6 +1382,8 @@ mod tests { FrameRequester::test_dummy(), loader, String::from("openai"), + true, + None, ); let mut items = Vec::new(); @@ -1291,6 +1432,8 @@ mod tests { FrameRequester::test_dummy(), loader, String::from("openai"), + true, + None, ); let mut items = Vec::new(); @@ -1335,6 +1478,8 @@ mod tests { FrameRequester::test_dummy(), loader, String::from("openai"), + true, + None, ); state.reset_pagination(); state.ingest_page(page( diff --git a/codex-rs/tui/src/snapshots/codex_tui__resume_picker__tests__resume_picker_table.snap b/codex-rs/tui/src/snapshots/codex_tui__resume_picker__tests__resume_picker_table.snap index 0883651432..c029043219 100644 --- a/codex-rs/tui/src/snapshots/codex_tui__resume_picker__tests__resume_picker_table.snap +++ b/codex-rs/tui/src/snapshots/codex_tui__resume_picker__tests__resume_picker_table.snap @@ -2,7 +2,7 @@ source: tui/src/resume_picker.rs expression: snapshot --- - Created Updated Conversation - 16 minutes ago 42 seconds ago Fix resume picker timestamps -> 1 hour ago 35 minutes ago Investigate lazy pagination cap - 2 hours ago 2 hours ago Explain the codebase + Updated Branch CWD Conversation + 42 seconds ago - - Fix resume picker timestamps +> 35 minutes ago - - Investigate lazy pagination cap + 2 hours ago - - Explain the codebase diff --git a/docs/getting-started.md b/docs/getting-started.md index 4930061c27..83c66c5724 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -21,6 +21,7 @@ Key flags: `--model/-m`, `--ask-for-approval/-a`. - Run `codex resume` to display the session picker UI - Resume most recent: `codex resume --last` - Resume by id: `codex resume ` (You can get session ids from /status or `~/.codex/sessions/`) +- The picker shows the session's original working directory and, when available, the Git branch it was recorded on Examples: From 4288091f63cdffc21cfeaea4290c734056188808 Mon Sep 17 00:00:00 2001 From: zhao-oai Date: Tue, 18 Nov 2025 17:40:22 -0800 Subject: [PATCH 16/18] update credit status details (#6862) --- .../src/models/credit_status_details.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/codex-rs/codex-backend-openapi-models/src/models/credit_status_details.rs b/codex-rs/codex-backend-openapi-models/src/models/credit_status_details.rs index d444a7a8d2..b62b88d715 100644 --- a/codex-rs/codex-backend-openapi-models/src/models/credit_status_details.rs +++ b/codex-rs/codex-backend-openapi-models/src/models/credit_status_details.rs @@ -12,6 +12,8 @@ use serde::Serialize; #[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)] pub struct CreditStatusDetails { + #[serde(rename = "has_credits")] + pub has_credits: bool, #[serde(rename = "unlimited")] pub unlimited: bool, #[serde( @@ -38,8 +40,9 @@ pub struct CreditStatusDetails { } impl CreditStatusDetails { - pub fn new(unlimited: bool) -> CreditStatusDetails { + pub fn new(has_credits: bool, unlimited: bool) -> CreditStatusDetails { CreditStatusDetails { + has_credits, unlimited, balance: None, approx_local_messages: None, From b395dc1be6d00cfb855279f39d6df215611e411b Mon Sep 17 00:00:00 2001 From: Celia Chen Date: Tue, 18 Nov 2025 17:55:24 -0800 Subject: [PATCH 17/18] [app-server] introduce `turn/completed` v2 event (#6800) similar to logic in `codex/codex-rs/exec/src/event_processor_with_jsonl_output.rs`. translation of v1 -> v2 events: `codex/event/task_complete` -> `turn/completed` `codex/event/turn_aborted` -> `turn/completed` with `interrupted` status `codex/event/error` -> `turn/completed` with `error` status this PR also makes `items` field in `Turn` optional. For now, we only populate it when we resume a thread, and leave it as None for all other places until we properly rewrite core to keep track of items. tested using the codex app server client. example new event: ``` < { < "method": "turn/completed", < "params": { < "turn": { < "id": "0", < "items": [], < "status": "interrupted" < } < } < } ``` --- .../app-server-protocol/src/protocol/v2.rs | 12 +- codex-rs/app-server-test-client/src/main.rs | 4 +- .../app-server/src/bespoke_event_handling.rs | 306 ++++++++++++++++++ .../app-server/src/codex_message_processor.rs | 22 +- .../tests/suite/v2/turn_interrupt.rs | 16 +- .../app-server/tests/suite/v2/turn_start.rs | 24 +- 6 files changed, 362 insertions(+), 22 deletions(-) diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index f4a6d636aa..05e0e3560e 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -519,9 +519,11 @@ pub struct AccountUpdatedNotification { #[ts(export_to = "v2/")] pub struct Turn { pub id: String, + /// This is currently only populated for resumed threads. + /// TODO: properly populate items for all turns. pub items: Vec, + #[serde(flatten)] pub status: TurnStatus, - pub error: Option, } #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] @@ -532,12 +534,12 @@ pub struct TurnError { } #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] -#[serde(rename_all = "camelCase")] -#[ts(export_to = "v2/")] +#[serde(tag = "status", rename_all = "camelCase")] +#[ts(tag = "status", export_to = "v2/")] pub enum TurnStatus { Completed, Interrupted, - Failed, + Failed { error: TurnError }, InProgress, } @@ -853,8 +855,6 @@ pub struct Usage { #[ts(export_to = "v2/")] pub struct TurnCompletedNotification { pub turn: Turn, - // TODO: should usage be stored on the Turn object, and we return that instead? - pub usage: Usage, } #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] diff --git a/codex-rs/app-server-test-client/src/main.rs b/codex-rs/app-server-test-client/src/main.rs index 8513ec0321..6130b2d511 100644 --- a/codex-rs/app-server-test-client/src/main.rs +++ b/codex-rs/app-server-test-client/src/main.rs @@ -46,6 +46,7 @@ use codex_app_server_protocol::ThreadStartParams; use codex_app_server_protocol::ThreadStartResponse; use codex_app_server_protocol::TurnStartParams; use codex_app_server_protocol::TurnStartResponse; +use codex_app_server_protocol::TurnStatus; use codex_app_server_protocol::UserInput as V2UserInput; use codex_protocol::ConversationId; use codex_protocol::protocol::Event; @@ -509,10 +510,9 @@ impl CodexClient { ServerNotification::TurnCompleted(payload) => { if payload.turn.id == turn_id { println!("\n< turn/completed notification: {:?}", payload.turn.status); - if let Some(error) = payload.turn.error { + if let TurnStatus::Failed { error } = &payload.turn.status { println!("[turn error] {}", error.message); } - println!("< usage: {:?}", payload.usage); break; } } diff --git a/codex-rs/app-server/src/bespoke_event_handling.rs b/codex-rs/app-server/src/bespoke_event_handling.rs index ce4f7590e2..b4fdbe3e03 100644 --- a/codex-rs/app-server/src/bespoke_event_handling.rs +++ b/codex-rs/app-server/src/bespoke_event_handling.rs @@ -1,5 +1,7 @@ use crate::codex_message_processor::ApiVersion; use crate::codex_message_processor::PendingInterrupts; +use crate::codex_message_processor::TurnSummary; +use crate::codex_message_processor::TurnSummaryStore; use crate::outgoing_message::OutgoingMessageSender; use codex_app_server_protocol::AccountRateLimitsUpdatedNotification; use codex_app_server_protocol::AgentMessageDeltaNotification; @@ -26,7 +28,11 @@ use codex_app_server_protocol::SandboxCommandAssessment as V2SandboxCommandAsses use codex_app_server_protocol::ServerNotification; use codex_app_server_protocol::ServerRequestPayload; use codex_app_server_protocol::ThreadItem; +use codex_app_server_protocol::Turn; +use codex_app_server_protocol::TurnCompletedNotification; +use codex_app_server_protocol::TurnError; use codex_app_server_protocol::TurnInterruptResponse; +use codex_app_server_protocol::TurnStatus; use codex_core::CodexConversation; use codex_core::parse_command::shlex_join; use codex_core::protocol::ApplyPatchApprovalRequestEvent; @@ -54,10 +60,14 @@ pub(crate) async fn apply_bespoke_event_handling( conversation: Arc, outgoing: Arc, pending_interrupts: PendingInterrupts, + turn_summary_store: TurnSummaryStore, api_version: ApiVersion, ) { let Event { id: event_id, msg } = event; match msg { + EventMsg::TaskComplete(_ev) => { + handle_turn_complete(conversation_id, event_id, &outgoing, &turn_summary_store).await; + } EventMsg::ApplyPatchApprovalRequest(ApplyPatchApprovalRequestEvent { call_id, changes, @@ -191,6 +201,9 @@ pub(crate) async fn apply_bespoke_event_handling( .await; } } + EventMsg::Error(ev) => { + handle_error(conversation_id, ev.message, &turn_summary_store).await; + } EventMsg::EnteredReviewMode(review_request) => { let notification = ItemStartedNotification { item: ThreadItem::CodeReview { @@ -326,12 +339,79 @@ pub(crate) async fn apply_bespoke_event_handling( } } } + + handle_turn_interrupted(conversation_id, event_id, &outgoing, &turn_summary_store) + .await; } _ => {} } } +async fn emit_turn_completed_with_status( + event_id: String, + status: TurnStatus, + outgoing: &OutgoingMessageSender, +) { + let notification = TurnCompletedNotification { + turn: Turn { + id: event_id, + items: vec![], + status, + }, + }; + outgoing + .send_server_notification(ServerNotification::TurnCompleted(notification)) + .await; +} + +async fn find_and_remove_turn_summary( + conversation_id: ConversationId, + turn_summary_store: &TurnSummaryStore, +) -> TurnSummary { + let mut map = turn_summary_store.lock().await; + map.remove(&conversation_id).unwrap_or_default() +} + +async fn handle_turn_complete( + conversation_id: ConversationId, + event_id: String, + outgoing: &OutgoingMessageSender, + turn_summary_store: &TurnSummaryStore, +) { + let turn_summary = find_and_remove_turn_summary(conversation_id, turn_summary_store).await; + + let status = if let Some(message) = turn_summary.last_error_message { + TurnStatus::Failed { + error: TurnError { message }, + } + } else { + TurnStatus::Completed + }; + + emit_turn_completed_with_status(event_id, status, outgoing).await; +} + +async fn handle_turn_interrupted( + conversation_id: ConversationId, + event_id: String, + outgoing: &OutgoingMessageSender, + turn_summary_store: &TurnSummaryStore, +) { + find_and_remove_turn_summary(conversation_id, turn_summary_store).await; + + emit_turn_completed_with_status(event_id, TurnStatus::Interrupted, outgoing).await; +} + +async fn handle_error( + conversation_id: ConversationId, + message: String, + turn_summary_store: &TurnSummaryStore, +) { + let mut map = turn_summary_store.lock().await; + map.entry(conversation_id).or_default().last_error_message = Some(message); +} + async fn on_patch_approval_response( event_id: String, receiver: oneshot::Receiver, @@ -536,13 +616,140 @@ async fn construct_mcp_tool_call_end_notification( #[cfg(test)] mod tests { use super::*; + use crate::CHANNEL_CAPACITY; + use crate::outgoing_message::OutgoingMessage; + use crate::outgoing_message::OutgoingMessageSender; + use anyhow::Result; + use anyhow::anyhow; + use anyhow::bail; use codex_core::protocol::McpInvocation; use mcp_types::CallToolResult; use mcp_types::ContentBlock; use mcp_types::TextContent; use pretty_assertions::assert_eq; use serde_json::Value as JsonValue; + use std::collections::HashMap; use std::time::Duration; + use tokio::sync::Mutex; + use tokio::sync::mpsc; + + fn new_turn_summary_store() -> TurnSummaryStore { + Arc::new(Mutex::new(HashMap::new())) + } + + #[tokio::test] + async fn test_handle_error_records_message() -> Result<()> { + let conversation_id = ConversationId::new(); + let turn_summary_store = new_turn_summary_store(); + + handle_error(conversation_id, "boom".to_string(), &turn_summary_store).await; + + let turn_summary = find_and_remove_turn_summary(conversation_id, &turn_summary_store).await; + assert_eq!(turn_summary.last_error_message, Some("boom".to_string())); + Ok(()) + } + + #[tokio::test] + async fn test_handle_turn_complete_emits_completed_without_error() -> Result<()> { + let conversation_id = ConversationId::new(); + let event_id = "complete1".to_string(); + let (tx, mut rx) = mpsc::channel(CHANNEL_CAPACITY); + let outgoing = Arc::new(OutgoingMessageSender::new(tx)); + let turn_summary_store = new_turn_summary_store(); + + handle_turn_complete( + conversation_id, + event_id.clone(), + &outgoing, + &turn_summary_store, + ) + .await; + + let msg = rx + .recv() + .await + .ok_or_else(|| anyhow!("should send one notification"))?; + match msg { + OutgoingMessage::AppServerNotification(ServerNotification::TurnCompleted(n)) => { + assert_eq!(n.turn.id, event_id); + assert_eq!(n.turn.status, TurnStatus::Completed); + } + other => bail!("unexpected message: {other:?}"), + } + assert!(rx.try_recv().is_err(), "no extra messages expected"); + Ok(()) + } + + #[tokio::test] + async fn test_handle_turn_interrupted_emits_interrupted_with_error() -> Result<()> { + let conversation_id = ConversationId::new(); + let event_id = "interrupt1".to_string(); + let turn_summary_store = new_turn_summary_store(); + handle_error(conversation_id, "oops".to_string(), &turn_summary_store).await; + let (tx, mut rx) = mpsc::channel(CHANNEL_CAPACITY); + let outgoing = Arc::new(OutgoingMessageSender::new(tx)); + + handle_turn_interrupted( + conversation_id, + event_id.clone(), + &outgoing, + &turn_summary_store, + ) + .await; + + let msg = rx + .recv() + .await + .ok_or_else(|| anyhow!("should send one notification"))?; + match msg { + OutgoingMessage::AppServerNotification(ServerNotification::TurnCompleted(n)) => { + assert_eq!(n.turn.id, event_id); + assert_eq!(n.turn.status, TurnStatus::Interrupted); + } + other => bail!("unexpected message: {other:?}"), + } + assert!(rx.try_recv().is_err(), "no extra messages expected"); + Ok(()) + } + + #[tokio::test] + async fn test_handle_turn_complete_emits_failed_with_error() -> Result<()> { + let conversation_id = ConversationId::new(); + let event_id = "complete_err1".to_string(); + let turn_summary_store = new_turn_summary_store(); + handle_error(conversation_id, "bad".to_string(), &turn_summary_store).await; + let (tx, mut rx) = mpsc::channel(CHANNEL_CAPACITY); + let outgoing = Arc::new(OutgoingMessageSender::new(tx)); + + handle_turn_complete( + conversation_id, + event_id.clone(), + &outgoing, + &turn_summary_store, + ) + .await; + + let msg = rx + .recv() + .await + .ok_or_else(|| anyhow!("should send one notification"))?; + match msg { + OutgoingMessage::AppServerNotification(ServerNotification::TurnCompleted(n)) => { + assert_eq!(n.turn.id, event_id); + assert_eq!( + n.turn.status, + TurnStatus::Failed { + error: TurnError { + message: "bad".to_string(), + } + } + ); + } + other => bail!("unexpected message: {other:?}"), + } + assert!(rx.try_recv().is_err(), "no extra messages expected"); + Ok(()) + } #[tokio::test] async fn test_construct_mcp_tool_call_begin_notification_with_args() { @@ -572,6 +779,105 @@ mod tests { assert_eq!(notification, expected); } + #[tokio::test] + async fn test_handle_turn_complete_emits_error_multiple_turns() -> Result<()> { + // Conversation A will have two turns; Conversation B will have one turn. + let conversation_a = ConversationId::new(); + let conversation_b = ConversationId::new(); + let turn_summary_store = new_turn_summary_store(); + + let (tx, mut rx) = mpsc::channel(CHANNEL_CAPACITY); + let outgoing = Arc::new(OutgoingMessageSender::new(tx)); + + // Turn 1 on conversation A + let a_turn1 = "a_turn1".to_string(); + handle_error(conversation_a, "a1".to_string(), &turn_summary_store).await; + handle_turn_complete( + conversation_a, + a_turn1.clone(), + &outgoing, + &turn_summary_store, + ) + .await; + + // Turn 1 on conversation B + let b_turn1 = "b_turn1".to_string(); + handle_error(conversation_b, "b1".to_string(), &turn_summary_store).await; + handle_turn_complete( + conversation_b, + b_turn1.clone(), + &outgoing, + &turn_summary_store, + ) + .await; + + // Turn 2 on conversation A + let a_turn2 = "a_turn2".to_string(); + handle_turn_complete( + conversation_a, + a_turn2.clone(), + &outgoing, + &turn_summary_store, + ) + .await; + + // Verify: A turn 1 + let msg = rx + .recv() + .await + .ok_or_else(|| anyhow!("should send first notification"))?; + match msg { + OutgoingMessage::AppServerNotification(ServerNotification::TurnCompleted(n)) => { + assert_eq!(n.turn.id, a_turn1); + assert_eq!( + n.turn.status, + TurnStatus::Failed { + error: TurnError { + message: "a1".to_string(), + } + } + ); + } + other => bail!("unexpected message: {other:?}"), + } + + // Verify: B turn 1 + let msg = rx + .recv() + .await + .ok_or_else(|| anyhow!("should send second notification"))?; + match msg { + OutgoingMessage::AppServerNotification(ServerNotification::TurnCompleted(n)) => { + assert_eq!(n.turn.id, b_turn1); + assert_eq!( + n.turn.status, + TurnStatus::Failed { + error: TurnError { + message: "b1".to_string(), + } + } + ); + } + other => bail!("unexpected message: {other:?}"), + } + + // Verify: A turn 2 + let msg = rx + .recv() + .await + .ok_or_else(|| anyhow!("should send third notification"))?; + match msg { + OutgoingMessage::AppServerNotification(ServerNotification::TurnCompleted(n)) => { + assert_eq!(n.turn.id, a_turn2); + assert_eq!(n.turn.status, TurnStatus::Completed); + } + other => bail!("unexpected message: {other:?}"), + } + + assert!(rx.try_recv().is_err(), "no extra messages expected"); + Ok(()) + } + #[tokio::test] async fn test_construct_mcp_tool_call_begin_notification_without_args() { let begin_event = McpToolCallBeginEvent { diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index 0172458074..6191fdf4a7 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -154,6 +154,14 @@ use uuid::Uuid; type PendingInterruptQueue = Vec<(RequestId, ApiVersion)>; pub(crate) type PendingInterrupts = Arc>>; +/// Per-conversation accumulation of the latest states e.g. error message while a turn runs. +#[derive(Default, Clone)] +pub(crate) struct TurnSummary { + pub(crate) last_error_message: Option, +} + +pub(crate) type TurnSummaryStore = Arc>>; + // Duration before a ChatGPT login attempt is abandoned. const LOGIN_CHATGPT_TIMEOUT: Duration = Duration::from_secs(10 * 60); struct ActiveLogin { @@ -178,6 +186,7 @@ pub(crate) struct CodexMessageProcessor { active_login: Arc>>, // Queue of pending interrupt requests per conversation. We reply when TurnAborted arrives. pending_interrupts: PendingInterrupts, + turn_summary_store: TurnSummaryStore, pending_fuzzy_searches: Arc>>>, feedback: CodexFeedback, } @@ -230,6 +239,7 @@ impl CodexMessageProcessor { conversation_listeners: HashMap::new(), active_login: Arc::new(Mutex::new(None)), pending_interrupts: Arc::new(Mutex::new(HashMap::new())), + turn_summary_store: Arc::new(Mutex::new(HashMap::new())), pending_fuzzy_searches: Arc::new(Mutex::new(HashMap::new())), feedback, } @@ -2363,9 +2373,6 @@ impl CodexMessageProcessor { } }; - // Keep a copy of v2 inputs for the notification payload. - let v2_inputs_for_notif = params.input.clone(); - // Map v2 input items to core input items. let mapped_items: Vec = params .input @@ -2405,12 +2412,8 @@ impl CodexMessageProcessor { Ok(turn_id) => { let turn = Turn { id: turn_id.clone(), - items: vec![ThreadItem::UserMessage { - id: turn_id, - content: v2_inputs_for_notif, - }], + items: vec![], status: TurnStatus::InProgress, - error: None, }; let response = TurnStartResponse { turn: turn.clone() }; @@ -2471,7 +2474,6 @@ impl CodexMessageProcessor { id: turn_id.clone(), items, status: TurnStatus::InProgress, - error: None, }; let response = TurnStartResponse { turn: turn.clone() }; self.outgoing.send_response(request_id, response).await; @@ -2591,6 +2593,7 @@ impl CodexMessageProcessor { let outgoing_for_task = self.outgoing.clone(); let pending_interrupts = self.pending_interrupts.clone(); + let turn_summary_store = self.turn_summary_store.clone(); let api_version_for_task = api_version; tokio::spawn(async move { loop { @@ -2647,6 +2650,7 @@ impl CodexMessageProcessor { conversation.clone(), outgoing_for_task.clone(), pending_interrupts.clone(), + turn_summary_store.clone(), api_version_for_task, ) .await; diff --git a/codex-rs/app-server/tests/suite/v2/turn_interrupt.rs b/codex-rs/app-server/tests/suite/v2/turn_interrupt.rs index d1deb60801..aa140c9e7f 100644 --- a/codex-rs/app-server/tests/suite/v2/turn_interrupt.rs +++ b/codex-rs/app-server/tests/suite/v2/turn_interrupt.rs @@ -5,14 +5,17 @@ use app_test_support::McpProcess; use app_test_support::create_mock_chat_completions_server; use app_test_support::create_shell_sse_response; use app_test_support::to_response; +use codex_app_server_protocol::JSONRPCNotification; use codex_app_server_protocol::JSONRPCResponse; use codex_app_server_protocol::RequestId; use codex_app_server_protocol::ThreadStartParams; use codex_app_server_protocol::ThreadStartResponse; +use codex_app_server_protocol::TurnCompletedNotification; use codex_app_server_protocol::TurnInterruptParams; use codex_app_server_protocol::TurnInterruptResponse; use codex_app_server_protocol::TurnStartParams; use codex_app_server_protocol::TurnStartResponse; +use codex_app_server_protocol::TurnStatus; use codex_app_server_protocol::UserInput as V2UserInput; use tempfile::TempDir; use tokio::time::timeout; @@ -99,7 +102,18 @@ async fn turn_interrupt_aborts_running_turn() -> Result<()> { .await??; let _resp: TurnInterruptResponse = to_response::(interrupt_resp)?; - // No fields to assert on; successful deserialization confirms proper response shape. + let completed_notif: JSONRPCNotification = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("turn/completed"), + ) + .await??; + let completed: TurnCompletedNotification = serde_json::from_value( + completed_notif + .params + .expect("turn/completed params must be present"), + )?; + assert_eq!(completed.turn.status, TurnStatus::Interrupted); + Ok(()) } diff --git a/codex-rs/app-server/tests/suite/v2/turn_start.rs b/codex-rs/app-server/tests/suite/v2/turn_start.rs index 433c7b4486..753e33c243 100644 --- a/codex-rs/app-server/tests/suite/v2/turn_start.rs +++ b/codex-rs/app-server/tests/suite/v2/turn_start.rs @@ -14,9 +14,11 @@ use codex_app_server_protocol::ServerRequest; use codex_app_server_protocol::ThreadItem; use codex_app_server_protocol::ThreadStartParams; use codex_app_server_protocol::ThreadStartResponse; +use codex_app_server_protocol::TurnCompletedNotification; use codex_app_server_protocol::TurnStartParams; use codex_app_server_protocol::TurnStartResponse; use codex_app_server_protocol::TurnStartedNotification; +use codex_app_server_protocol::TurnStatus; use codex_app_server_protocol::UserInput as V2UserInput; use codex_core::protocol_config_types::ReasoningEffort; use codex_core::protocol_config_types::ReasoningSummary; @@ -118,13 +120,17 @@ async fn turn_start_emits_notifications_and_accepts_model_override() -> Result<( ) .await??; - // And we should ultimately get a task_complete without having to add a - // legacy conversation listener explicitly (auto-attached by thread/start). - let _task_complete: JSONRPCNotification = timeout( + let completed_notif: JSONRPCNotification = timeout( DEFAULT_READ_TIMEOUT, - mcp.read_stream_until_notification_message("codex/event/task_complete"), + mcp.read_stream_until_notification_message("turn/completed"), ) .await??; + let completed: TurnCompletedNotification = serde_json::from_value( + completed_notif + .params + .expect("turn/completed params must be present"), + )?; + assert_eq!(completed.turn.status, TurnStatus::Completed); Ok(()) } @@ -274,6 +280,11 @@ async fn turn_start_exec_approval_toggle_v2() -> Result<()> { mcp.read_stream_until_notification_message("codex/event/task_complete"), ) .await??; + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("turn/completed"), + ) + .await??; // Second turn with approval_policy=never should not elicit approval let second_turn_id = mcp @@ -302,6 +313,11 @@ async fn turn_start_exec_approval_toggle_v2() -> Result<()> { mcp.read_stream_until_notification_message("codex/event/task_complete"), ) .await??; + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("turn/completed"), + ) + .await??; Ok(()) } From cac0a6a29d1060877c588ba266835c06f4c45e3c Mon Sep 17 00:00:00 2001 From: pakrym-oai Date: Tue, 18 Nov 2025 18:21:57 -0800 Subject: [PATCH 18/18] Remote compaction on by-default (#6866) --- codex-rs/core/src/features.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codex-rs/core/src/features.rs b/codex-rs/core/src/features.rs index 5f579defd4..a3457e363c 100644 --- a/codex-rs/core/src/features.rs +++ b/codex-rs/core/src/features.rs @@ -313,7 +313,7 @@ pub const FEATURES: &[FeatureSpec] = &[ id: Feature::RemoteCompaction, key: "remote_compaction", stage: Stage::Experimental, - default_enabled: false, + default_enabled: true, }, FeatureSpec { id: Feature::ParallelToolCalls,