From c2ec477d939af2a275255666b37fe9b0acd7492a Mon Sep 17 00:00:00 2001 From: Celia Chen Date: Wed, 19 Nov 2025 11:51:21 -0800 Subject: [PATCH 01/42] [core] add optional status_code to error events (#6865) We want to better uncover error status code for clients. Add an optional status_code to error events (thread error, error, stream error) so app server could uncover the status code from the client side later. in event log: ``` < { < "method": "codex/event/stream_error", < "params": { < "conversationId": "019a9a32-f576-7292-9711-8e57e8063536", < "id": "0", < "msg": { < "message": "Reconnecting... 5/5", < "status_code": 401, < "type": "stream_error" < } < } < } < { < "method": "codex/event/error", < "params": { < "conversationId": "019a9a32-f576-7292-9711-8e57e8063536", < "id": "0", < "msg": { < "message": "exceeded retry limit, last status: 401 Unauthorized, request id: 9a0cb03a485067f7-SJC", < "status_code": 401, < "type": "error" < } < } < } ``` --- codex-rs/core/src/codex.rs | 13 ++-- codex-rs/core/src/compact.rs | 14 ++-- codex-rs/core/src/compact_remote.rs | 7 +- codex-rs/core/src/error.rs | 71 +++++++++++++++++++ codex-rs/docs/protocol_v1.md | 2 +- .../src/event_processor_with_human_output.rs | 4 +- .../tests/event_processor_with_json_output.rs | 3 + codex-rs/protocol/src/protocol.rs | 4 ++ codex-rs/tui/src/chatwidget.rs | 6 +- codex-rs/tui/src/chatwidget/tests.rs | 1 + 10 files changed, 101 insertions(+), 24 deletions(-) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index e5b4e4c316..32589dabb3 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -66,6 +66,7 @@ use crate::context_manager::ContextManager; use crate::environment_context::EnvironmentContext; use crate::error::CodexErr; use crate::error::Result as CodexResult; +use crate::error::http_status_code_value; #[cfg(test)] use crate::exec::StreamOutput; use crate::mcp::auth::compute_auth_statuses; @@ -79,7 +80,6 @@ use crate::protocol::ApplyPatchApprovalRequestEvent; use crate::protocol::AskForApproval; use crate::protocol::BackgroundEventEvent; use crate::protocol::DeprecationNoticeEvent; -use crate::protocol::ErrorEvent; use crate::protocol::Event; use crate::protocol::EventMsg; use crate::protocol::ExecApprovalRequestEvent; @@ -133,6 +133,7 @@ use codex_protocol::user_input::UserInput; use codex_utils_readiness::Readiness; use codex_utils_readiness::ReadinessFlag; use codex_utils_tokenizer::warm_model_cache; +use reqwest::StatusCode; /// The high-level interface to the Codex system. /// It operates as a queue pair where you send submissions and receive events. @@ -1186,9 +1187,11 @@ impl Session { &self, turn_context: &TurnContext, message: impl Into, + http_status_code: Option, ) { let event = EventMsg::StreamError(StreamErrorEvent { message: message.into(), + http_status_code: http_status_code_value(http_status_code), }); self.send_event(turn_context, event).await; } @@ -1680,6 +1683,7 @@ mod handlers { id: sub_id.clone(), msg: EventMsg::Error(ErrorEvent { message: "Failed to shutdown rollout recorder".to_string(), + http_status_code: None, }), }; sess.send_event_raw(event).await; @@ -1933,10 +1937,8 @@ pub(crate) async fn run_task( } Err(e) => { info!("Turn error: {e:#}"); - let event = EventMsg::Error(ErrorEvent { - message: e.to_string(), - }); - sess.send_event(&turn_context, event).await; + sess.send_event(&turn_context, EventMsg::Error(e.to_error_event(None))) + .await; // let the user continue the conversation break; } @@ -2060,6 +2062,7 @@ async fn run_turn( sess.notify_stream_error( &turn_context, format!("Reconnecting... {retries}/{max_retries}"), + e.http_status_code(), ) .await; diff --git a/codex-rs/core/src/compact.rs b/codex-rs/core/src/compact.rs index 8c38f93937..67b5b9342d 100644 --- a/codex-rs/core/src/compact.rs +++ b/codex-rs/core/src/compact.rs @@ -10,7 +10,6 @@ use crate::error::Result as CodexResult; use crate::features::Feature; use crate::protocol::AgentMessageEvent; use crate::protocol::CompactedItem; -use crate::protocol::ErrorEvent; use crate::protocol::EventMsg; use crate::protocol::TaskStartedEvent; use crate::protocol::TurnContextItem; @@ -128,10 +127,8 @@ async fn run_compact_task_inner( continue; } sess.set_total_tokens_full(turn_context.as_ref()).await; - let event = EventMsg::Error(ErrorEvent { - message: e.to_string(), - }); - sess.send_event(&turn_context, event).await; + sess.send_event(&turn_context, EventMsg::Error(e.to_error_event(None))) + .await; return; } Err(e) => { @@ -141,15 +138,14 @@ async fn run_compact_task_inner( sess.notify_stream_error( turn_context.as_ref(), format!("Reconnecting... {retries}/{max_retries}"), + e.http_status_code(), ) .await; tokio::time::sleep(delay).await; continue; } else { - let event = EventMsg::Error(ErrorEvent { - message: e.to_string(), - }); - sess.send_event(&turn_context, event).await; + sess.send_event(&turn_context, EventMsg::Error(e.to_error_event(None))) + .await; return; } } diff --git a/codex-rs/core/src/compact_remote.rs b/codex-rs/core/src/compact_remote.rs index 0d2e0f138d..aa32942dce 100644 --- a/codex-rs/core/src/compact_remote.rs +++ b/codex-rs/core/src/compact_remote.rs @@ -6,7 +6,6 @@ use crate::codex::TurnContext; use crate::error::Result as CodexResult; use crate::protocol::AgentMessageEvent; use crate::protocol::CompactedItem; -use crate::protocol::ErrorEvent; use crate::protocol::EventMsg; use crate::protocol::RolloutItem; use crate::protocol::TaskStartedEvent; @@ -30,10 +29,8 @@ pub(crate) async fn run_remote_compact_task(sess: Arc, turn_context: Ar async fn run_remote_compact_task_inner(sess: &Arc, turn_context: &Arc) { if let Err(err) = run_remote_compact_task_inner_impl(sess, turn_context).await { - let event = EventMsg::Error(ErrorEvent { - message: format!("Error running remote compact task: {err}"), - }); - sess.send_event(turn_context, event).await; + let event = err.to_error_event(Some("Error running remote compact task".to_string())); + sess.send_event(turn_context, EventMsg::Error(event)).await; } } diff --git a/codex-rs/core/src/error.rs b/codex-rs/core/src/error.rs index b2027dc940..d20d5cfb46 100644 --- a/codex-rs/core/src/error.rs +++ b/codex-rs/core/src/error.rs @@ -10,6 +10,7 @@ use chrono::Local; use chrono::Utc; use codex_async_utils::CancelErr; use codex_protocol::ConversationId; +use codex_protocol::protocol::ErrorEvent; use codex_protocol::protocol::RateLimitSnapshot; use reqwest::StatusCode; use serde_json; @@ -430,6 +431,37 @@ impl CodexErr { pub fn downcast_ref(&self) -> Option<&T> { (self as &dyn std::any::Any).downcast_ref::() } + + pub fn http_status_code(&self) -> Option { + match self { + CodexErr::UnexpectedStatus(err) => Some(err.status), + CodexErr::RetryLimit(err) => Some(err.status), + CodexErr::UsageLimitReached(_) | CodexErr::UsageNotIncluded => { + Some(StatusCode::TOO_MANY_REQUESTS) + } + CodexErr::InternalServerError => Some(StatusCode::INTERNAL_SERVER_ERROR), + CodexErr::ResponseStreamFailed(err) => err.source.status(), + CodexErr::ConnectionFailed(err) => err.source.status(), + _ => None, + } + } + + pub fn to_error_event(&self, message_prefix: Option) -> ErrorEvent { + let error_message = self.to_string(); + let message: String = match message_prefix { + Some(prefix) => format!("{prefix}: {error_message}"), + None => error_message, + }; + + ErrorEvent { + message, + http_status_code: http_status_code_value(self.http_status_code()), + } + } +} + +pub fn http_status_code_value(http_status_code: Option) -> Option { + http_status_code.as_ref().map(StatusCode::as_u16) } pub fn get_error_message_ui(e: &CodexErr) -> String { @@ -775,4 +807,43 @@ mod tests { assert_eq!(err.to_string(), expected); }); } + + #[test] + fn error_event_includes_http_status_code_when_available() { + let err = CodexErr::UnexpectedStatus(UnexpectedResponseError { + status: StatusCode::BAD_REQUEST, + body: "oops".to_string(), + request_id: Some("req-1".to_string()), + }); + let event = err.to_error_event(None); + + assert_eq!( + event.message, + "unexpected status 400 Bad Request: oops, request id: req-1" + ); + assert_eq!( + event.http_status_code, + Some(StatusCode::BAD_REQUEST.as_u16()) + ); + } + + #[test] + fn error_event_omits_http_status_code_when_unknown() { + let event = CodexErr::Fatal("boom".to_string()).to_error_event(None); + + assert_eq!(event.message, "Fatal error: boom"); + assert_eq!(event.http_status_code, None); + } + + #[test] + fn error_event_applies_message_wrapper() { + let event = CodexErr::Fatal("boom".to_string()) + .to_error_event(Some("Error running remote compact task".to_string())); + + assert_eq!( + event.message, + "Error running remote compact task: Fatal error: boom" + ); + assert_eq!(event.http_status_code, None); + } } diff --git a/codex-rs/docs/protocol_v1.md b/codex-rs/docs/protocol_v1.md index f6405c9bc3..c6af7c9260 100644 --- a/codex-rs/docs/protocol_v1.md +++ b/codex-rs/docs/protocol_v1.md @@ -72,7 +72,7 @@ For complete documentation of the `Op` and `EventMsg` variants, refer to [protoc - `EventMsg::AgentMessage` – Messages from the `Model` - `EventMsg::ExecApprovalRequest` – Request approval from user to execute a command - `EventMsg::TaskComplete` – A task completed successfully - - `EventMsg::Error` – A task stopped with an error + - `EventMsg::Error` – A task stopped with an error (includes an optional `http_status_code` when available) - `EventMsg::Warning` – A non-fatal warning that the client should surface to the user - `EventMsg::TurnComplete` – Contains a `response_id` bookmark for last `response_id` executed by the task. This can be used to continue the task at a later point in time, perhaps with additional user input. diff --git a/codex-rs/exec/src/event_processor_with_human_output.rs b/codex-rs/exec/src/event_processor_with_human_output.rs index 2d550fea46..86af82118b 100644 --- a/codex-rs/exec/src/event_processor_with_human_output.rs +++ b/codex-rs/exec/src/event_processor_with_human_output.rs @@ -161,7 +161,7 @@ impl EventProcessor for EventProcessorWithHumanOutput { fn process_event(&mut self, event: Event) -> CodexStatus { let Event { id: _, msg } = event; match msg { - EventMsg::Error(ErrorEvent { message }) => { + EventMsg::Error(ErrorEvent { message, .. }) => { let prefix = "ERROR:".style(self.red); ts_msg!(self, "{prefix} {message}"); } @@ -221,7 +221,7 @@ impl EventProcessor for EventProcessorWithHumanOutput { EventMsg::BackgroundEvent(BackgroundEventEvent { message }) => { ts_msg!(self, "{}", message.style(self.dimmed)); } - EventMsg::StreamError(StreamErrorEvent { message }) => { + EventMsg::StreamError(StreamErrorEvent { message, .. }) => { ts_msg!(self, "{}", message.style(self.dimmed)); } EventMsg::TaskStarted(_) => { diff --git a/codex-rs/exec/tests/event_processor_with_json_output.rs b/codex-rs/exec/tests/event_processor_with_json_output.rs index 5053f61921..444d32304a 100644 --- a/codex-rs/exec/tests/event_processor_with_json_output.rs +++ b/codex-rs/exec/tests/event_processor_with_json_output.rs @@ -539,6 +539,7 @@ fn error_event_produces_error() { "e1", EventMsg::Error(codex_core::protocol::ErrorEvent { message: "boom".to_string(), + http_status_code: Some(500), }), )); assert_eq!( @@ -578,6 +579,7 @@ fn stream_error_event_produces_error() { "e1", EventMsg::StreamError(codex_core::protocol::StreamErrorEvent { message: "retrying".to_string(), + http_status_code: Some(500), }), )); assert_eq!( @@ -596,6 +598,7 @@ fn error_followed_by_task_complete_produces_turn_failed() { "e1", EventMsg::Error(ErrorEvent { message: "boom".to_string(), + http_status_code: Some(500), }), ); assert_eq!( diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index 1825d7636b..0158ce9cfe 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -686,6 +686,8 @@ pub struct ExitedReviewModeEvent { #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, TS)] pub struct ErrorEvent { pub message: String, + #[serde(default)] + pub http_status_code: Option, } #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, TS)] @@ -1363,6 +1365,8 @@ pub struct UndoCompletedEvent { #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, TS)] pub struct StreamErrorEvent { pub message: String, + #[serde(default)] + pub http_status_code: Option, } #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, TS)] diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 2642648a08..e832b58902 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -1627,7 +1627,7 @@ impl ChatWidget { self.on_rate_limit_snapshot(ev.rate_limits); } EventMsg::Warning(WarningEvent { message }) => self.on_warning(message), - EventMsg::Error(ErrorEvent { message }) => self.on_error(message), + EventMsg::Error(ErrorEvent { message, .. }) => self.on_error(message), EventMsg::McpStartupUpdate(ev) => self.on_mcp_startup_update(ev), EventMsg::McpStartupComplete(ev) => self.on_mcp_startup_complete(ev), EventMsg::TurnAborted(ev) => match ev.reason { @@ -1670,7 +1670,9 @@ impl ChatWidget { } EventMsg::UndoStarted(ev) => self.on_undo_started(ev), EventMsg::UndoCompleted(ev) => self.on_undo_completed(ev), - EventMsg::StreamError(StreamErrorEvent { message }) => self.on_stream_error(message), + EventMsg::StreamError(StreamErrorEvent { message, .. }) => { + self.on_stream_error(message) + } EventMsg::UserMessage(ev) => { if from_replay { self.on_user_message_event(ev); diff --git a/codex-rs/tui/src/chatwidget/tests.rs b/codex-rs/tui/src/chatwidget/tests.rs index 0f67579b6e..a6ba546472 100644 --- a/codex-rs/tui/src/chatwidget/tests.rs +++ b/codex-rs/tui/src/chatwidget/tests.rs @@ -2502,6 +2502,7 @@ fn stream_error_updates_status_indicator() { id: "sub-1".into(), msg: EventMsg::StreamError(StreamErrorEvent { message: msg.to_string(), + http_status_code: None, }), }); From 056c8f82792d58c0305902e9dbc27c80b4fc9e32 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Wed, 19 Nov 2025 13:55:29 -0800 Subject: [PATCH 02/42] fix: prepare ExecPolicy in exec-server for execpolicy2 cutover (#6888) This PR introduces an extra layer of abstraction to prepare us for the migration to execpolicy2: - introduces a new trait, `EscalationPolicy`, whose `determine_action()` method is responsible for producing the `EscalateAction` - the existing `ExecPolicy` typedef is changed to return an intermediate `ExecPolicyOutcome` instead of `EscalateAction` - the default implementation of `EscalationPolicy`, `McpEscalationPolicy`, composes `ExecPolicy` - the `ExecPolicyOutcome` includes `codex_execpolicy2::Decision`, which has a `Prompt` variant - when `McpEscalationPolicy` gets `Decision::Prompt` back from `ExecPolicy`, it prompts the user via an MCP elicitation and maps the result into an `ElicitationAction` - now that the end user can reply to an elicitation with `Decline` or `Cancel`, we introduce a new variant, `EscalateAction::Deny`, which the client handles by returning exit code `1` without running anything Note the way the elicitation is created is still not quite right, but I will fix that once we have things running end-to-end for real in a follow-up PR. --- codex-rs/Cargo.lock | 2 + codex-rs/exec-server/Cargo.toml | 2 + codex-rs/exec-server/src/posix.rs | 74 +++++++++-- .../exec-server/src/posix/escalate_client.rs | 7 ++ .../src/posix/escalate_protocol.rs | 2 + .../exec-server/src/posix/escalate_server.rs | 71 ++++++++--- .../src/posix/escalation_policy.rs | 14 +++ codex-rs/exec-server/src/posix/mcp.rs | 33 ++--- .../src/posix/mcp_escalation_policy.rs | 117 ++++++++++++++++++ 9 files changed, 266 insertions(+), 56 deletions(-) create mode 100644 codex-rs/exec-server/src/posix/escalation_policy.rs create mode 100644 codex-rs/exec-server/src/posix/mcp_escalation_policy.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 6ace22c766..66a1c208e4 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1188,6 +1188,7 @@ name = "codex-exec-server" version = "0.0.0" dependencies = [ "anyhow", + "async-trait", "clap", "codex-core", "libc", @@ -1196,6 +1197,7 @@ dependencies = [ "rmcp", "serde", "serde_json", + "shlex", "socket2 0.6.0", "tempfile", "tokio", diff --git a/codex-rs/exec-server/Cargo.toml b/codex-rs/exec-server/Cargo.toml index 24ee460f50..60e0facff1 100644 --- a/codex-rs/exec-server/Cargo.toml +++ b/codex-rs/exec-server/Cargo.toml @@ -12,6 +12,7 @@ workspace = true [dependencies] anyhow = { workspace = true } +async-trait = { workspace = true } clap = { workspace = true, features = ["derive"] } codex-core = { workspace = true } libc = { workspace = true } @@ -31,6 +32,7 @@ rmcp = { workspace = true, default-features = false, features = [ ] } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } +shlex = { workspace = true } socket2 = { workspace = true } tokio = { workspace = true, features = [ "io-std", diff --git a/codex-rs/exec-server/src/posix.rs b/codex-rs/exec-server/src/posix.rs index 179fc7a6fc..bbb624df09 100644 --- a/codex-rs/exec-server/src/posix.rs +++ b/codex-rs/exec-server/src/posix.rs @@ -64,25 +64,17 @@ use tracing_subscriber::{self}; use crate::posix::escalate_protocol::EscalateAction; use crate::posix::escalate_server::EscalateServer; +use crate::posix::escalation_policy::EscalationPolicy; +use crate::posix::mcp_escalation_policy::ExecPolicyOutcome; mod escalate_client; mod escalate_protocol; mod escalate_server; +mod escalation_policy; mod mcp; +mod mcp_escalation_policy; mod socket; -fn dummy_exec_policy(file: &Path, argv: &[String], _workdir: &Path) -> EscalateAction { - // TODO: execpolicy - if file == Path::new("/opt/homebrew/bin/gh") - && let [_, arg1, arg2, ..] = argv - && arg1 == "issue" - && arg2 == "list" - { - return EscalateAction::Escalate; - } - EscalateAction::Run -} - #[derive(Parser)] #[command(version)] pub struct Cli { @@ -135,7 +127,7 @@ pub async fn main() -> anyhow::Result<()> { } Some(Commands::ShellExec(args)) => { let bash_path = mcp::get_bash_path()?; - let escalate_server = EscalateServer::new(bash_path, dummy_exec_policy); + let escalate_server = EscalateServer::new(bash_path, DummyEscalationPolicy {}); let result = escalate_server .exec( args.command.clone(), @@ -162,3 +154,59 @@ pub async fn main() -> anyhow::Result<()> { } } } + +// TODO: replace with execpolicy2 + +struct DummyEscalationPolicy; + +#[async_trait::async_trait] +impl EscalationPolicy for DummyEscalationPolicy { + async fn determine_action( + &self, + file: &Path, + argv: &[String], + workdir: &Path, + ) -> Result { + let outcome = dummy_exec_policy(file, argv, workdir); + let action = match outcome { + ExecPolicyOutcome::Allow { + run_with_escalated_permissions, + } => { + if run_with_escalated_permissions { + EscalateAction::Escalate + } else { + EscalateAction::Run + } + } + ExecPolicyOutcome::Forbidden => EscalateAction::Deny { + reason: Some("Execution forbidden by policy".to_string()), + }, + ExecPolicyOutcome::Prompt { .. } => EscalateAction::Deny { + reason: Some("Could not prompt user for permission".to_string()), + }, + }; + Ok(action) + } +} + +fn dummy_exec_policy(file: &Path, argv: &[String], _workdir: &Path) -> ExecPolicyOutcome { + if file.ends_with("/rm") { + ExecPolicyOutcome::Forbidden + } else if file.ends_with("/git") { + ExecPolicyOutcome::Prompt { + run_with_escalated_permissions: false, + } + } else if file == Path::new("/opt/homebrew/bin/gh") + && let [_, arg1, arg2, ..] = argv + && arg1 == "issue" + && arg2 == "list" + { + ExecPolicyOutcome::Allow { + run_with_escalated_permissions: true, + } + } else { + ExecPolicyOutcome::Allow { + run_with_escalated_permissions: false, + } + } +} diff --git a/codex-rs/exec-server/src/posix/escalate_client.rs b/codex-rs/exec-server/src/posix/escalate_client.rs index 2add1dade2..bea4b6fa5f 100644 --- a/codex-rs/exec-server/src/posix/escalate_client.rs +++ b/codex-rs/exec-server/src/posix/escalate_client.rs @@ -98,5 +98,12 @@ pub(crate) async fn run(file: String, argv: Vec) -> anyhow::Result Err(err.into()) } + EscalateAction::Deny { reason } => { + match reason { + Some(reason) => eprintln!("Execution denied: {reason}"), + None => eprintln!("Execution denied"), + } + Ok(1) + } } } diff --git a/codex-rs/exec-server/src/posix/escalate_protocol.rs b/codex-rs/exec-server/src/posix/escalate_protocol.rs index 09d33fc982..e3fc27d071 100644 --- a/codex-rs/exec-server/src/posix/escalate_protocol.rs +++ b/codex-rs/exec-server/src/posix/escalate_protocol.rs @@ -34,6 +34,8 @@ pub(super) enum EscalateAction { Run, /// The command should be escalated to the server for execution. Escalate, + /// The command should not be executed. + Deny { reason: Option }, } /// The client sends this to the server to forward its open FDs. diff --git a/codex-rs/exec-server/src/posix/escalate_server.rs b/codex-rs/exec-server/src/posix/escalate_server.rs index 6d058ff0f0..4f259f0506 100644 --- a/codex-rs/exec-server/src/posix/escalate_server.rs +++ b/codex-rs/exec-server/src/posix/escalate_server.rs @@ -1,8 +1,8 @@ use std::collections::HashMap; use std::os::fd::AsRawFd; -use std::path::Path; use std::path::PathBuf; use std::process::Stdio; +use std::sync::Arc; use std::time::Duration; use anyhow::Context as _; @@ -21,25 +21,24 @@ use crate::posix::escalate_protocol::EscalateRequest; use crate::posix::escalate_protocol::EscalateResponse; use crate::posix::escalate_protocol::SuperExecMessage; use crate::posix::escalate_protocol::SuperExecResult; +use crate::posix::escalation_policy::EscalationPolicy; use crate::posix::socket::AsyncDatagramSocket; use crate::posix::socket::AsyncSocket; -/// This is the policy which decides how to handle an exec() call. -/// -/// `file` is the absolute, canonical path to the executable to run, i.e. the first arg to exec. -/// `argv` is the argv, including the program name (`argv[0]`). -/// `workdir` is the absolute, canonical path to the working directory in which to execute the -/// command. -pub(crate) type ExecPolicy = fn(file: &Path, argv: &[String], workdir: &Path) -> EscalateAction; - pub(crate) struct EscalateServer { bash_path: PathBuf, - policy: ExecPolicy, + policy: Arc, } impl EscalateServer { - pub fn new(bash_path: PathBuf, policy: ExecPolicy) -> Self { - Self { bash_path, policy } + pub fn new

(bash_path: PathBuf, policy: P) -> Self + where + P: EscalationPolicy + Send + Sync + 'static, + { + Self { + bash_path, + policy: Arc::new(policy), + } } pub async fn exec( @@ -53,7 +52,7 @@ impl EscalateServer { let client_socket = escalate_client.into_inner(); client_socket.set_cloexec(false)?; - let escalate_task = tokio::spawn(escalate_task(escalate_server, self.policy)); + let escalate_task = tokio::spawn(escalate_task(escalate_server, self.policy.clone())); let mut env = env.clone(); env.insert( ESCALATE_SOCKET_ENV_VAR.to_string(), @@ -96,7 +95,10 @@ impl EscalateServer { } } -async fn escalate_task(socket: AsyncDatagramSocket, policy: ExecPolicy) -> anyhow::Result<()> { +async fn escalate_task( + socket: AsyncDatagramSocket, + policy: Arc, +) -> anyhow::Result<()> { loop { let (_, mut fds) = socket.receive_with_fds().await?; if fds.len() != 1 { @@ -104,6 +106,7 @@ async fn escalate_task(socket: AsyncDatagramSocket, policy: ExecPolicy) -> anyho continue; } let stream_socket = AsyncSocket::from_fd(fds.remove(0))?; + let policy = policy.clone(); tokio::spawn(async move { if let Err(err) = handle_escalate_session_with_policy(stream_socket, policy).await { tracing::error!("escalate session failed: {err:?}"); @@ -122,7 +125,7 @@ pub(crate) struct ExecResult { async fn handle_escalate_session_with_policy( socket: AsyncSocket, - policy: ExecPolicy, + policy: Arc, ) -> anyhow::Result<()> { let EscalateRequest { file, @@ -132,8 +135,12 @@ async fn handle_escalate_session_with_policy( } = socket.receive::().await?; let file = PathBuf::from(&file).absolutize()?.into_owned(); let workdir = PathBuf::from(&workdir).absolutize()?.into_owned(); - let action = policy(file.as_path(), &argv, &workdir); + let action = policy + .determine_action(file.as_path(), &argv, &workdir) + .await?; + tracing::debug!("decided {action:?} for {file:?} {argv:?} {workdir:?}"); + match action { EscalateAction::Run => { socket @@ -195,6 +202,13 @@ async fn handle_escalate_session_with_policy( }) .await?; } + EscalateAction::Deny { reason } => { + socket + .send(EscalateResponse { + action: EscalateAction::Deny { reason }, + }) + .await?; + } } Ok(()) } @@ -204,14 +218,33 @@ mod tests { use super::*; use pretty_assertions::assert_eq; use std::collections::HashMap; + use std::path::Path; use std::path::PathBuf; + struct DeterministicEscalationPolicy { + action: EscalateAction, + } + + #[async_trait::async_trait] + impl EscalationPolicy for DeterministicEscalationPolicy { + async fn determine_action( + &self, + _file: &Path, + _argv: &[String], + _workdir: &Path, + ) -> Result { + Ok(self.action.clone()) + } + } + #[tokio::test] async fn handle_escalate_session_respects_run_in_sandbox_decision() -> anyhow::Result<()> { let (server, client) = AsyncSocket::pair()?; let server_task = tokio::spawn(handle_escalate_session_with_policy( server, - |_file, _argv, _workdir| EscalateAction::Run, + Arc::new(DeterministicEscalationPolicy { + action: EscalateAction::Run, + }), )); client @@ -238,7 +271,9 @@ mod tests { let (server, client) = AsyncSocket::pair()?; let server_task = tokio::spawn(handle_escalate_session_with_policy( server, - |_file, _argv, _workdir| EscalateAction::Escalate, + Arc::new(DeterministicEscalationPolicy { + action: EscalateAction::Escalate, + }), )); client diff --git a/codex-rs/exec-server/src/posix/escalation_policy.rs b/codex-rs/exec-server/src/posix/escalation_policy.rs new file mode 100644 index 0000000000..a7fcc4f627 --- /dev/null +++ b/codex-rs/exec-server/src/posix/escalation_policy.rs @@ -0,0 +1,14 @@ +use std::path::Path; + +use crate::posix::escalate_protocol::EscalateAction; + +/// Decides what action to take in response to an execve request from a client. +#[async_trait::async_trait] +pub(crate) trait EscalationPolicy: Send + Sync { + async fn determine_action( + &self, + file: &Path, + argv: &[String], + workdir: &Path, + ) -> Result; +} diff --git a/codex-rs/exec-server/src/posix/mcp.rs b/codex-rs/exec-server/src/posix/mcp.rs index f4e2b19d4f..2a6e84dd9e 100644 --- a/codex-rs/exec-server/src/posix/mcp.rs +++ b/codex-rs/exec-server/src/posix/mcp.rs @@ -18,9 +18,10 @@ use rmcp::tool_handler; use rmcp::tool_router; use rmcp::transport::stdio; -use crate::posix::escalate_server; use crate::posix::escalate_server::EscalateServer; -use crate::posix::escalate_server::ExecPolicy; +use crate::posix::escalate_server::{self}; +use crate::posix::mcp_escalation_policy::ExecPolicy; +use crate::posix::mcp_escalation_policy::McpEscalationPolicy; /// Path to our patched bash. const CODEX_BASH_PATH_ENV_VAR: &str = "CODEX_BASH_PATH"; @@ -81,10 +82,13 @@ impl ExecTool { #[tool] async fn shell( &self, - _context: RequestContext, + context: RequestContext, Parameters(params): Parameters, ) -> Result { - let escalate_server = EscalateServer::new(self.bash_path.clone(), self.policy); + let escalate_server = EscalateServer::new( + self.bash_path.clone(), + McpEscalationPolicy::new(self.policy, context), + ); let result = escalate_server .exec( params.command, @@ -99,27 +103,6 @@ impl ExecTool { ExecResult::from(result), )?])) } - - #[allow(dead_code)] - async fn prompt( - &self, - command: String, - workdir: String, - context: RequestContext, - ) -> Result { - context - .peer - .create_elicitation(CreateElicitationRequestParam { - message: format!("Allow Codex to run `{command:?}` in `{workdir:?}`?"), - #[allow(clippy::expect_used)] - requested_schema: ElicitationSchema::builder() - .property("dummy", PrimitiveSchema::String(StringSchema::new())) - .build() - .expect("failed to build elicitation schema"), - }) - .await - .map_err(|e| McpError::internal_error(e.to_string(), None)) - } } #[tool_handler] diff --git a/codex-rs/exec-server/src/posix/mcp_escalation_policy.rs b/codex-rs/exec-server/src/posix/mcp_escalation_policy.rs new file mode 100644 index 0000000000..24f42fe91b --- /dev/null +++ b/codex-rs/exec-server/src/posix/mcp_escalation_policy.rs @@ -0,0 +1,117 @@ +use std::path::Path; + +use rmcp::ErrorData as McpError; +use rmcp::RoleServer; +use rmcp::model::CreateElicitationRequestParam; +use rmcp::model::CreateElicitationResult; +use rmcp::model::ElicitationAction; +use rmcp::model::ElicitationSchema; +use rmcp::model::PrimitiveSchema; +use rmcp::model::StringSchema; +use rmcp::service::RequestContext; + +use crate::posix::escalate_protocol::EscalateAction; +use crate::posix::escalation_policy::EscalationPolicy; + +/// This is the policy which decides how to handle an exec() call. +/// +/// `file` is the absolute, canonical path to the executable to run, i.e. the first arg to exec. +/// `argv` is the argv, including the program name (`argv[0]`). +/// `workdir` is the absolute, canonical path to the working directory in which to execute the +/// command. +pub(crate) type ExecPolicy = fn(file: &Path, argv: &[String], workdir: &Path) -> ExecPolicyOutcome; + +pub(crate) enum ExecPolicyOutcome { + Allow { + run_with_escalated_permissions: bool, + }, + Prompt { + run_with_escalated_permissions: bool, + }, + Forbidden, +} + +/// ExecPolicy with access to the MCP RequestContext so that it can leverage +/// elicitations. +pub(crate) struct McpEscalationPolicy { + policy: ExecPolicy, + context: RequestContext, +} + +impl McpEscalationPolicy { + pub(crate) fn new(policy: ExecPolicy, context: RequestContext) -> Self { + Self { policy, context } + } + + async fn prompt( + &self, + _file: &Path, + argv: &[String], + workdir: &Path, + context: RequestContext, + ) -> Result { + let command = shlex::try_join(argv.iter().map(String::as_str)).unwrap_or_default(); + context + .peer + .create_elicitation(CreateElicitationRequestParam { + message: format!("Allow Codex to run `{command:?}` in `{workdir:?}`?"), + #[allow(clippy::expect_used)] + requested_schema: ElicitationSchema::builder() + .property("dummy", PrimitiveSchema::String(StringSchema::new())) + .build() + .expect("failed to build elicitation schema"), + }) + .await + .map_err(|e| McpError::internal_error(e.to_string(), None)) + } +} + +#[async_trait::async_trait] +impl EscalationPolicy for McpEscalationPolicy { + async fn determine_action( + &self, + file: &Path, + argv: &[String], + workdir: &Path, + ) -> Result { + let outcome = (self.policy)(file, argv, workdir); + let action = match outcome { + ExecPolicyOutcome::Allow { + run_with_escalated_permissions, + } => { + if run_with_escalated_permissions { + EscalateAction::Escalate + } else { + EscalateAction::Run + } + } + ExecPolicyOutcome::Prompt { + run_with_escalated_permissions, + } => { + let result = self + .prompt(file, argv, workdir, self.context.clone()) + .await?; + // TODO: Extract reason from `result.content`. + match result.action { + ElicitationAction::Accept => { + if run_with_escalated_permissions { + EscalateAction::Escalate + } else { + EscalateAction::Run + } + } + ElicitationAction::Decline => EscalateAction::Deny { + reason: Some("User declined execution".to_string()), + }, + ElicitationAction::Cancel => EscalateAction::Deny { + reason: Some("User cancelled execution".to_string()), + }, + } + } + ExecPolicyOutcome::Forbidden => EscalateAction::Deny { + reason: Some("Execution forbidden by policy".to_string()), + }, + }; + Ok(action) + } +} From 2fde03b4a040babc9b98c389680cb568b464da85 Mon Sep 17 00:00:00 2001 From: iceweasel-oai Date: Wed, 19 Nov 2025 13:59:17 -0800 Subject: [PATCH 03/42] stop over-reporting world-writable directories (#6936) Fix world-writable audit false positives by expanding generic permissions with MapGenericMask and then checking only concrete write bits. The earlier check looked for FILE_GENERIC_WRITE/generic masks directly, which shares bits with read permissions and could flag an Everyone read ACE as writable. --- codex-rs/windows-sandbox-rs/src/audit.rs | 58 +++++++++++++----------- 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/codex-rs/windows-sandbox-rs/src/audit.rs b/codex-rs/windows-sandbox-rs/src/audit.rs index 8873bc2e91..cba8d06335 100644 --- a/codex-rs/windows-sandbox-rs/src/audit.rs +++ b/codex-rs/windows-sandbox-rs/src/audit.rs @@ -9,15 +9,29 @@ use std::path::PathBuf; use std::time::Duration; use std::time::Instant; use windows_sys::Win32::Foundation::CloseHandle; -use windows_sys::Win32::Foundation::LocalFree; use windows_sys::Win32::Foundation::ERROR_SUCCESS; use windows_sys::Win32::Foundation::HLOCAL; use windows_sys::Win32::Foundation::INVALID_HANDLE_VALUE; +use windows_sys::Win32::Foundation::LocalFree; +use windows_sys::Win32::Security::ACCESS_ALLOWED_ACE; +use windows_sys::Win32::Security::ACE_HEADER; +use windows_sys::Win32::Security::ACL; +use windows_sys::Win32::Security::ACL_SIZE_INFORMATION; +use windows_sys::Win32::Security::AclSizeInformation; use windows_sys::Win32::Security::Authorization::GetNamedSecurityInfoW; use windows_sys::Win32::Security::Authorization::GetSecurityInfo; +use windows_sys::Win32::Security::DACL_SECURITY_INFORMATION; +use windows_sys::Win32::Security::EqualSid; +use windows_sys::Win32::Security::GetAce; +use windows_sys::Win32::Security::GetAclInformation; +use windows_sys::Win32::Security::MapGenericMask; +use windows_sys::Win32::Security::GENERIC_MAPPING; use windows_sys::Win32::Storage::FileSystem::CreateFileW; +use windows_sys::Win32::Storage::FileSystem::FILE_ALL_ACCESS; use windows_sys::Win32::Storage::FileSystem::FILE_APPEND_DATA; use windows_sys::Win32::Storage::FileSystem::FILE_FLAG_BACKUP_SEMANTICS; +use windows_sys::Win32::Storage::FileSystem::FILE_GENERIC_EXECUTE; +use windows_sys::Win32::Storage::FileSystem::FILE_GENERIC_READ; use windows_sys::Win32::Storage::FileSystem::FILE_GENERIC_WRITE; use windows_sys::Win32::Storage::FileSystem::FILE_SHARE_DELETE; use windows_sys::Win32::Storage::FileSystem::FILE_SHARE_READ; @@ -26,17 +40,6 @@ use windows_sys::Win32::Storage::FileSystem::FILE_WRITE_ATTRIBUTES; use windows_sys::Win32::Storage::FileSystem::FILE_WRITE_DATA; use windows_sys::Win32::Storage::FileSystem::FILE_WRITE_EA; use windows_sys::Win32::Storage::FileSystem::OPEN_EXISTING; -const GENERIC_ALL_MASK: u32 = 0x1000_0000; -const GENERIC_WRITE_MASK: u32 = 0x4000_0000; -use windows_sys::Win32::Security::AclSizeInformation; -use windows_sys::Win32::Security::EqualSid; -use windows_sys::Win32::Security::GetAce; -use windows_sys::Win32::Security::GetAclInformation; -use windows_sys::Win32::Security::ACCESS_ALLOWED_ACE; -use windows_sys::Win32::Security::ACE_HEADER; -use windows_sys::Win32::Security::ACL; -use windows_sys::Win32::Security::ACL_SIZE_INFORMATION; -use windows_sys::Win32::Security::DACL_SECURITY_INFORMATION; // Preflight scan limits const MAX_ITEMS_PER_DIR: i32 = 1000; @@ -304,7 +307,7 @@ pub fn world_writable_warning_details( } } // Fast mask-based check: does the DACL contain any ACCESS_ALLOWED ACE for -// Everyone that includes generic or specific write bits? Skips inherit-only +// Everyone that grants write after generic bits are expanded? Skips inherit-only // ACEs (do not apply to the current object). unsafe fn dacl_quick_world_write_mask_allows(p_dacl: *mut ACL, psid_world: *mut c_void) -> bool { if p_dacl.is_null() { @@ -321,6 +324,12 @@ unsafe fn dacl_quick_world_write_mask_allows(p_dacl: *mut ACL, psid_world: *mut if ok == 0 { return false; } + let mapping = GENERIC_MAPPING { + GenericRead: FILE_GENERIC_READ, + GenericWrite: FILE_GENERIC_WRITE, + GenericExecute: FILE_GENERIC_EXECUTE, + GenericAll: FILE_ALL_ACCESS, + }; for i in 0..(info.AceCount as usize) { let mut p_ace: *mut c_void = std::ptr::null_mut(); if GetAce(p_dacl as *const ACL, i as u32, &mut p_ace) == 0 { @@ -337,19 +346,16 @@ unsafe fn dacl_quick_world_write_mask_allows(p_dacl: *mut ACL, psid_world: *mut let base = p_ace as usize; let sid_ptr = (base + std::mem::size_of::() + std::mem::size_of::()) as *mut c_void; // skip header + mask - if EqualSid(sid_ptr, psid_world) != 0 { - let ace = &*(p_ace as *const ACCESS_ALLOWED_ACE); - let mask = ace.Mask; - let writey = FILE_GENERIC_WRITE - | FILE_WRITE_DATA - | FILE_APPEND_DATA - | FILE_WRITE_EA - | FILE_WRITE_ATTRIBUTES - | GENERIC_WRITE_MASK - | GENERIC_ALL_MASK; - if (mask & writey) != 0 { - return true; - } + if EqualSid(sid_ptr, psid_world) == 0 { + continue; + } + let ace = &*(p_ace as *const ACCESS_ALLOWED_ACE); + let mut mask = ace.Mask; + // Expand generic bits to concrete file rights before checking for write. + MapGenericMask(&mut mask, &mapping); + let write_mask = FILE_WRITE_DATA | FILE_APPEND_DATA | FILE_WRITE_EA | FILE_WRITE_ATTRIBUTES; + if (mask & write_mask) != 0 { + return true; } } false From 692989c2770065baafe57d7b6edc89bbd7c61273 Mon Sep 17 00:00:00 2001 From: Beehive Innovations Date: Thu, 20 Nov 2025 02:50:07 +0400 Subject: [PATCH 04/42] fix(context left after review): review footer context after `/review` (#5610) ## Summary - show live review token usage while `/review` runs and restore the main session indicator afterward - add regression coverage for the footer behavior ## Testing - just fmt - cargo test -p codex-tui Fixes #5604 --------- Signed-off-by: Fahad --- codex-rs/tui/src/bottom_pane/mod.rs | 5 ++ codex-rs/tui/src/chatwidget.rs | 47 +++++++++++--- codex-rs/tui/src/chatwidget/tests.rs | 94 ++++++++++++++++++++++++++++ 3 files changed, 138 insertions(+), 8 deletions(-) diff --git a/codex-rs/tui/src/bottom_pane/mod.rs b/codex-rs/tui/src/bottom_pane/mod.rs index da2efb63c1..6738d7672d 100644 --- a/codex-rs/tui/src/bottom_pane/mod.rs +++ b/codex-rs/tui/src/bottom_pane/mod.rs @@ -114,6 +114,11 @@ impl BottomPane { self.status.as_ref() } + #[cfg(test)] + pub(crate) fn context_window_percent(&self) -> Option { + self.context_window_percent + } + fn active_view(&self) -> Option<&dyn BottomPaneView> { self.view_stack.last().map(std::convert::AsRef::as_ref) } diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index e832b58902..3a79330b74 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -290,6 +290,8 @@ pub(crate) struct ChatWidget { pending_notification: Option, // Simple review mode flag; used to adjust layout and banners. is_review_mode: bool, + // Snapshot of token usage to restore after review mode exits. + pre_review_token_info: Option>, // Whether to add a final message separator after the last message needs_final_message_separator: bool, @@ -489,16 +491,39 @@ impl ChatWidget { } pub(crate) fn set_token_info(&mut self, info: Option) { - if let Some(info) = info { - let context_window = info - .model_context_window - .or(self.config.model_context_window); - let percent = context_window.map(|window| { + match info { + Some(info) => self.apply_token_info(info), + None => { + self.bottom_pane.set_context_window_percent(None); + self.token_info = None; + } + } + } + + fn apply_token_info(&mut self, info: TokenUsageInfo) { + let percent = self.context_remaining_percent(&info); + self.bottom_pane.set_context_window_percent(percent); + self.token_info = Some(info); + } + + fn context_remaining_percent(&self, info: &TokenUsageInfo) -> Option { + info.model_context_window + .or(self.config.model_context_window) + .map(|window| { info.last_token_usage .percent_of_context_window_remaining(window) - }); - self.bottom_pane.set_context_window_percent(percent); - self.token_info = Some(info); + }) + } + + fn restore_pre_review_token_info(&mut self) { + if let Some(saved) = self.pre_review_token_info.take() { + match saved { + Some(info) => self.apply_token_info(info), + None => { + self.bottom_pane.set_context_window_percent(None); + self.token_info = None; + } + } } } @@ -1150,6 +1175,7 @@ impl ChatWidget { suppress_session_configured_redraw: false, pending_notification: None, is_review_mode: false, + pre_review_token_info: None, needs_final_message_separator: false, last_rendered_width: std::cell::Cell::new(None), feedback, @@ -1223,6 +1249,7 @@ impl ChatWidget { suppress_session_configured_redraw: true, pending_notification: None, is_review_mode: false, + pre_review_token_info: None, needs_final_message_separator: false, last_rendered_width: std::cell::Cell::new(None), feedback, @@ -1693,6 +1720,9 @@ impl ChatWidget { fn on_entered_review_mode(&mut self, review: ReviewRequest) { // Enter review mode and emit a concise banner + if self.pre_review_token_info.is_none() { + self.pre_review_token_info = Some(self.token_info.clone()); + } self.is_review_mode = true; let banner = format!(">> Code review started: {} <<", review.user_facing_hint); self.add_to_history(history_cell::new_review_status_line(banner)); @@ -1733,6 +1763,7 @@ impl ChatWidget { } self.is_review_mode = false; + self.restore_pre_review_token_info(); // Append a finishing banner at the end of this turn. self.add_to_history(history_cell::new_review_status_line( "<< Code review finished >>".to_string(), diff --git a/codex-rs/tui/src/chatwidget/tests.rs b/codex-rs/tui/src/chatwidget/tests.rs index a6ba546472..a4cef22418 100644 --- a/codex-rs/tui/src/chatwidget/tests.rs +++ b/codex-rs/tui/src/chatwidget/tests.rs @@ -38,6 +38,9 @@ use codex_core::protocol::ReviewRequest; use codex_core::protocol::StreamErrorEvent; use codex_core::protocol::TaskCompleteEvent; use codex_core::protocol::TaskStartedEvent; +use codex_core::protocol::TokenCountEvent; +use codex_core::protocol::TokenUsage; +use codex_core::protocol::TokenUsageInfo; use codex_core::protocol::UndoCompletedEvent; use codex_core::protocol::UndoStartedEvent; use codex_core::protocol::ViewImageToolCallEvent; @@ -215,6 +218,81 @@ fn exited_review_mode_emits_results_and_finishes() { assert!(!chat.is_review_mode); } +/// Exiting review restores the pre-review context window indicator. +#[test] +fn review_restores_context_window_indicator() { + let (mut chat, mut rx, _ops) = make_chatwidget_manual(); + + let context_window = 13_000; + let pre_review_tokens = 12_700; // ~30% remaining after subtracting baseline. + let review_tokens = 12_030; // ~97% remaining after subtracting baseline. + + chat.handle_codex_event(Event { + id: "token-before".into(), + msg: EventMsg::TokenCount(TokenCountEvent { + info: Some(make_token_info(pre_review_tokens, context_window)), + rate_limits: None, + }), + }); + assert_eq!(chat.bottom_pane.context_window_percent(), Some(30)); + + chat.handle_codex_event(Event { + id: "review-start".into(), + msg: EventMsg::EnteredReviewMode(ReviewRequest { + prompt: "Review the latest changes".to_string(), + user_facing_hint: "feature branch".to_string(), + append_to_original_thread: true, + }), + }); + + chat.handle_codex_event(Event { + id: "token-review".into(), + msg: EventMsg::TokenCount(TokenCountEvent { + info: Some(make_token_info(review_tokens, context_window)), + rate_limits: None, + }), + }); + assert_eq!(chat.bottom_pane.context_window_percent(), Some(97)); + + chat.handle_codex_event(Event { + id: "review-end".into(), + msg: EventMsg::ExitedReviewMode(ExitedReviewModeEvent { + review_output: None, + }), + }); + let _ = drain_insert_history(&mut rx); + + assert_eq!(chat.bottom_pane.context_window_percent(), Some(30)); + assert!(!chat.is_review_mode); +} + +/// Receiving a TokenCount event without usage clears the context indicator. +#[test] +fn token_count_none_resets_context_indicator() { + let (mut chat, _rx, _ops) = make_chatwidget_manual(); + + let context_window = 13_000; + let pre_compact_tokens = 12_700; + + chat.handle_codex_event(Event { + id: "token-before".into(), + msg: EventMsg::TokenCount(TokenCountEvent { + info: Some(make_token_info(pre_compact_tokens, context_window)), + rate_limits: None, + }), + }); + assert_eq!(chat.bottom_pane.context_window_percent(), Some(30)); + + chat.handle_codex_event(Event { + id: "token-cleared".into(), + msg: EventMsg::TokenCount(TokenCountEvent { + info: None, + rate_limits: None, + }), + }); + assert_eq!(chat.bottom_pane.context_window_percent(), None); +} + #[cfg_attr( target_os = "macos", ignore = "system configuration APIs are blocked under macOS seatbelt" @@ -292,6 +370,7 @@ fn make_chatwidget_manual() -> ( suppress_session_configured_redraw: false, pending_notification: None, is_review_mode: false, + pre_review_token_info: None, needs_final_message_separator: false, last_rendered_width: std::cell::Cell::new(None), feedback: codex_feedback::CodexFeedback::new(), @@ -338,6 +417,21 @@ fn lines_to_single_string(lines: &[ratatui::text::Line<'static>]) -> String { s } +fn make_token_info(total_tokens: i64, context_window: i64) -> TokenUsageInfo { + fn usage(total_tokens: i64) -> TokenUsage { + TokenUsage { + total_tokens, + ..TokenUsage::default() + } + } + + TokenUsageInfo { + total_token_usage: usage(total_tokens), + last_token_usage: usage(total_tokens), + model_context_window: Some(context_window), + } +} + #[test] fn rate_limit_warnings_emit_thresholds() { let mut state = RateLimitWarningState::default(); From a6597a9958d6ae54e1a3601c653a42cd7d70979a Mon Sep 17 00:00:00 2001 From: Lionel Cheng <60159831+lionelchg@users.noreply.github.com> Date: Wed, 19 Nov 2025 15:52:24 -0800 Subject: [PATCH 05/42] Fix/correct reasoning display (#6749) This closes #6748 by implementing fallback to `model_family.default_reasoning_effort` in `reasoning_effort` display of `/status` when no `model_reasoning_effort` is set in the configuration. ## common/src/config_summary.rs - `create_config_summary_entries` now fills the "reasoning effort" entry with the explicit `config.model_reasoning_effort` when present and falls back to `config.model_family.default_reasoning_effort` when it is `None`, instead of emitting the literal string `none`. - This ensures downstream consumers such as `tui/src/status/helpers.rs` continue to work unchanged while automatically picking up model-family defaults when the user has not selected a reasoning effort. ## tui/src/status/helpers.rs / core/src/model_family.rs `ModelFamily::default_reasoning_effort` metadata is set to `medium` for both `gpt-5*-codex` and `gpt-5` models following the default behaviour of the API and recommendation of the codebase: - per https://platform.openai.com/docs/api-reference/responses/create `gpt-5` defaults to `medium` reasoning when no preset is passed - there is no mention of the preset for `gpt-5.1-codex` in the API docs but `medium` is the default setting for `gpt-5.1-codex` as per `codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__model_reasoning_selection_popup.snap` --------- Signed-off-by: lionelchg Co-authored-by: Eric Traut --- codex-rs/common/src/config_summary.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/codex-rs/common/src/config_summary.rs b/codex-rs/common/src/config_summary.rs index dabc606ce1..8fc1bb26f3 100644 --- a/codex-rs/common/src/config_summary.rs +++ b/codex-rs/common/src/config_summary.rs @@ -15,13 +15,12 @@ pub fn create_config_summary_entries(config: &Config) -> Vec<(&'static str, Stri if config.model_provider.wire_api == WireApi::Responses && config.model_family.supports_reasoning_summaries { - entries.push(( - "reasoning effort", - config - .model_reasoning_effort - .map(|effort| effort.to_string()) - .unwrap_or_else(|| "none".to_string()), - )); + let reasoning_effort = config + .model_reasoning_effort + .or(config.model_family.default_reasoning_effort) + .map(|effort| effort.to_string()) + .unwrap_or_else(|| "none".to_string()); + entries.push(("reasoning effort", reasoning_effort)); entries.push(( "reasoning summaries", config.model_reasoning_summary.to_string(), From 13d378f2ce3a95bdcea912c573c19fc4dca87ca3 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Wed, 19 Nov 2025 16:38:14 -0800 Subject: [PATCH 06/42] chore: refactor exec-server to prepare it for standalone MCP use (#6944) This PR reorganizes things slightly so that: - Instead of a single multitool executable, `codex-exec-server`, we now have two executables: - `codex-exec-mcp-server` to launch the MCP server - `codex-execve-wrapper` is the `execve(2)` wrapper to use with the `BASH_EXEC_WRAPPER` environment variable - `BASH_EXEC_WRAPPER` must be a single executable: it cannot be a command string composed of an executable with args (i.e., it no longer adds the `escalate` subcommand, as before) - `codex-exec-mcp-server` takes `--bash` and `--execve` as options. Though if `--execve` is not specified, the MCP server will check the directory containing `std::env::current_exe()` and attempt to use the file named `codex-execve-wrapper` within it. In development, this works out since these executables are side-by-side in the `target/debug` folder. With respect to testing, this also fixes an important bug in `dummy_exec_policy()`, as I was using `ends_with()` as if it applied to a `String`, but in this case, it is used with a `&Path`, so the semantics are slightly different. Putting this all together, I was able to test this by running the following: ``` ~/code/codex/codex-rs$ npx @modelcontextprotocol/inspector \ ./target/debug/codex-exec-mcp-server --bash ~/code/bash/bash ``` If I try to run `git status` in `/Users/mbolin/code/codex` via the `shell` tool from the MCP server: image then I get prompted with the following elicitation, as expected: image Though a current limitation is that the `shell` tool defaults to a timeout of 10s, which means I only have 10s to respond to the elicitation. Ideally, the time spent waiting for a response from a human should not count against the timeout for the command execution. I will address this in a subsequent PR. --- Note `~/code/bash/bash` was created by doing: ``` cd ~/code git clone https://github.com/bminor/bash cd bash git checkout a8a1c2fac029404d3f42cd39f5a20f24b6e4fe4b ./configure make ``` The patch: ``` diff --git a/execute_cmd.c b/execute_cmd.c index 070f5119..d20ad2b9 100644 --- a/execute_cmd.c +++ b/execute_cmd.c @@ -6129,6 +6129,19 @@ shell_execve (char *command, char **args, char **env) char sample[HASH_BANG_BUFSIZ]; size_t larray; + char* exec_wrapper = getenv("BASH_EXEC_WRAPPER"); + if (exec_wrapper && *exec_wrapper && !whitespace (*exec_wrapper)) + { + char *orig_command = command; + + larray = strvec_len (args); + + memmove (args + 2, args, (++larray) * sizeof (char *)); + args[0] = exec_wrapper; + args[1] = orig_command; + command = exec_wrapper; + } + ``` --- codex-rs/exec-server/Cargo.toml | 12 +- .../src/bin/main_execve_wrapper.rs | 8 + .../exec-server/src/bin/main_mcp_server.rs | 8 + codex-rs/exec-server/src/lib.rs | 8 + codex-rs/exec-server/src/main.rs | 11 -- codex-rs/exec-server/src/posix.rs | 163 +++++++----------- .../exec-server/src/posix/escalate_server.rs | 18 +- codex-rs/exec-server/src/posix/mcp.rs | 8 +- 8 files changed, 113 insertions(+), 123 deletions(-) create mode 100644 codex-rs/exec-server/src/bin/main_execve_wrapper.rs create mode 100644 codex-rs/exec-server/src/bin/main_mcp_server.rs create mode 100644 codex-rs/exec-server/src/lib.rs delete mode 100644 codex-rs/exec-server/src/main.rs diff --git a/codex-rs/exec-server/Cargo.toml b/codex-rs/exec-server/Cargo.toml index 60e0facff1..54cead4118 100644 --- a/codex-rs/exec-server/Cargo.toml +++ b/codex-rs/exec-server/Cargo.toml @@ -4,8 +4,16 @@ name = "codex-exec-server" version = { workspace = true } [[bin]] -name = "codex-exec-server" -path = "src/main.rs" +name = "codex-execve-wrapper" +path = "src/bin/main_execve_wrapper.rs" + +[[bin]] +name = "codex-exec-mcp-server" +path = "src/bin/main_mcp_server.rs" + +[lib] +name = "codex_exec_server" +path = "src/lib.rs" [lints] workspace = true diff --git a/codex-rs/exec-server/src/bin/main_execve_wrapper.rs b/codex-rs/exec-server/src/bin/main_execve_wrapper.rs new file mode 100644 index 0000000000..3ab346e8ef --- /dev/null +++ b/codex-rs/exec-server/src/bin/main_execve_wrapper.rs @@ -0,0 +1,8 @@ +#[cfg(not(unix))] +fn main() { + eprintln!("codex-execve-wrapper is only implemented for UNIX"); + std::process::exit(1); +} + +#[cfg(unix)] +pub use codex_exec_server::main_execve_wrapper as main; diff --git a/codex-rs/exec-server/src/bin/main_mcp_server.rs b/codex-rs/exec-server/src/bin/main_mcp_server.rs new file mode 100644 index 0000000000..6c75ae4237 --- /dev/null +++ b/codex-rs/exec-server/src/bin/main_mcp_server.rs @@ -0,0 +1,8 @@ +#[cfg(not(unix))] +fn main() { + eprintln!("codex-exec-mcp-server is only implemented for UNIX"); + std::process::exit(1); +} + +#[cfg(unix)] +pub use codex_exec_server::main_mcp_server as main; diff --git a/codex-rs/exec-server/src/lib.rs b/codex-rs/exec-server/src/lib.rs new file mode 100644 index 0000000000..adec09d4de --- /dev/null +++ b/codex-rs/exec-server/src/lib.rs @@ -0,0 +1,8 @@ +#[cfg(unix)] +mod posix; + +#[cfg(unix)] +pub use posix::main_execve_wrapper; + +#[cfg(unix)] +pub use posix::main_mcp_server; diff --git a/codex-rs/exec-server/src/main.rs b/codex-rs/exec-server/src/main.rs deleted file mode 100644 index 23a18b2525..0000000000 --- a/codex-rs/exec-server/src/main.rs +++ /dev/null @@ -1,11 +0,0 @@ -#[cfg(target_os = "windows")] -fn main() { - eprintln!("codex-exec-server is not implemented on Windows targets"); - std::process::exit(1); -} - -#[cfg(not(target_os = "windows"))] -mod posix; - -#[cfg(not(target_os = "windows"))] -pub use posix::main; diff --git a/codex-rs/exec-server/src/posix.rs b/codex-rs/exec-server/src/posix.rs index bbb624df09..239eaf61f2 100644 --- a/codex-rs/exec-server/src/posix.rs +++ b/codex-rs/exec-server/src/posix.rs @@ -56,15 +56,12 @@ //! o<-----x //! use std::path::Path; +use std::path::PathBuf; use clap::Parser; -use clap::Subcommand; use tracing_subscriber::EnvFilter; use tracing_subscriber::{self}; -use crate::posix::escalate_protocol::EscalateAction; -use crate::posix::escalate_server::EscalateServer; -use crate::posix::escalation_policy::EscalationPolicy; use crate::posix::mcp_escalation_policy::ExecPolicyOutcome; mod escalate_client; @@ -75,124 +72,84 @@ mod mcp; mod mcp_escalation_policy; mod socket; +/// Default value of --execve option relative to the current executable. +/// Note this must match the name of the binary as specified in Cargo.toml. +const CODEX_EXECVE_WRAPPER_EXE_NAME: &str = "codex-execve-wrapper"; + #[derive(Parser)] -#[command(version)] -pub struct Cli { - #[command(subcommand)] - subcommand: Option, -} +struct McpServerCli { + /// Executable to delegate execve(2) calls to in Bash. + #[arg(long = "execve")] + execve_wrapper: Option, -#[derive(Subcommand)] -enum Commands { - Escalate(EscalateArgs), - ShellExec(ShellExecArgs), -} - -/// Invoked from within the sandbox to (potentially) escalate permissions. -#[derive(Parser, Debug)] -struct EscalateArgs { - file: String, - - #[arg(trailing_var_arg = true)] - argv: Vec, -} - -impl EscalateArgs { - /// This is the escalate client. It talks to the escalate server to determine whether to exec() - /// the command directly or to proxy to the escalate server. - async fn run(self) -> anyhow::Result { - let EscalateArgs { file, argv } = self; - escalate_client::run(file, argv).await - } -} - -/// Debugging command to emulate an MCP "shell" tool call. -#[derive(Parser, Debug)] -struct ShellExecArgs { - command: String, + /// Path to Bash that has been patched to support execve() wrapping. + #[arg(long = "bash")] + bash_path: Option, } #[tokio::main] -pub async fn main() -> anyhow::Result<()> { - let cli = Cli::parse(); +pub async fn main_mcp_server() -> anyhow::Result<()> { tracing_subscriber::fmt() .with_env_filter(EnvFilter::from_default_env()) .with_writer(std::io::stderr) .with_ansi(false) .init(); - match cli.subcommand { - Some(Commands::Escalate(args)) => { - std::process::exit(args.run().await?); - } - Some(Commands::ShellExec(args)) => { - let bash_path = mcp::get_bash_path()?; - let escalate_server = EscalateServer::new(bash_path, DummyEscalationPolicy {}); - let result = escalate_server - .exec( - args.command.clone(), - std::env::vars().collect(), - std::env::current_dir()?, - None, - ) - .await?; - println!("{result:?}"); - std::process::exit(result.exit_code); - } + let cli = McpServerCli::parse(); + let execve_wrapper = match cli.execve_wrapper { + Some(path) => path, None => { - let bash_path = mcp::get_bash_path()?; - - tracing::info!("Starting MCP server"); - let service = mcp::serve(bash_path, dummy_exec_policy) - .await - .inspect_err(|e| { - tracing::error!("serving error: {:?}", e); - })?; - - service.waiting().await?; - Ok(()) + let cwd = std::env::current_exe()?; + cwd.parent() + .map(|p| p.join(CODEX_EXECVE_WRAPPER_EXE_NAME)) + .ok_or_else(|| { + anyhow::anyhow!("failed to determine execve wrapper path from current exe") + })? } - } + }; + let bash_path = match cli.bash_path { + Some(path) => path, + None => mcp::get_bash_path()?, + }; + + tracing::info!("Starting MCP server"); + let service = mcp::serve(bash_path, execve_wrapper, dummy_exec_policy) + .await + .inspect_err(|e| { + tracing::error!("serving error: {:?}", e); + })?; + + service.waiting().await?; + Ok(()) +} + +#[derive(Parser)] +pub struct ExecveWrapperCli { + file: String, + + #[arg(trailing_var_arg = true)] + argv: Vec, +} + +#[tokio::main] +pub async fn main_execve_wrapper() -> anyhow::Result<()> { + tracing_subscriber::fmt() + .with_env_filter(EnvFilter::from_default_env()) + .with_writer(std::io::stderr) + .with_ansi(false) + .init(); + + let ExecveWrapperCli { file, argv } = ExecveWrapperCli::parse(); + let exit_code = escalate_client::run(file, argv).await?; + std::process::exit(exit_code); } // TODO: replace with execpolicy2 -struct DummyEscalationPolicy; - -#[async_trait::async_trait] -impl EscalationPolicy for DummyEscalationPolicy { - async fn determine_action( - &self, - file: &Path, - argv: &[String], - workdir: &Path, - ) -> Result { - let outcome = dummy_exec_policy(file, argv, workdir); - let action = match outcome { - ExecPolicyOutcome::Allow { - run_with_escalated_permissions, - } => { - if run_with_escalated_permissions { - EscalateAction::Escalate - } else { - EscalateAction::Run - } - } - ExecPolicyOutcome::Forbidden => EscalateAction::Deny { - reason: Some("Execution forbidden by policy".to_string()), - }, - ExecPolicyOutcome::Prompt { .. } => EscalateAction::Deny { - reason: Some("Could not prompt user for permission".to_string()), - }, - }; - Ok(action) - } -} - fn dummy_exec_policy(file: &Path, argv: &[String], _workdir: &Path) -> ExecPolicyOutcome { - if file.ends_with("/rm") { + if file.ends_with("rm") { ExecPolicyOutcome::Forbidden - } else if file.ends_with("/git") { + } else if file.ends_with("git") { ExecPolicyOutcome::Prompt { run_with_escalated_permissions: false, } diff --git a/codex-rs/exec-server/src/posix/escalate_server.rs b/codex-rs/exec-server/src/posix/escalate_server.rs index 4f259f0506..1f76687ce0 100644 --- a/codex-rs/exec-server/src/posix/escalate_server.rs +++ b/codex-rs/exec-server/src/posix/escalate_server.rs @@ -27,16 +27,18 @@ use crate::posix::socket::AsyncSocket; pub(crate) struct EscalateServer { bash_path: PathBuf, + execve_wrapper: PathBuf, policy: Arc, } impl EscalateServer { - pub fn new

(bash_path: PathBuf, policy: P) -> Self + pub fn new

(bash_path: PathBuf, execve_wrapper: PathBuf, policy: P) -> Self where P: EscalationPolicy + Send + Sync + 'static, { Self { bash_path, + execve_wrapper, policy: Arc::new(policy), } } @@ -60,8 +62,15 @@ impl EscalateServer { ); env.insert( BASH_EXEC_WRAPPER_ENV_VAR.to_string(), - format!("{} escalate", std::env::current_exe()?.to_string_lossy()), + self.execve_wrapper.to_string_lossy().to_string(), ); + + // TODO: use the sandbox policy and cwd from the calling client. + // Note that sandbox_cwd is ignored for ReadOnly, but needs to be legit + // for `SandboxPolicy::WorkspaceWrite`. + let sandbox_policy = SandboxPolicy::ReadOnly; + let sandbox_cwd = PathBuf::from("/__NONEXISTENT__"); + let result = process_exec_tool_call( codex_core::exec::ExecParams { command: vec![ @@ -77,9 +86,8 @@ impl EscalateServer { arg0: None, }, get_platform_sandbox().unwrap_or(SandboxType::None), - // TODO: use the sandbox policy and cwd from the calling client - &SandboxPolicy::ReadOnly, - &PathBuf::from("/__NONEXISTENT__"), // This is ignored for ReadOnly + &sandbox_policy, + &sandbox_cwd, &None, None, ) diff --git a/codex-rs/exec-server/src/posix/mcp.rs b/codex-rs/exec-server/src/posix/mcp.rs index 2a6e84dd9e..f5785dc5d0 100644 --- a/codex-rs/exec-server/src/posix/mcp.rs +++ b/codex-rs/exec-server/src/posix/mcp.rs @@ -65,15 +65,17 @@ impl From for ExecResult { pub struct ExecTool { tool_router: ToolRouter, bash_path: PathBuf, + execve_wrapper: PathBuf, policy: ExecPolicy, } #[tool_router] impl ExecTool { - pub fn new(bash_path: PathBuf, policy: ExecPolicy) -> Self { + pub fn new(bash_path: PathBuf, execve_wrapper: PathBuf, policy: ExecPolicy) -> Self { Self { tool_router: Self::tool_router(), bash_path, + execve_wrapper, policy, } } @@ -87,6 +89,7 @@ impl ExecTool { ) -> Result { let escalate_server = EscalateServer::new( self.bash_path.clone(), + self.execve_wrapper.clone(), McpEscalationPolicy::new(self.policy, context), ); let result = escalate_server @@ -130,8 +133,9 @@ impl ServerHandler for ExecTool { pub(crate) async fn serve( bash_path: PathBuf, + execve_wrapper: PathBuf, policy: ExecPolicy, ) -> Result, rmcp::service::ServerInitializeError> { - let tool = ExecTool::new(bash_path, policy); + let tool = ExecTool::new(bash_path, execve_wrapper, policy); tool.serve(stdio()).await } From b00a7cf40d7faceab3ed3789c0429d729c91f7b6 Mon Sep 17 00:00:00 2001 From: Dylan Hurd Date: Wed, 19 Nov 2025 16:41:38 -0800 Subject: [PATCH 07/42] fix(shell) fallback shells (#6948) ## Summary Add fallbacks when user_shell_path does not resolve to a known shell type ## Testing - [x] Tests still pass --- codex-rs/core/src/shell.rs | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/codex-rs/core/src/shell.rs b/codex-rs/core/src/shell.rs index 7bfec089c5..1fb2b97dd7 100644 --- a/codex-rs/core/src/shell.rs +++ b/codex-rs/core/src/shell.rs @@ -204,10 +204,21 @@ pub async fn default_user_shell() -> Shell { if cfg!(windows) { get_shell(ShellType::PowerShell, None).unwrap_or(Shell::Unknown) } else { - get_user_shell_path() + let user_default_shell = get_user_shell_path() .and_then(|shell| detect_shell_type(&shell)) - .and_then(|shell_type| get_shell(shell_type, None)) - .unwrap_or(Shell::Unknown) + .and_then(|shell_type| get_shell(shell_type, None)); + + let shell_with_fallback = if cfg!(target_os = "macos") { + user_default_shell + .or_else(|| get_shell(ShellType::Zsh, None)) + .or_else(|| get_shell(ShellType::Bash, None)) + } else { + user_default_shell + .or_else(|| get_shell(ShellType::Bash, None)) + .or_else(|| get_shell(ShellType::Zsh, None)) + }; + + shell_with_fallback.unwrap_or(Shell::Unknown) } } From 65c13f1ae72c6aac8016a63bf9d436547954c9bb Mon Sep 17 00:00:00 2001 From: zhao-oai Date: Wed, 19 Nov 2025 16:50:43 -0800 Subject: [PATCH 08/42] execpolicy2 core integration (#6641) This PR threads execpolicy2 into codex-core. activated via feature flag: exec_policy (on by default) reads and parses all .codexpolicy files in `codex_home/codex` refactored tool runtime API to integrate execpolicy logic --------- Co-authored-by: Michael Bolin --- README.md | 33 ++ codex-rs/Cargo.lock | 1 + codex-rs/Cargo.toml | 1 + codex-rs/core/Cargo.toml | 1 + codex-rs/core/src/codex.rs | 13 + .../command_safety/is_dangerous_command.rs | 7 +- codex-rs/core/src/exec_policy.rs | 365 ++++++++++++++++++ codex-rs/core/src/features.rs | 8 + codex-rs/core/src/lib.rs | 1 + codex-rs/core/src/sandboxing/mod.rs | 22 ++ codex-rs/core/src/tools/handlers/shell.rs | 9 + codex-rs/core/src/tools/orchestrator.rs | 70 ++-- codex-rs/core/src/tools/runtimes/shell.rs | 19 +- .../core/src/tools/runtimes/unified_exec.rs | 21 +- codex-rs/core/src/tools/sandboxing.rs | 52 ++- .../core/src/unified_exec/session_manager.rs | 9 + codex-rs/core/tests/suite/exec_policy.rs | 101 +++++ codex-rs/core/tests/suite/mod.rs | 1 + codex-rs/execpolicy2/README.md | 30 +- codex-rs/execpolicy2/src/policy.rs | 4 + codex-rs/tui/src/chatwidget/agent.rs | 19 +- 21 files changed, 692 insertions(+), 95 deletions(-) create mode 100644 codex-rs/core/src/exec_policy.rs create mode 100644 codex-rs/core/tests/suite/exec_policy.rs diff --git a/README.md b/README.md index 8141610035..e34afbd288 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,39 @@ Codex can access MCP servers. To configure them, refer to the [config docs](./do Codex CLI supports a rich set of configuration options, with preferences stored in `~/.codex/config.toml`. For full configuration options, see [Configuration](./docs/config.md). +### Execpolicy quickstart + +Codex can enforce your own rules-based execution policy before it runs shell commands. + +1. Create a policy directory: `mkdir -p ~/.codex/policy`. +2. Create one or more `.codexpolicy` files into that folder. Codex automatically loads every `.codexpolicy` file in there on startup. +3. Write `prefix_rule` entries to describe the commands you want to allow, prompt, or block: + +```starlark +prefix_rule( + pattern = ["git", ["push", "fetch"]], + decision = "prompt", # allow | prompt | forbidden + match = [["git", "push", "origin", "main"]], # examples that must match + not_match = [["git", "status"]], # examples that must not match +) +``` + +- `pattern` is a list of shell tokens, evaluated from left to right; wrap tokens in a nested list to express alternatives (e.g., match both `push` and `fetch`). +- `decision` sets the severity; Codex picks the strictest decision when multiple rules match. +- `match` and `not_match` act as (optional) unit tests. Codex validates them when it loads your policy, so you get feedback if an example has unexpected behavior. + +In this example rule, if Codex wants to run commands with the prefix `git push` or `git fetch`, it will first ask for user approval. + +Note: If Codex wants to run a command that matches with multiple rules, it will use the strictest decision among the matched rules (forbidden > prompt > allow). + +Use the [`execpolicy2` CLI](./codex-rs/execpolicy2/README.md) to preview decisions before you save a rule: + +```shell +cargo run -p codex-execpolicy2 -- check --policy ~/.codex/policy/default.codexpolicy git push origin main +``` + +Pass multiple `--policy` flags to test how several files combine. See the [`codex-rs/execpolicy2` README](./codex-rs/execpolicy2/README.md) for a more detailed walkthrough of the available syntax. + --- ### Docs & FAQ diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 66a1c208e4..798b8d71b3 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1086,6 +1086,7 @@ dependencies = [ "codex-apply-patch", "codex-arg0", "codex-async-utils", + "codex-execpolicy2", "codex-file-search", "codex-git", "codex-keyring-store", diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index 2e88aba713..b0cf1e70fb 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -67,6 +67,7 @@ codex-chatgpt = { path = "chatgpt" } codex-common = { path = "common" } codex-core = { path = "core" } codex-exec = { path = "exec" } +codex-execpolicy2 = { path = "execpolicy2" } codex-feedback = { path = "feedback" } codex-file-search = { path = "file-search" } codex-git = { path = "utils/git" } diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index 4d8f43778c..db06f6b795 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -22,6 +22,7 @@ chrono = { workspace = true, features = ["serde"] } codex-app-server-protocol = { workspace = true } codex-apply-patch = { workspace = true } codex-async-utils = { workspace = true } +codex-execpolicy2 = { workspace = true } codex-file-search = { workspace = true } codex-git = { workspace = true } codex-keyring-store = { workspace = true } diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 32589dabb3..897f1f2944 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -121,6 +121,7 @@ use crate::user_instructions::UserInstructions; use crate::user_notification::UserNotification; use crate::util::backoff; use codex_async_utils::OrCancelExt; +use codex_execpolicy2::Policy as ExecPolicy; use codex_otel::otel_event_manager::OtelEventManager; use codex_protocol::config_types::ReasoningEffort as ReasoningEffortConfig; use codex_protocol::config_types::ReasoningSummary as ReasoningSummaryConfig; @@ -167,6 +168,10 @@ impl Codex { let user_instructions = get_user_instructions(&config).await; + let exec_policy = crate::exec_policy::exec_policy_for(&config.features, &config.codex_home) + .await + .map_err(|err| CodexErr::Fatal(format!("failed to load execpolicy: {err}")))?; + let config = Arc::new(config); let session_configuration = SessionConfiguration { @@ -183,6 +188,7 @@ impl Codex { cwd: config.cwd.clone(), original_config_do_not_use: Arc::clone(&config), features: config.features.clone(), + exec_policy, session_source, }; @@ -280,6 +286,7 @@ pub(crate) struct TurnContext { pub(crate) final_output_json_schema: Option, pub(crate) codex_linux_sandbox_exe: Option, pub(crate) tool_call_gate: Arc, + pub(crate) exec_policy: Arc, pub(crate) truncation_policy: TruncationPolicy, } @@ -336,6 +343,8 @@ pub(crate) struct SessionConfiguration { /// Set of feature flags for this session features: Features, + /// Execpolicy policy, applied only when enabled by feature flag. + exec_policy: Arc, // TODO(pakrym): Remove config from here original_config_do_not_use: Arc, @@ -436,6 +445,7 @@ impl Session { final_output_json_schema: None, codex_linux_sandbox_exe: config.codex_linux_sandbox_exe.clone(), tool_call_gate: Arc::new(ReadinessFlag::new()), + exec_policy: session_configuration.exec_policy.clone(), truncation_policy: TruncationPolicy::new(&per_turn_config), } } @@ -1789,6 +1799,7 @@ async fn spawn_review_thread( final_output_json_schema: None, codex_linux_sandbox_exe: parent_turn_context.codex_linux_sandbox_exe.clone(), tool_call_gate: Arc::new(ReadinessFlag::new()), + exec_policy: parent_turn_context.exec_policy.clone(), truncation_policy: TruncationPolicy::new(&per_turn_config), }; @@ -2608,6 +2619,7 @@ mod tests { cwd: config.cwd.clone(), original_config_do_not_use: Arc::clone(&config), features: Features::default(), + exec_policy: Arc::new(codex_execpolicy2::Policy::empty()), session_source: SessionSource::Exec, }; @@ -2685,6 +2697,7 @@ mod tests { cwd: config.cwd.clone(), original_config_do_not_use: Arc::clone(&config), features: Features::default(), + exec_policy: Arc::new(codex_execpolicy2::Policy::empty()), session_source: SessionSource::Exec, }; diff --git a/codex-rs/core/src/command_safety/is_dangerous_command.rs b/codex-rs/core/src/command_safety/is_dangerous_command.rs index 09594bb1c5..5df2023f06 100644 --- a/codex-rs/core/src/command_safety/is_dangerous_command.rs +++ b/codex-rs/core/src/command_safety/is_dangerous_command.rs @@ -1,6 +1,8 @@ use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::SandboxPolicy; +use crate::sandboxing::SandboxPermissions; + use crate::bash::parse_shell_lc_plain_commands; use crate::is_safe_command::is_known_safe_command; @@ -8,7 +10,7 @@ pub fn requires_initial_appoval( policy: AskForApproval, sandbox_policy: &SandboxPolicy, command: &[String], - with_escalated_permissions: bool, + sandbox_permissions: SandboxPermissions, ) -> bool { if is_known_safe_command(command) { return false; @@ -24,8 +26,7 @@ pub fn requires_initial_appoval( // In restricted sandboxes (ReadOnly/WorkspaceWrite), do not prompt for // non‑escalated, non‑dangerous commands — let the sandbox enforce // restrictions (e.g., block network/write) without a user prompt. - let wants_escalation: bool = with_escalated_permissions; - if wants_escalation { + if sandbox_permissions.requires_escalated_permissions() { return true; } command_might_be_dangerous(command) diff --git a/codex-rs/core/src/exec_policy.rs b/codex-rs/core/src/exec_policy.rs new file mode 100644 index 0000000000..c55dbfed23 --- /dev/null +++ b/codex-rs/core/src/exec_policy.rs @@ -0,0 +1,365 @@ +use std::io::ErrorKind; +use std::path::Path; +use std::path::PathBuf; +use std::sync::Arc; + +use crate::command_safety::is_dangerous_command::requires_initial_appoval; +use codex_execpolicy2::Decision; +use codex_execpolicy2::Evaluation; +use codex_execpolicy2::Policy; +use codex_execpolicy2::PolicyParser; +use codex_protocol::protocol::AskForApproval; +use codex_protocol::protocol::SandboxPolicy; +use thiserror::Error; +use tokio::fs; + +use crate::bash::parse_shell_lc_plain_commands; +use crate::features::Feature; +use crate::features::Features; +use crate::sandboxing::SandboxPermissions; +use crate::tools::sandboxing::ApprovalRequirement; + +const FORBIDDEN_REASON: &str = "execpolicy forbids this command"; +const PROMPT_REASON: &str = "execpolicy requires approval for this command"; +const POLICY_DIR_NAME: &str = "policy"; +const POLICY_EXTENSION: &str = "codexpolicy"; + +#[derive(Debug, Error)] +pub enum ExecPolicyError { + #[error("failed to read execpolicy files from {dir}: {source}")] + ReadDir { + dir: PathBuf, + source: std::io::Error, + }, + + #[error("failed to read execpolicy file {path}: {source}")] + ReadFile { + path: PathBuf, + source: std::io::Error, + }, + + #[error("failed to parse execpolicy file {path}: {source}")] + ParsePolicy { + path: String, + source: codex_execpolicy2::Error, + }, +} + +pub(crate) async fn exec_policy_for( + features: &Features, + codex_home: &Path, +) -> Result, ExecPolicyError> { + if !features.enabled(Feature::ExecPolicy) { + return Ok(Arc::new(Policy::empty())); + } + + let policy_dir = codex_home.join(POLICY_DIR_NAME); + let policy_paths = collect_policy_files(&policy_dir).await?; + + let mut parser = PolicyParser::new(); + for policy_path in &policy_paths { + let contents = + fs::read_to_string(policy_path) + .await + .map_err(|source| ExecPolicyError::ReadFile { + path: policy_path.clone(), + source, + })?; + let identifier = policy_path.to_string_lossy().to_string(); + parser + .parse(&identifier, &contents) + .map_err(|source| ExecPolicyError::ParsePolicy { + path: identifier, + source, + })?; + } + + let policy = Arc::new(parser.build()); + tracing::debug!( + "loaded execpolicy from {} files in {}", + policy_paths.len(), + policy_dir.display() + ); + + Ok(policy) +} + +fn evaluate_with_policy( + policy: &Policy, + command: &[String], + approval_policy: AskForApproval, +) -> Option { + let commands = parse_shell_lc_plain_commands(command).unwrap_or_else(|| vec![command.to_vec()]); + let evaluation = policy.check_multiple(commands.iter()); + + match evaluation { + Evaluation::Match { decision, .. } => match decision { + Decision::Forbidden => Some(ApprovalRequirement::Forbidden { + reason: FORBIDDEN_REASON.to_string(), + }), + Decision::Prompt => { + let reason = PROMPT_REASON.to_string(); + if matches!(approval_policy, AskForApproval::Never) { + Some(ApprovalRequirement::Forbidden { reason }) + } else { + Some(ApprovalRequirement::NeedsApproval { + reason: Some(reason), + }) + } + } + Decision::Allow => Some(ApprovalRequirement::Skip), + }, + Evaluation::NoMatch => None, + } +} + +pub(crate) fn create_approval_requirement_for_command( + policy: &Policy, + command: &[String], + approval_policy: AskForApproval, + sandbox_policy: &SandboxPolicy, + sandbox_permissions: SandboxPermissions, +) -> ApprovalRequirement { + if let Some(requirement) = evaluate_with_policy(policy, command, approval_policy) { + return requirement; + } + + if requires_initial_appoval( + approval_policy, + sandbox_policy, + command, + sandbox_permissions, + ) { + ApprovalRequirement::NeedsApproval { reason: None } + } else { + ApprovalRequirement::Skip + } +} + +async fn collect_policy_files(dir: &Path) -> Result, ExecPolicyError> { + let mut read_dir = match fs::read_dir(dir).await { + Ok(read_dir) => read_dir, + Err(err) if err.kind() == ErrorKind::NotFound => return Ok(Vec::new()), + Err(source) => { + return Err(ExecPolicyError::ReadDir { + dir: dir.to_path_buf(), + source, + }); + } + }; + + let mut policy_paths = Vec::new(); + while let Some(entry) = + read_dir + .next_entry() + .await + .map_err(|source| ExecPolicyError::ReadDir { + dir: dir.to_path_buf(), + source, + })? + { + let path = entry.path(); + let file_type = entry + .file_type() + .await + .map_err(|source| ExecPolicyError::ReadDir { + dir: dir.to_path_buf(), + source, + })?; + + if path + .extension() + .and_then(|ext| ext.to_str()) + .is_some_and(|ext| ext == POLICY_EXTENSION) + && file_type.is_file() + { + policy_paths.push(path); + } + } + + policy_paths.sort(); + + Ok(policy_paths) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::features::Feature; + use crate::features::Features; + use codex_protocol::protocol::AskForApproval; + use codex_protocol::protocol::SandboxPolicy; + use pretty_assertions::assert_eq; + use std::fs; + use tempfile::tempdir; + + #[tokio::test] + async fn returns_empty_policy_when_feature_disabled() { + let mut features = Features::with_defaults(); + features.disable(Feature::ExecPolicy); + let temp_dir = tempdir().expect("create temp dir"); + + let policy = exec_policy_for(&features, temp_dir.path()) + .await + .expect("policy result"); + + let commands = [vec!["rm".to_string()]]; + assert!(matches!( + policy.check_multiple(commands.iter()), + Evaluation::NoMatch + )); + assert!(!temp_dir.path().join(POLICY_DIR_NAME).exists()); + } + + #[tokio::test] + async fn collect_policy_files_returns_empty_when_dir_missing() { + let temp_dir = tempdir().expect("create temp dir"); + + let policy_dir = temp_dir.path().join(POLICY_DIR_NAME); + let files = collect_policy_files(&policy_dir) + .await + .expect("collect policy files"); + + assert!(files.is_empty()); + } + + #[tokio::test] + async fn loads_policies_from_policy_subdirectory() { + let temp_dir = tempdir().expect("create temp dir"); + let policy_dir = temp_dir.path().join(POLICY_DIR_NAME); + fs::create_dir_all(&policy_dir).expect("create policy dir"); + fs::write( + policy_dir.join("deny.codexpolicy"), + r#"prefix_rule(pattern=["rm"], decision="forbidden")"#, + ) + .expect("write policy file"); + + let policy = exec_policy_for(&Features::with_defaults(), temp_dir.path()) + .await + .expect("policy result"); + let command = [vec!["rm".to_string()]]; + assert!(matches!( + policy.check_multiple(command.iter()), + Evaluation::Match { .. } + )); + } + + #[tokio::test] + async fn ignores_policies_outside_policy_dir() { + let temp_dir = tempdir().expect("create temp dir"); + fs::write( + temp_dir.path().join("root.codexpolicy"), + r#"prefix_rule(pattern=["ls"], decision="prompt")"#, + ) + .expect("write policy file"); + + let policy = exec_policy_for(&Features::with_defaults(), temp_dir.path()) + .await + .expect("policy result"); + let command = [vec!["ls".to_string()]]; + assert!(matches!( + policy.check_multiple(command.iter()), + Evaluation::NoMatch + )); + } + + #[test] + fn evaluates_bash_lc_inner_commands() { + let policy_src = r#" +prefix_rule(pattern=["rm"], decision="forbidden") +"#; + let mut parser = PolicyParser::new(); + parser + .parse("test.codexpolicy", policy_src) + .expect("parse policy"); + let policy = parser.build(); + + let forbidden_script = vec![ + "bash".to_string(), + "-lc".to_string(), + "rm -rf /tmp".to_string(), + ]; + + let requirement = + evaluate_with_policy(&policy, &forbidden_script, AskForApproval::OnRequest) + .expect("expected match for forbidden command"); + + assert_eq!( + requirement, + ApprovalRequirement::Forbidden { + reason: FORBIDDEN_REASON.to_string() + } + ); + } + + #[test] + fn approval_requirement_prefers_execpolicy_match() { + let policy_src = r#"prefix_rule(pattern=["rm"], decision="prompt")"#; + let mut parser = PolicyParser::new(); + parser + .parse("test.codexpolicy", policy_src) + .expect("parse policy"); + let policy = parser.build(); + let command = vec!["rm".to_string()]; + + let requirement = create_approval_requirement_for_command( + &policy, + &command, + AskForApproval::OnRequest, + &SandboxPolicy::DangerFullAccess, + SandboxPermissions::UseDefault, + ); + + assert_eq!( + requirement, + ApprovalRequirement::NeedsApproval { + reason: Some(PROMPT_REASON.to_string()) + } + ); + } + + #[test] + fn approval_requirement_respects_approval_policy() { + let policy_src = r#"prefix_rule(pattern=["rm"], decision="prompt")"#; + let mut parser = PolicyParser::new(); + parser + .parse("test.codexpolicy", policy_src) + .expect("parse policy"); + let policy = parser.build(); + let command = vec!["rm".to_string()]; + + let requirement = create_approval_requirement_for_command( + &policy, + &command, + AskForApproval::Never, + &SandboxPolicy::DangerFullAccess, + SandboxPermissions::UseDefault, + ); + + assert_eq!( + requirement, + ApprovalRequirement::Forbidden { + reason: PROMPT_REASON.to_string() + } + ); + } + + #[test] + fn approval_requirement_falls_back_to_heuristics() { + let command = vec!["python".to_string()]; + + let empty_policy = Policy::empty(); + let requirement = create_approval_requirement_for_command( + &empty_policy, + &command, + AskForApproval::UnlessTrusted, + &SandboxPolicy::ReadOnly, + SandboxPermissions::UseDefault, + ); + + assert_eq!( + requirement, + ApprovalRequirement::NeedsApproval { reason: None } + ); + } +} diff --git a/codex-rs/core/src/features.rs b/codex-rs/core/src/features.rs index 1a7fa21b45..75622e5254 100644 --- a/codex-rs/core/src/features.rs +++ b/codex-rs/core/src/features.rs @@ -42,6 +42,8 @@ pub enum Feature { ViewImageTool, /// Allow the model to request web searches. WebSearchRequest, + /// Gate the execpolicy enforcement for shell/unified exec. + ExecPolicy, /// Enable the model-based risk assessments for sandboxed commands. SandboxCommandAssessment, /// Enable Windows sandbox (restricted token) on Windows. @@ -297,6 +299,12 @@ pub const FEATURES: &[FeatureSpec] = &[ stage: Stage::Stable, default_enabled: false, }, + FeatureSpec { + id: Feature::ExecPolicy, + key: "exec_policy", + stage: Stage::Experimental, + default_enabled: true, + }, FeatureSpec { id: Feature::SandboxCommandAssessment, key: "experimental_sandbox_command_assessment", diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index 3e74633456..2ae11d79d0 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -25,6 +25,7 @@ mod environment_context; pub mod error; pub mod exec; pub mod exec_env; +mod exec_policy; pub mod features; mod flags; pub mod git_info; diff --git a/codex-rs/core/src/sandboxing/mod.rs b/codex-rs/core/src/sandboxing/mod.rs index 5e564f5109..4ecb2a8c12 100644 --- a/codex-rs/core/src/sandboxing/mod.rs +++ b/codex-rs/core/src/sandboxing/mod.rs @@ -26,6 +26,28 @@ use std::collections::HashMap; use std::path::Path; use std::path::PathBuf; +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum SandboxPermissions { + UseDefault, + RequireEscalated, +} + +impl SandboxPermissions { + pub fn requires_escalated_permissions(self) -> bool { + matches!(self, SandboxPermissions::RequireEscalated) + } +} + +impl From for SandboxPermissions { + fn from(with_escalated_permissions: bool) -> Self { + if with_escalated_permissions { + SandboxPermissions::RequireEscalated + } else { + SandboxPermissions::UseDefault + } + } +} + #[derive(Clone, Debug)] pub struct CommandSpec { pub program: String, diff --git a/codex-rs/core/src/tools/handlers/shell.rs b/codex-rs/core/src/tools/handlers/shell.rs index b7d6f7511d..fcd2f5b0c9 100644 --- a/codex-rs/core/src/tools/handlers/shell.rs +++ b/codex-rs/core/src/tools/handlers/shell.rs @@ -9,9 +9,11 @@ use crate::apply_patch::convert_apply_patch_to_protocol; use crate::codex::TurnContext; use crate::exec::ExecParams; use crate::exec_env::create_env; +use crate::exec_policy::create_approval_requirement_for_command; use crate::function_tool::FunctionCallError; use crate::is_safe_command::is_known_safe_command; use crate::protocol::ExecCommandSource; +use crate::sandboxing::SandboxPermissions; use crate::tools::context::ToolInvocation; use crate::tools::context::ToolOutput; use crate::tools::context::ToolPayload; @@ -302,6 +304,13 @@ impl ShellHandler { env: exec_params.env.clone(), with_escalated_permissions: exec_params.with_escalated_permissions, justification: exec_params.justification.clone(), + approval_requirement: create_approval_requirement_for_command( + &turn.exec_policy, + &exec_params.command, + turn.approval_policy, + &turn.sandbox_policy, + SandboxPermissions::from(exec_params.with_escalated_permissions.unwrap_or(false)), + ), }; let mut orchestrator = ToolOrchestrator::new(); let mut runtime = ShellRuntime::new(); diff --git a/codex-rs/core/src/tools/orchestrator.rs b/codex-rs/core/src/tools/orchestrator.rs index 878e48e8be..7e8e152f67 100644 --- a/codex-rs/core/src/tools/orchestrator.rs +++ b/codex-rs/core/src/tools/orchestrator.rs @@ -11,11 +11,13 @@ use crate::error::get_error_message_ui; use crate::exec::ExecToolCallOutput; use crate::sandboxing::SandboxManager; use crate::tools::sandboxing::ApprovalCtx; +use crate::tools::sandboxing::ApprovalRequirement; use crate::tools::sandboxing::ProvidesSandboxRetryData; use crate::tools::sandboxing::SandboxAttempt; use crate::tools::sandboxing::ToolCtx; use crate::tools::sandboxing::ToolError; use crate::tools::sandboxing::ToolRuntime; +use crate::tools::sandboxing::default_approval_requirement; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::ReviewDecision; @@ -49,40 +51,52 @@ impl ToolOrchestrator { let otel_cfg = codex_otel::otel_event_manager::ToolDecisionSource::Config; // 1) Approval - let needs_initial_approval = - tool.wants_initial_approval(req, approval_policy, &turn_ctx.sandbox_policy); let mut already_approved = false; - if needs_initial_approval { - let mut risk = None; - - if let Some(metadata) = req.sandbox_retry_data() { - risk = tool_ctx - .session - .assess_sandbox_command(turn_ctx, &tool_ctx.call_id, &metadata.command, None) - .await; + let requirement = tool.approval_requirement(req).unwrap_or_else(|| { + default_approval_requirement(approval_policy, &turn_ctx.sandbox_policy) + }); + match requirement { + ApprovalRequirement::Skip => { + otel.tool_decision(otel_tn, otel_ci, ReviewDecision::Approved, otel_cfg); } + ApprovalRequirement::Forbidden { reason } => { + return Err(ToolError::Rejected(reason)); + } + ApprovalRequirement::NeedsApproval { reason } => { + let mut risk = None; - let approval_ctx = ApprovalCtx { - session: tool_ctx.session, - turn: turn_ctx, - call_id: &tool_ctx.call_id, - retry_reason: None, - risk, - }; - let decision = tool.start_approval_async(req, approval_ctx).await; - - otel.tool_decision(otel_tn, otel_ci, decision, otel_user.clone()); - - match decision { - ReviewDecision::Denied | ReviewDecision::Abort => { - return Err(ToolError::Rejected("rejected by user".to_string())); + if let Some(metadata) = req.sandbox_retry_data() { + risk = tool_ctx + .session + .assess_sandbox_command( + turn_ctx, + &tool_ctx.call_id, + &metadata.command, + None, + ) + .await; } - ReviewDecision::Approved | ReviewDecision::ApprovedForSession => {} + + let approval_ctx = ApprovalCtx { + session: tool_ctx.session, + turn: turn_ctx, + call_id: &tool_ctx.call_id, + retry_reason: reason, + risk, + }; + let decision = tool.start_approval_async(req, approval_ctx).await; + + otel.tool_decision(otel_tn, otel_ci, decision, otel_user.clone()); + + match decision { + ReviewDecision::Denied | ReviewDecision::Abort => { + return Err(ToolError::Rejected("rejected by user".to_string())); + } + ReviewDecision::Approved | ReviewDecision::ApprovedForSession => {} + } + already_approved = true; } - already_approved = true; - } else { - otel.tool_decision(otel_tn, otel_ci, ReviewDecision::Approved, otel_cfg); } // 2) First attempt under the selected sandbox. diff --git a/codex-rs/core/src/tools/runtimes/shell.rs b/codex-rs/core/src/tools/runtimes/shell.rs index bf7ae7fa3b..d71c4498e6 100644 --- a/codex-rs/core/src/tools/runtimes/shell.rs +++ b/codex-rs/core/src/tools/runtimes/shell.rs @@ -4,13 +4,12 @@ Runtime: shell Executes shell requests under the orchestrator: asks for approval when needed, builds a CommandSpec, and runs it under the current SandboxAttempt. */ -use crate::command_safety::is_dangerous_command::requires_initial_appoval; use crate::exec::ExecToolCallOutput; -use crate::protocol::SandboxPolicy; use crate::sandboxing::execute_env; use crate::tools::runtimes::build_command_spec; use crate::tools::sandboxing::Approvable; use crate::tools::sandboxing::ApprovalCtx; +use crate::tools::sandboxing::ApprovalRequirement; use crate::tools::sandboxing::ProvidesSandboxRetryData; use crate::tools::sandboxing::SandboxAttempt; use crate::tools::sandboxing::SandboxRetryData; @@ -20,7 +19,6 @@ use crate::tools::sandboxing::ToolCtx; use crate::tools::sandboxing::ToolError; use crate::tools::sandboxing::ToolRuntime; use crate::tools::sandboxing::with_cached_approval; -use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::ReviewDecision; use futures::future::BoxFuture; use std::path::PathBuf; @@ -33,6 +31,7 @@ pub struct ShellRequest { pub env: std::collections::HashMap, pub with_escalated_permissions: Option, pub justification: Option, + pub approval_requirement: ApprovalRequirement, } impl ProvidesSandboxRetryData for ShellRequest { @@ -114,18 +113,8 @@ impl Approvable for ShellRuntime { }) } - fn wants_initial_approval( - &self, - req: &ShellRequest, - policy: AskForApproval, - sandbox_policy: &SandboxPolicy, - ) -> bool { - requires_initial_appoval( - policy, - sandbox_policy, - &req.command, - req.with_escalated_permissions.unwrap_or(false), - ) + fn approval_requirement(&self, req: &ShellRequest) -> Option { + Some(req.approval_requirement.clone()) } fn wants_escalated_first_attempt(&self, req: &ShellRequest) -> bool { diff --git a/codex-rs/core/src/tools/runtimes/unified_exec.rs b/codex-rs/core/src/tools/runtimes/unified_exec.rs index cddac1924e..5b18476bfc 100644 --- a/codex-rs/core/src/tools/runtimes/unified_exec.rs +++ b/codex-rs/core/src/tools/runtimes/unified_exec.rs @@ -1,4 +1,3 @@ -use crate::command_safety::is_dangerous_command::requires_initial_appoval; /* Runtime: unified exec @@ -10,6 +9,7 @@ use crate::error::SandboxErr; use crate::tools::runtimes::build_command_spec; use crate::tools::sandboxing::Approvable; use crate::tools::sandboxing::ApprovalCtx; +use crate::tools::sandboxing::ApprovalRequirement; use crate::tools::sandboxing::ProvidesSandboxRetryData; use crate::tools::sandboxing::SandboxAttempt; use crate::tools::sandboxing::SandboxRetryData; @@ -22,9 +22,7 @@ use crate::tools::sandboxing::with_cached_approval; use crate::unified_exec::UnifiedExecError; use crate::unified_exec::UnifiedExecSession; use crate::unified_exec::UnifiedExecSessionManager; -use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::ReviewDecision; -use codex_protocol::protocol::SandboxPolicy; use futures::future::BoxFuture; use std::collections::HashMap; use std::path::PathBuf; @@ -36,6 +34,7 @@ pub struct UnifiedExecRequest { pub env: HashMap, pub with_escalated_permissions: Option, pub justification: Option, + pub approval_requirement: ApprovalRequirement, } impl ProvidesSandboxRetryData for UnifiedExecRequest { @@ -65,6 +64,7 @@ impl UnifiedExecRequest { env: HashMap, with_escalated_permissions: Option, justification: Option, + approval_requirement: ApprovalRequirement, ) -> Self { Self { command, @@ -72,6 +72,7 @@ impl UnifiedExecRequest { env, with_escalated_permissions, justification, + approval_requirement, } } } @@ -129,18 +130,8 @@ impl Approvable for UnifiedExecRuntime<'_> { }) } - fn wants_initial_approval( - &self, - req: &UnifiedExecRequest, - policy: AskForApproval, - sandbox_policy: &SandboxPolicy, - ) -> bool { - requires_initial_appoval( - policy, - sandbox_policy, - &req.command, - req.with_escalated_permissions.unwrap_or(false), - ) + fn approval_requirement(&self, req: &UnifiedExecRequest) -> Option { + Some(req.approval_requirement.clone()) } fn wants_escalated_first_attempt(&self, req: &UnifiedExecRequest) -> bool { diff --git a/codex-rs/core/src/tools/sandboxing.rs b/codex-rs/core/src/tools/sandboxing.rs index da1c22b549..e694c7fbef 100644 --- a/codex-rs/core/src/tools/sandboxing.rs +++ b/codex-rs/core/src/tools/sandboxing.rs @@ -86,6 +86,37 @@ pub(crate) struct ApprovalCtx<'a> { pub risk: Option, } +// Specifies what tool orchestrator should do with a given tool call. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum ApprovalRequirement { + /// No approval required for this tool call + Skip, + /// Approval required for this tool call + NeedsApproval { reason: Option }, + /// Execution forbidden for this tool call + Forbidden { reason: String }, +} + +/// - Never, OnFailure: do not ask +/// - OnRequest: ask unless sandbox policy is DangerFullAccess +/// - UnlessTrusted: always ask +pub(crate) fn default_approval_requirement( + policy: AskForApproval, + sandbox_policy: &SandboxPolicy, +) -> ApprovalRequirement { + let needs_approval = match policy { + AskForApproval::Never | AskForApproval::OnFailure => false, + AskForApproval::OnRequest => !matches!(sandbox_policy, SandboxPolicy::DangerFullAccess), + AskForApproval::UnlessTrusted => true, + }; + + if needs_approval { + ApprovalRequirement::NeedsApproval { reason: None } + } else { + ApprovalRequirement::Skip + } +} + pub(crate) trait Approvable { type ApprovalKey: Hash + Eq + Clone + Debug + Serialize; @@ -106,22 +137,11 @@ pub(crate) trait Approvable { matches!(policy, AskForApproval::Never) } - /// Decide whether an initial user approval should be requested before the - /// first attempt. Defaults to the orchestrator's behavior (pre‑refactor): - /// - Never, OnFailure: do not ask - /// - OnRequest: ask unless sandbox policy is DangerFullAccess - /// - UnlessTrusted: always ask - fn wants_initial_approval( - &self, - _req: &Req, - policy: AskForApproval, - sandbox_policy: &SandboxPolicy, - ) -> bool { - match policy { - AskForApproval::Never | AskForApproval::OnFailure => false, - AskForApproval::OnRequest => !matches!(sandbox_policy, SandboxPolicy::DangerFullAccess), - AskForApproval::UnlessTrusted => true, - } + /// Override the default approval requirement. Return `Some(_)` to specify + /// a custom requirement, or `None` to fall back to + /// policy-based default. + fn approval_requirement(&self, _req: &Req) -> Option { + None } /// Decide we can request an approval for no-sandbox execution. diff --git a/codex-rs/core/src/unified_exec/session_manager.rs b/codex-rs/core/src/unified_exec/session_manager.rs index f6dce8644c..93340bb2d4 100644 --- a/codex-rs/core/src/unified_exec/session_manager.rs +++ b/codex-rs/core/src/unified_exec/session_manager.rs @@ -11,10 +11,12 @@ use crate::codex::TurnContext; use crate::exec::ExecToolCallOutput; use crate::exec::StreamOutput; use crate::exec_env::create_env; +use crate::exec_policy::create_approval_requirement_for_command; use crate::protocol::BackgroundEventEvent; use crate::protocol::EventMsg; use crate::protocol::ExecCommandSource; use crate::sandboxing::ExecEnv; +use crate::sandboxing::SandboxPermissions; use crate::tools::events::ToolEmitter; use crate::tools::events::ToolEventCtx; use crate::tools::events::ToolEventFailure; @@ -449,6 +451,13 @@ impl UnifiedExecSessionManager { create_env(&context.turn.shell_environment_policy), with_escalated_permissions, justification, + create_approval_requirement_for_command( + &context.turn.exec_policy, + command, + context.turn.approval_policy, + &context.turn.sandbox_policy, + SandboxPermissions::from(with_escalated_permissions.unwrap_or(false)), + ), ); let tool_ctx = ToolCtx { session: context.session.as_ref(), diff --git a/codex-rs/core/tests/suite/exec_policy.rs b/codex-rs/core/tests/suite/exec_policy.rs new file mode 100644 index 0000000000..bca917a252 --- /dev/null +++ b/codex-rs/core/tests/suite/exec_policy.rs @@ -0,0 +1,101 @@ +#![allow(clippy::unwrap_used, clippy::expect_used)] + +use anyhow::Result; +use codex_core::protocol::AskForApproval; +use codex_core::protocol::EventMsg; +use codex_core::protocol::Op; +use codex_core::protocol::SandboxPolicy; +use codex_protocol::config_types::ReasoningSummary; +use codex_protocol::user_input::UserInput; +use core_test_support::responses::ev_assistant_message; +use core_test_support::responses::ev_completed; +use core_test_support::responses::ev_function_call; +use core_test_support::responses::ev_response_created; +use core_test_support::responses::mount_sse_once; +use core_test_support::responses::sse; +use core_test_support::responses::start_mock_server; +use core_test_support::test_codex::test_codex; +use core_test_support::wait_for_event; +use serde_json::json; +use std::fs; + +#[tokio::test] +async fn execpolicy_blocks_shell_invocation() -> Result<()> { + let mut builder = test_codex().with_config(|config| { + let policy_path = config.codex_home.join("policy").join("policy.codexpolicy"); + fs::create_dir_all( + policy_path + .parent() + .expect("policy directory must have a parent"), + ) + .expect("create policy directory"); + fs::write( + &policy_path, + r#"prefix_rule(pattern=["echo"], decision="forbidden")"#, + ) + .expect("write policy file"); + }); + let server = start_mock_server().await; + let test = builder.build(&server).await?; + + let call_id = "shell-forbidden"; + let args = json!({ + "command": ["echo", "blocked"], + "timeout_ms": 1_000, + }); + + mount_sse_once( + &server, + sse(vec![ + ev_response_created("resp-1"), + ev_function_call(call_id, "shell", &serde_json::to_string(&args)?), + ev_completed("resp-1"), + ]), + ) + .await; + mount_sse_once( + &server, + sse(vec![ + ev_assistant_message("msg-1", "done"), + ev_completed("resp-2"), + ]), + ) + .await; + + let session_model = test.session_configured.model.clone(); + test.codex + .submit(Op::UserTurn { + items: vec![UserInput::Text { + text: "run shell command".into(), + }], + final_output_json_schema: None, + cwd: test.cwd_path().to_path_buf(), + approval_policy: AskForApproval::Never, + sandbox_policy: SandboxPolicy::DangerFullAccess, + model: session_model, + effort: None, + summary: ReasoningSummary::Auto, + }) + .await?; + + let EventMsg::ExecCommandEnd(end) = wait_for_event(&test.codex, |event| { + matches!(event, EventMsg::ExecCommandEnd(_)) + }) + .await + else { + unreachable!() + }; + wait_for_event(&test.codex, |event| { + matches!(event, EventMsg::TaskComplete(_)) + }) + .await; + + assert!( + end.aggregated_output + .contains("execpolicy forbids this command"), + "unexpected output: {}", + end.aggregated_output + ); + + Ok(()) +} diff --git a/codex-rs/core/tests/suite/mod.rs b/codex-rs/core/tests/suite/mod.rs index ef248901e5..60b828b1c3 100644 --- a/codex-rs/core/tests/suite/mod.rs +++ b/codex-rs/core/tests/suite/mod.rs @@ -28,6 +28,7 @@ mod compact_remote; mod compact_resume_fork; mod deprecation_notice; mod exec; +mod exec_policy; mod fork_conversation; mod grep_files; mod items; diff --git a/codex-rs/execpolicy2/README.md b/codex-rs/execpolicy2/README.md index 8cf5302fb6..d77ec3838d 100644 --- a/codex-rs/execpolicy2/README.md +++ b/codex-rs/execpolicy2/README.md @@ -5,7 +5,7 @@ - This release covers only the prefix-rule subset of the planned execpolicy v2 language; a richer language will follow. - Tokens are matched in order; any `pattern` element may be a list to denote alternatives. `decision` defaults to `allow`; valid values: `allow`, `prompt`, `forbidden`. - `match` / `not_match` supply example invocations that are validated at load time (think of them as unit tests); examples can be token arrays or strings (strings are tokenized with `shlex`). -- The CLI always prints the JSON serialization of the evaluation result (whether a match or not). +- The CLI always prints the JSON serialization of the evaluation result. ## Policy shapes - Prefix rules use Starlark syntax: @@ -18,6 +18,20 @@ prefix_rule( ) ``` +## CLI +- Provide one or more policy files (for example `src/default.codexpolicy`) to check a command: +```bash +cargo run -p codex-execpolicy2 -- check --policy path/to/policy.codexpolicy git status +``` +- Pass multiple `--policy` flags to merge rules, evaluated in the order provided: +```bash +cargo run -p codex-execpolicy2 -- check --policy base.codexpolicy --policy overrides.codexpolicy git status +``` +- Output is JSON by default; pass `--pretty` for pretty-printed JSON +- Example outcomes: + - Match: `{"match": { ... "decision": "allow" ... }}` + - No match: `"noMatch"` + ## Response shapes - Match: ```json @@ -43,17 +57,3 @@ prefix_rule( - `matchedRules` lists every rule whose prefix matched the command; `matchedPrefix` is the exact prefix that matched. - The effective `decision` is the strictest severity across all matches (`forbidden` > `prompt` > `allow`). - -## CLI -- Provide one or more policy files (for example `src/default.codexpolicy`) to check a command: -```bash -cargo run -p codex-execpolicy2 -- check --policy path/to/policy.codexpolicy git status -``` -- Pass multiple `--policy` flags to merge rules, evaluated in the order provided: -```bash -cargo run -p codex-execpolicy2 -- check --policy base.codexpolicy --policy overrides.codexpolicy git status -``` -- Output is newline-delimited JSON by default; pass `--pretty` for pretty-printed JSON if desired. -- Example outcomes: - - Match: `{"match": { ... "decision": "allow" ... }}` - - No match: `"noMatch"` diff --git a/codex-rs/execpolicy2/src/policy.rs b/codex-rs/execpolicy2/src/policy.rs index 12416b050b..a543e5c4b0 100644 --- a/codex-rs/execpolicy2/src/policy.rs +++ b/codex-rs/execpolicy2/src/policy.rs @@ -15,6 +15,10 @@ impl Policy { Self { rules_by_program } } + pub fn empty() -> Self { + Self::new(MultiMap::new()) + } + pub fn rules(&self) -> &MultiMap { &self.rules_by_program } diff --git a/codex-rs/tui/src/chatwidget/agent.rs b/codex-rs/tui/src/chatwidget/agent.rs index bf15b6c4ad..0abddac50d 100644 --- a/codex-rs/tui/src/chatwidget/agent.rs +++ b/codex-rs/tui/src/chatwidget/agent.rs @@ -4,6 +4,9 @@ use codex_core::CodexConversation; use codex_core::ConversationManager; use codex_core::NewConversation; use codex_core::config::Config; +use codex_core::protocol::ErrorEvent; +use codex_core::protocol::Event; +use codex_core::protocol::EventMsg; use codex_core::protocol::Op; use tokio::sync::mpsc::UnboundedSender; use tokio::sync::mpsc::unbounded_channel; @@ -28,9 +31,19 @@ pub(crate) fn spawn_agent( session_configured, } = match server.new_conversation(config).await { Ok(v) => v, - Err(e) => { - // TODO: surface this error to the user. - tracing::error!("failed to initialize codex: {e}"); + #[allow(clippy::print_stderr)] + Err(err) => { + let message = err.to_string(); + eprintln!("{message}"); + app_event_tx_clone.send(AppEvent::CodexEvent(Event { + id: "".to_string(), + msg: EventMsg::Error(ErrorEvent { + message, + http_status_code: None, + }), + })); + app_event_tx_clone.send(AppEvent::ExitRequest); + tracing::error!("failed to initialize codex: {err}"); return; } }; From 92e304673374c5da74673ee0a80ca3169897830e Mon Sep 17 00:00:00 2001 From: pakrym-oai Date: Wed, 19 Nov 2025 16:56:37 -0800 Subject: [PATCH 09/42] Single pass truncation (#6914) --- codex-rs/core/src/truncate.rs | 114 +++++++++++++++++++++++----------- 1 file changed, 79 insertions(+), 35 deletions(-) diff --git a/codex-rs/core/src/truncate.rs b/codex-rs/core/src/truncate.rs index 7756b0c09a..bf883c0612 100644 --- a/codex-rs/core/src/truncate.rs +++ b/codex-rs/core/src/truncate.rs @@ -185,6 +185,7 @@ fn truncate_with_byte_estimate(s: &str, policy: TruncationPolicy) -> String { if s.is_empty() { return String::new(); } + let total_chars = s.chars().count(); let max_bytes = policy.byte_budget(); @@ -204,24 +205,55 @@ fn truncate_with_byte_estimate(s: &str, policy: TruncationPolicy) -> String { let total_bytes = s.len(); let (left_budget, right_budget) = split_budget(max_bytes); - let prefix_end = pick_prefix_end(s, left_budget); - let mut suffix_start = pick_suffix_start(s, right_budget); - if suffix_start < prefix_end { - suffix_start = prefix_end; - } - let left_chars = s[..prefix_end].chars().count(); - let right_chars = s[suffix_start..].chars().count(); - let removed_chars = total_chars - .saturating_sub(left_chars) - .saturating_sub(right_chars); + let (removed_chars, left, right) = split_string(s, left_budget, right_budget); let marker = format_truncation_marker( policy, removed_units_for_source(policy, total_bytes.saturating_sub(max_bytes), removed_chars), ); - assemble_truncated_output(&s[..prefix_end], &s[suffix_start..], &marker) + assemble_truncated_output(left, right, &marker) +} + +fn split_string(s: &str, beginning_bytes: usize, end_bytes: usize) -> (usize, &str, &str) { + if s.is_empty() { + return (0, "", ""); + } + + let len = s.len(); + let tail_start_target = len.saturating_sub(end_bytes); + let mut prefix_end = 0usize; + let mut suffix_start = len; + let mut removed_chars = 0usize; + let mut suffix_started = false; + + for (idx, ch) in s.char_indices() { + let char_end = idx + ch.len_utf8(); + if char_end <= beginning_bytes { + prefix_end = char_end; + continue; + } + + if idx >= tail_start_target { + if !suffix_started { + suffix_start = idx; + suffix_started = true; + } + continue; + } + + removed_chars = removed_chars.saturating_add(1); + } + + if suffix_start < prefix_end { + suffix_start = prefix_end; + } + + let before = &s[..prefix_end]; + let after = &s[suffix_start..]; + + (removed_chars, before, after) } fn format_truncation_marker(policy: TruncationPolicy, removed_count: u64) -> String { @@ -270,42 +302,54 @@ fn approx_tokens_from_byte_count(bytes: usize) -> u64 { / (APPROX_BYTES_PER_TOKEN as u64) } -fn truncate_on_boundary(input: &str, max_len: usize) -> &str { - if input.len() <= max_len { - return input; - } - let mut end = max_len; - while end > 0 && !input.is_char_boundary(end) { - end -= 1; - } - &input[..end] -} - -fn pick_prefix_end(s: &str, left_budget: usize) -> usize { - truncate_on_boundary(s, left_budget).len() -} - -fn pick_suffix_start(s: &str, right_budget: usize) -> usize { - let start_tail = s.len().saturating_sub(right_budget); - let mut idx = start_tail.min(s.len()); - while idx < s.len() && !s.is_char_boundary(idx) { - idx += 1; - } - idx -} - #[cfg(test)] mod tests { use super::TruncationPolicy; use super::approx_token_count; use super::formatted_truncate_text; + use super::split_string; use super::truncate_function_output_items_with_policy; use super::truncate_text; use super::truncate_with_token_budget; use codex_protocol::models::FunctionCallOutputContentItem; use pretty_assertions::assert_eq; + #[test] + fn split_string_works() { + assert_eq!(split_string("hello world", 5, 5), (1, "hello", "world")); + assert_eq!(split_string("abc", 0, 0), (3, "", "")); + } + + #[test] + fn split_string_handles_empty_string() { + assert_eq!(split_string("", 4, 4), (0, "", "")); + } + + #[test] + fn split_string_only_keeps_prefix_when_tail_budget_is_zero() { + assert_eq!(split_string("abcdef", 3, 0), (3, "abc", "")); + } + + #[test] + fn split_string_only_keeps_suffix_when_prefix_budget_is_zero() { + assert_eq!(split_string("abcdef", 0, 3), (3, "", "def")); + } + + #[test] + fn split_string_handles_overlapping_budgets_without_removal() { + assert_eq!(split_string("abcdef", 4, 4), (0, "abcd", "ef")); + } + + #[test] + fn split_string_respects_utf8_boundaries() { + assert_eq!(split_string("😀abc😀", 5, 5), (1, "😀a", "c😀")); + + assert_eq!(split_string("😀😀😀😀😀", 1, 1), (5, "", "")); + assert_eq!(split_string("😀😀😀😀😀", 7, 7), (3, "😀", "😀")); + assert_eq!(split_string("😀😀😀😀😀", 8, 8), (1, "😀😀", "😀😀")); + } + #[test] fn truncate_bytes_less_than_placeholder_returns_placeholder() { let content = "example output"; From 74a75679d993908ee90d7194aec02386d507c8de Mon Sep 17 00:00:00 2001 From: zhao-oai Date: Wed, 19 Nov 2025 16:57:27 -0800 Subject: [PATCH 10/42] update execpolicy quickstart readme (#6952) --- README.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index e34afbd288..4fed276405 100644 --- a/README.md +++ b/README.md @@ -69,12 +69,12 @@ Codex can access MCP servers. To configure them, refer to the [config docs](./do Codex CLI supports a rich set of configuration options, with preferences stored in `~/.codex/config.toml`. For full configuration options, see [Configuration](./docs/config.md). -### Execpolicy quickstart +### Execpolicy Quickstart Codex can enforce your own rules-based execution policy before it runs shell commands. 1. Create a policy directory: `mkdir -p ~/.codex/policy`. -2. Create one or more `.codexpolicy` files into that folder. Codex automatically loads every `.codexpolicy` file in there on startup. +2. Create one or more `.codexpolicy` files in that folder. Codex automatically loads every `.codexpolicy` file in there on startup. 3. Write `prefix_rule` entries to describe the commands you want to allow, prompt, or block: ```starlark @@ -87,14 +87,12 @@ prefix_rule( ``` - `pattern` is a list of shell tokens, evaluated from left to right; wrap tokens in a nested list to express alternatives (e.g., match both `push` and `fetch`). -- `decision` sets the severity; Codex picks the strictest decision when multiple rules match. +- `decision` sets the severity; Codex picks the strictest decision when multiple rules match (forbidden > prompt > allow). - `match` and `not_match` act as (optional) unit tests. Codex validates them when it loads your policy, so you get feedback if an example has unexpected behavior. In this example rule, if Codex wants to run commands with the prefix `git push` or `git fetch`, it will first ask for user approval. -Note: If Codex wants to run a command that matches with multiple rules, it will use the strictest decision among the matched rules (forbidden > prompt > allow). - -Use the [`execpolicy2` CLI](./codex-rs/execpolicy2/README.md) to preview decisions before you save a rule: +Use [`execpolicy2` CLI](./codex-rs/execpolicy2/README.md) to preview decisions for policy files: ```shell cargo run -p codex-execpolicy2 -- check --policy ~/.codex/policy/default.codexpolicy git push origin main From 6d67b8b2832e614086bb9af41fe337f668507f5a Mon Sep 17 00:00:00 2001 From: Ahmed Ibrahim Date: Wed, 19 Nov 2025 17:17:04 -0800 Subject: [PATCH 11/42] stop model migration screen after first time. (#6954) it got serialized wrong. --- codex-rs/core/src/config/types.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/codex-rs/core/src/config/types.rs b/codex-rs/core/src/config/types.rs index b7f16a9028..8f5077410e 100644 --- a/codex-rs/core/src/config/types.rs +++ b/codex-rs/core/src/config/types.rs @@ -379,6 +379,7 @@ pub struct Notice { /// Tracks whether the user has seen the model migration prompt pub hide_gpt5_1_migration_prompt: Option, /// Tracks whether the user has seen the gpt-5.1-codex-max migration prompt + #[serde(rename = "hide_gpt-5.1-codex-max_migration_prompt")] pub hide_gpt_5_1_codex_max_migration_prompt: Option, } From 72a1453ac5b4ee60f19876542b7d7203e83cde62 Mon Sep 17 00:00:00 2001 From: Celia Chen Date: Wed, 19 Nov 2025 17:26:14 -0800 Subject: [PATCH 12/42] Revert "[core] add optional status_code to error events (#6865)" (#6955) This reverts commit c2ec477d939af2a275255666b37fe9b0acd7492a. # External (non-OpenAI) Pull Request Requirements Before opening this Pull Request, please read the dedicated "Contributing" markdown file or your PR may be closed: https://github.com/openai/codex/blob/main/docs/contributing.md If your PR conforms to our contribution guidelines, replace this text with a detailed and high quality description of your changes. Include a link to a bug report or enhancement request. --- codex-rs/core/src/codex.rs | 13 ++-- codex-rs/core/src/compact.rs | 14 ++-- codex-rs/core/src/compact_remote.rs | 7 +- codex-rs/core/src/error.rs | 71 ------------------- codex-rs/docs/protocol_v1.md | 2 +- .../src/event_processor_with_human_output.rs | 4 +- .../tests/event_processor_with_json_output.rs | 3 - codex-rs/protocol/src/protocol.rs | 4 -- codex-rs/tui/src/chatwidget.rs | 6 +- codex-rs/tui/src/chatwidget/agent.rs | 5 +- codex-rs/tui/src/chatwidget/tests.rs | 1 - 11 files changed, 25 insertions(+), 105 deletions(-) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 897f1f2944..c793eff352 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -66,7 +66,6 @@ use crate::context_manager::ContextManager; use crate::environment_context::EnvironmentContext; use crate::error::CodexErr; use crate::error::Result as CodexResult; -use crate::error::http_status_code_value; #[cfg(test)] use crate::exec::StreamOutput; use crate::mcp::auth::compute_auth_statuses; @@ -80,6 +79,7 @@ use crate::protocol::ApplyPatchApprovalRequestEvent; use crate::protocol::AskForApproval; use crate::protocol::BackgroundEventEvent; use crate::protocol::DeprecationNoticeEvent; +use crate::protocol::ErrorEvent; use crate::protocol::Event; use crate::protocol::EventMsg; use crate::protocol::ExecApprovalRequestEvent; @@ -134,7 +134,6 @@ use codex_protocol::user_input::UserInput; use codex_utils_readiness::Readiness; use codex_utils_readiness::ReadinessFlag; use codex_utils_tokenizer::warm_model_cache; -use reqwest::StatusCode; /// The high-level interface to the Codex system. /// It operates as a queue pair where you send submissions and receive events. @@ -1197,11 +1196,9 @@ impl Session { &self, turn_context: &TurnContext, message: impl Into, - http_status_code: Option, ) { let event = EventMsg::StreamError(StreamErrorEvent { message: message.into(), - http_status_code: http_status_code_value(http_status_code), }); self.send_event(turn_context, event).await; } @@ -1693,7 +1690,6 @@ mod handlers { id: sub_id.clone(), msg: EventMsg::Error(ErrorEvent { message: "Failed to shutdown rollout recorder".to_string(), - http_status_code: None, }), }; sess.send_event_raw(event).await; @@ -1948,8 +1944,10 @@ pub(crate) async fn run_task( } Err(e) => { info!("Turn error: {e:#}"); - sess.send_event(&turn_context, EventMsg::Error(e.to_error_event(None))) - .await; + let event = EventMsg::Error(ErrorEvent { + message: e.to_string(), + }); + sess.send_event(&turn_context, event).await; // let the user continue the conversation break; } @@ -2073,7 +2071,6 @@ async fn run_turn( sess.notify_stream_error( &turn_context, format!("Reconnecting... {retries}/{max_retries}"), - e.http_status_code(), ) .await; diff --git a/codex-rs/core/src/compact.rs b/codex-rs/core/src/compact.rs index 67b5b9342d..8c38f93937 100644 --- a/codex-rs/core/src/compact.rs +++ b/codex-rs/core/src/compact.rs @@ -10,6 +10,7 @@ use crate::error::Result as CodexResult; use crate::features::Feature; use crate::protocol::AgentMessageEvent; use crate::protocol::CompactedItem; +use crate::protocol::ErrorEvent; use crate::protocol::EventMsg; use crate::protocol::TaskStartedEvent; use crate::protocol::TurnContextItem; @@ -127,8 +128,10 @@ async fn run_compact_task_inner( continue; } sess.set_total_tokens_full(turn_context.as_ref()).await; - sess.send_event(&turn_context, EventMsg::Error(e.to_error_event(None))) - .await; + let event = EventMsg::Error(ErrorEvent { + message: e.to_string(), + }); + sess.send_event(&turn_context, event).await; return; } Err(e) => { @@ -138,14 +141,15 @@ async fn run_compact_task_inner( sess.notify_stream_error( turn_context.as_ref(), format!("Reconnecting... {retries}/{max_retries}"), - e.http_status_code(), ) .await; tokio::time::sleep(delay).await; continue; } else { - sess.send_event(&turn_context, EventMsg::Error(e.to_error_event(None))) - .await; + let event = EventMsg::Error(ErrorEvent { + message: e.to_string(), + }); + sess.send_event(&turn_context, event).await; return; } } diff --git a/codex-rs/core/src/compact_remote.rs b/codex-rs/core/src/compact_remote.rs index aa32942dce..0d2e0f138d 100644 --- a/codex-rs/core/src/compact_remote.rs +++ b/codex-rs/core/src/compact_remote.rs @@ -6,6 +6,7 @@ use crate::codex::TurnContext; use crate::error::Result as CodexResult; use crate::protocol::AgentMessageEvent; use crate::protocol::CompactedItem; +use crate::protocol::ErrorEvent; use crate::protocol::EventMsg; use crate::protocol::RolloutItem; use crate::protocol::TaskStartedEvent; @@ -29,8 +30,10 @@ pub(crate) async fn run_remote_compact_task(sess: Arc, turn_context: Ar async fn run_remote_compact_task_inner(sess: &Arc, turn_context: &Arc) { if let Err(err) = run_remote_compact_task_inner_impl(sess, turn_context).await { - let event = err.to_error_event(Some("Error running remote compact task".to_string())); - sess.send_event(turn_context, EventMsg::Error(event)).await; + let event = EventMsg::Error(ErrorEvent { + message: format!("Error running remote compact task: {err}"), + }); + sess.send_event(turn_context, event).await; } } diff --git a/codex-rs/core/src/error.rs b/codex-rs/core/src/error.rs index d20d5cfb46..b2027dc940 100644 --- a/codex-rs/core/src/error.rs +++ b/codex-rs/core/src/error.rs @@ -10,7 +10,6 @@ use chrono::Local; use chrono::Utc; use codex_async_utils::CancelErr; use codex_protocol::ConversationId; -use codex_protocol::protocol::ErrorEvent; use codex_protocol::protocol::RateLimitSnapshot; use reqwest::StatusCode; use serde_json; @@ -431,37 +430,6 @@ impl CodexErr { pub fn downcast_ref(&self) -> Option<&T> { (self as &dyn std::any::Any).downcast_ref::() } - - pub fn http_status_code(&self) -> Option { - match self { - CodexErr::UnexpectedStatus(err) => Some(err.status), - CodexErr::RetryLimit(err) => Some(err.status), - CodexErr::UsageLimitReached(_) | CodexErr::UsageNotIncluded => { - Some(StatusCode::TOO_MANY_REQUESTS) - } - CodexErr::InternalServerError => Some(StatusCode::INTERNAL_SERVER_ERROR), - CodexErr::ResponseStreamFailed(err) => err.source.status(), - CodexErr::ConnectionFailed(err) => err.source.status(), - _ => None, - } - } - - pub fn to_error_event(&self, message_prefix: Option) -> ErrorEvent { - let error_message = self.to_string(); - let message: String = match message_prefix { - Some(prefix) => format!("{prefix}: {error_message}"), - None => error_message, - }; - - ErrorEvent { - message, - http_status_code: http_status_code_value(self.http_status_code()), - } - } -} - -pub fn http_status_code_value(http_status_code: Option) -> Option { - http_status_code.as_ref().map(StatusCode::as_u16) } pub fn get_error_message_ui(e: &CodexErr) -> String { @@ -807,43 +775,4 @@ mod tests { assert_eq!(err.to_string(), expected); }); } - - #[test] - fn error_event_includes_http_status_code_when_available() { - let err = CodexErr::UnexpectedStatus(UnexpectedResponseError { - status: StatusCode::BAD_REQUEST, - body: "oops".to_string(), - request_id: Some("req-1".to_string()), - }); - let event = err.to_error_event(None); - - assert_eq!( - event.message, - "unexpected status 400 Bad Request: oops, request id: req-1" - ); - assert_eq!( - event.http_status_code, - Some(StatusCode::BAD_REQUEST.as_u16()) - ); - } - - #[test] - fn error_event_omits_http_status_code_when_unknown() { - let event = CodexErr::Fatal("boom".to_string()).to_error_event(None); - - assert_eq!(event.message, "Fatal error: boom"); - assert_eq!(event.http_status_code, None); - } - - #[test] - fn error_event_applies_message_wrapper() { - let event = CodexErr::Fatal("boom".to_string()) - .to_error_event(Some("Error running remote compact task".to_string())); - - assert_eq!( - event.message, - "Error running remote compact task: Fatal error: boom" - ); - assert_eq!(event.http_status_code, None); - } } diff --git a/codex-rs/docs/protocol_v1.md b/codex-rs/docs/protocol_v1.md index c6af7c9260..f6405c9bc3 100644 --- a/codex-rs/docs/protocol_v1.md +++ b/codex-rs/docs/protocol_v1.md @@ -72,7 +72,7 @@ For complete documentation of the `Op` and `EventMsg` variants, refer to [protoc - `EventMsg::AgentMessage` – Messages from the `Model` - `EventMsg::ExecApprovalRequest` – Request approval from user to execute a command - `EventMsg::TaskComplete` – A task completed successfully - - `EventMsg::Error` – A task stopped with an error (includes an optional `http_status_code` when available) + - `EventMsg::Error` – A task stopped with an error - `EventMsg::Warning` – A non-fatal warning that the client should surface to the user - `EventMsg::TurnComplete` – Contains a `response_id` bookmark for last `response_id` executed by the task. This can be used to continue the task at a later point in time, perhaps with additional user input. diff --git a/codex-rs/exec/src/event_processor_with_human_output.rs b/codex-rs/exec/src/event_processor_with_human_output.rs index 86af82118b..2d550fea46 100644 --- a/codex-rs/exec/src/event_processor_with_human_output.rs +++ b/codex-rs/exec/src/event_processor_with_human_output.rs @@ -161,7 +161,7 @@ impl EventProcessor for EventProcessorWithHumanOutput { fn process_event(&mut self, event: Event) -> CodexStatus { let Event { id: _, msg } = event; match msg { - EventMsg::Error(ErrorEvent { message, .. }) => { + EventMsg::Error(ErrorEvent { message }) => { let prefix = "ERROR:".style(self.red); ts_msg!(self, "{prefix} {message}"); } @@ -221,7 +221,7 @@ impl EventProcessor for EventProcessorWithHumanOutput { EventMsg::BackgroundEvent(BackgroundEventEvent { message }) => { ts_msg!(self, "{}", message.style(self.dimmed)); } - EventMsg::StreamError(StreamErrorEvent { message, .. }) => { + EventMsg::StreamError(StreamErrorEvent { message }) => { ts_msg!(self, "{}", message.style(self.dimmed)); } EventMsg::TaskStarted(_) => { diff --git a/codex-rs/exec/tests/event_processor_with_json_output.rs b/codex-rs/exec/tests/event_processor_with_json_output.rs index 444d32304a..5053f61921 100644 --- a/codex-rs/exec/tests/event_processor_with_json_output.rs +++ b/codex-rs/exec/tests/event_processor_with_json_output.rs @@ -539,7 +539,6 @@ fn error_event_produces_error() { "e1", EventMsg::Error(codex_core::protocol::ErrorEvent { message: "boom".to_string(), - http_status_code: Some(500), }), )); assert_eq!( @@ -579,7 +578,6 @@ fn stream_error_event_produces_error() { "e1", EventMsg::StreamError(codex_core::protocol::StreamErrorEvent { message: "retrying".to_string(), - http_status_code: Some(500), }), )); assert_eq!( @@ -598,7 +596,6 @@ fn error_followed_by_task_complete_produces_turn_failed() { "e1", EventMsg::Error(ErrorEvent { message: "boom".to_string(), - http_status_code: Some(500), }), ); assert_eq!( diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index 0158ce9cfe..1825d7636b 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -686,8 +686,6 @@ pub struct ExitedReviewModeEvent { #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, TS)] pub struct ErrorEvent { pub message: String, - #[serde(default)] - pub http_status_code: Option, } #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, TS)] @@ -1365,8 +1363,6 @@ pub struct UndoCompletedEvent { #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, TS)] pub struct StreamErrorEvent { pub message: String, - #[serde(default)] - pub http_status_code: Option, } #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, TS)] diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 3a79330b74..987a956fe6 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -1654,7 +1654,7 @@ impl ChatWidget { self.on_rate_limit_snapshot(ev.rate_limits); } EventMsg::Warning(WarningEvent { message }) => self.on_warning(message), - EventMsg::Error(ErrorEvent { message, .. }) => self.on_error(message), + EventMsg::Error(ErrorEvent { message }) => self.on_error(message), EventMsg::McpStartupUpdate(ev) => self.on_mcp_startup_update(ev), EventMsg::McpStartupComplete(ev) => self.on_mcp_startup_complete(ev), EventMsg::TurnAborted(ev) => match ev.reason { @@ -1697,9 +1697,7 @@ impl ChatWidget { } EventMsg::UndoStarted(ev) => self.on_undo_started(ev), EventMsg::UndoCompleted(ev) => self.on_undo_completed(ev), - EventMsg::StreamError(StreamErrorEvent { message, .. }) => { - self.on_stream_error(message) - } + EventMsg::StreamError(StreamErrorEvent { message }) => self.on_stream_error(message), EventMsg::UserMessage(ev) => { if from_replay { self.on_user_message_event(ev); diff --git a/codex-rs/tui/src/chatwidget/agent.rs b/codex-rs/tui/src/chatwidget/agent.rs index 0abddac50d..3c326fed86 100644 --- a/codex-rs/tui/src/chatwidget/agent.rs +++ b/codex-rs/tui/src/chatwidget/agent.rs @@ -37,10 +37,7 @@ pub(crate) fn spawn_agent( eprintln!("{message}"); app_event_tx_clone.send(AppEvent::CodexEvent(Event { id: "".to_string(), - msg: EventMsg::Error(ErrorEvent { - message, - http_status_code: None, - }), + msg: EventMsg::Error(ErrorEvent { message }), })); app_event_tx_clone.send(AppEvent::ExitRequest); tracing::error!("failed to initialize codex: {err}"); diff --git a/codex-rs/tui/src/chatwidget/tests.rs b/codex-rs/tui/src/chatwidget/tests.rs index a4cef22418..d60a20b8f1 100644 --- a/codex-rs/tui/src/chatwidget/tests.rs +++ b/codex-rs/tui/src/chatwidget/tests.rs @@ -2596,7 +2596,6 @@ fn stream_error_updates_status_indicator() { id: "sub-1".into(), msg: EventMsg::StreamError(StreamErrorEvent { message: msg.to_string(), - http_status_code: None, }), }); From fb9849e1e3252048638ceef5715d8cfbfece34e3 Mon Sep 17 00:00:00 2001 From: zhao-oai Date: Wed, 19 Nov 2025 19:14:10 -0800 Subject: [PATCH 13/42] migrating execpolicy -> execpolicy-legacy and execpolicy2 -> execpolicy (#6956) --- codex-rs/Cargo.lock | 32 +-- codex-rs/Cargo.toml | 4 +- codex-rs/core/Cargo.toml | 2 +- codex-rs/core/src/codex.rs | 6 +- codex-rs/core/src/exec_policy.rs | 10 +- codex-rs/exec-server/src/posix.rs | 2 +- codex-rs/execpolicy-legacy/Cargo.toml | 34 +++ codex-rs/execpolicy-legacy/README.md | 183 ++++++++++++++++ .../build.rs | 0 .../src/arg_matcher.rs | 0 .../src/arg_resolver.rs | 0 .../src/arg_type.rs | 0 .../src/default.policy | 0 codex-rs/execpolicy-legacy/src/error.rs | 96 +++++++++ .../src/exec_call.rs | 0 .../src/execv_checker.rs | 0 codex-rs/execpolicy-legacy/src/lib.rs | 45 ++++ codex-rs/execpolicy-legacy/src/main.rs | 169 +++++++++++++++ .../src/opt.rs | 0 codex-rs/execpolicy-legacy/src/policy.rs | 103 +++++++++ .../src/policy_parser.rs | 0 .../src/program.rs | 0 .../src/sed_command.rs | 0 .../src/valid_exec.rs | 0 .../tests/all.rs | 0 .../tests/suite/bad.rs | 4 +- .../tests/suite/cp.rs | 22 +- .../tests/suite/good.rs | 4 +- .../tests/suite/head.rs | 24 +-- .../tests/suite/literal.rs | 18 +- .../tests/suite/ls.rs | 22 +- .../tests/suite/mod.rs | 0 .../tests/suite/parse_sed_command.rs | 4 +- .../tests/suite/pwd.rs | 18 +- .../tests/suite/sed.rs | 24 +-- codex-rs/execpolicy/Cargo.toml | 22 +- codex-rs/execpolicy/README.md | 204 ++++-------------- .../examples/example.codexpolicy | 0 .../src/decision.rs | 0 codex-rs/execpolicy/src/error.rs | 110 ++-------- codex-rs/execpolicy/src/lib.rs | 52 +---- codex-rs/execpolicy/src/main.rs | 199 +++++------------ .../{execpolicy2 => execpolicy}/src/parser.rs | 0 codex-rs/execpolicy/src/policy.rs | 161 ++++++-------- .../{execpolicy2 => execpolicy}/src/rule.rs | 0 .../tests/basic.rs | 16 +- codex-rs/execpolicy2/Cargo.toml | 29 --- codex-rs/execpolicy2/README.md | 59 ----- codex-rs/execpolicy2/src/error.rs | 26 --- codex-rs/execpolicy2/src/lib.rs | 15 -- codex-rs/execpolicy2/src/main.rs | 66 ------ codex-rs/execpolicy2/src/policy.rs | 84 -------- 52 files changed, 938 insertions(+), 931 deletions(-) create mode 100644 codex-rs/execpolicy-legacy/Cargo.toml create mode 100644 codex-rs/execpolicy-legacy/README.md rename codex-rs/{execpolicy => execpolicy-legacy}/build.rs (100%) rename codex-rs/{execpolicy => execpolicy-legacy}/src/arg_matcher.rs (100%) rename codex-rs/{execpolicy => execpolicy-legacy}/src/arg_resolver.rs (100%) rename codex-rs/{execpolicy => execpolicy-legacy}/src/arg_type.rs (100%) rename codex-rs/{execpolicy => execpolicy-legacy}/src/default.policy (100%) create mode 100644 codex-rs/execpolicy-legacy/src/error.rs rename codex-rs/{execpolicy => execpolicy-legacy}/src/exec_call.rs (100%) rename codex-rs/{execpolicy => execpolicy-legacy}/src/execv_checker.rs (100%) create mode 100644 codex-rs/execpolicy-legacy/src/lib.rs create mode 100644 codex-rs/execpolicy-legacy/src/main.rs rename codex-rs/{execpolicy => execpolicy-legacy}/src/opt.rs (100%) create mode 100644 codex-rs/execpolicy-legacy/src/policy.rs rename codex-rs/{execpolicy => execpolicy-legacy}/src/policy_parser.rs (100%) rename codex-rs/{execpolicy => execpolicy-legacy}/src/program.rs (100%) rename codex-rs/{execpolicy => execpolicy-legacy}/src/sed_command.rs (100%) rename codex-rs/{execpolicy => execpolicy-legacy}/src/valid_exec.rs (100%) rename codex-rs/{execpolicy => execpolicy-legacy}/tests/all.rs (100%) rename codex-rs/{execpolicy => execpolicy-legacy}/tests/suite/bad.rs (72%) rename codex-rs/{execpolicy => execpolicy-legacy}/tests/suite/cp.rs (81%) rename codex-rs/{execpolicy => execpolicy-legacy}/tests/suite/good.rs (72%) rename codex-rs/{execpolicy => execpolicy-legacy}/tests/suite/head.rs (87%) rename codex-rs/{execpolicy => execpolicy-legacy}/tests/suite/literal.rs (78%) rename codex-rs/{execpolicy => execpolicy-legacy}/tests/suite/ls.rs (91%) rename codex-rs/{execpolicy => execpolicy-legacy}/tests/suite/mod.rs (100%) rename codex-rs/{execpolicy => execpolicy-legacy}/tests/suite/parse_sed_command.rs (84%) rename codex-rs/{execpolicy => execpolicy-legacy}/tests/suite/pwd.rs (82%) rename codex-rs/{execpolicy => execpolicy-legacy}/tests/suite/sed.rs (82%) rename codex-rs/{execpolicy2 => execpolicy}/examples/example.codexpolicy (100%) rename codex-rs/{execpolicy2 => execpolicy}/src/decision.rs (100%) rename codex-rs/{execpolicy2 => execpolicy}/src/parser.rs (100%) rename codex-rs/{execpolicy2 => execpolicy}/src/rule.rs (100%) rename codex-rs/{execpolicy2 => execpolicy}/tests/basic.rs (97%) delete mode 100644 codex-rs/execpolicy2/Cargo.toml delete mode 100644 codex-rs/execpolicy2/README.md delete mode 100644 codex-rs/execpolicy2/src/error.rs delete mode 100644 codex-rs/execpolicy2/src/lib.rs delete mode 100644 codex-rs/execpolicy2/src/main.rs delete mode 100644 codex-rs/execpolicy2/src/policy.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 798b8d71b3..b8175ee36b 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1086,7 +1086,7 @@ dependencies = [ "codex-apply-patch", "codex-arg0", "codex-async-utils", - "codex-execpolicy2", + "codex-execpolicy", "codex-file-search", "codex-git", "codex-keyring-store", @@ -1209,6 +1209,21 @@ dependencies = [ [[package]] name = "codex-execpolicy" version = "0.0.0" +dependencies = [ + "anyhow", + "clap", + "multimap", + "pretty_assertions", + "serde", + "serde_json", + "shlex", + "starlark", + "thiserror 2.0.17", +] + +[[package]] +name = "codex-execpolicy-legacy" +version = "0.0.0" dependencies = [ "allocative", "anyhow", @@ -1226,21 +1241,6 @@ dependencies = [ "tempfile", ] -[[package]] -name = "codex-execpolicy2" -version = "0.0.0" -dependencies = [ - "anyhow", - "clap", - "multimap", - "pretty_assertions", - "serde", - "serde_json", - "shlex", - "starlark", - "thiserror 2.0.17", -] - [[package]] name = "codex-feedback" version = "0.0.0" diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index b0cf1e70fb..e2aa981381 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -18,7 +18,7 @@ members = [ "exec", "exec-server", "execpolicy", - "execpolicy2", + "execpolicy-legacy", "keyring-store", "file-search", "linux-sandbox", @@ -67,7 +67,7 @@ codex-chatgpt = { path = "chatgpt" } codex-common = { path = "common" } codex-core = { path = "core" } codex-exec = { path = "exec" } -codex-execpolicy2 = { path = "execpolicy2" } +codex-execpolicy = { path = "execpolicy" } codex-feedback = { path = "feedback" } codex-file-search = { path = "file-search" } codex-git = { path = "utils/git" } diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index db06f6b795..bc7b3668d0 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -22,7 +22,7 @@ chrono = { workspace = true, features = ["serde"] } codex-app-server-protocol = { workspace = true } codex-apply-patch = { workspace = true } codex-async-utils = { workspace = true } -codex-execpolicy2 = { workspace = true } +codex-execpolicy = { workspace = true } codex-file-search = { workspace = true } codex-git = { workspace = true } codex-keyring-store = { workspace = true } diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index c793eff352..182ca67f41 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -121,7 +121,7 @@ use crate::user_instructions::UserInstructions; use crate::user_notification::UserNotification; use crate::util::backoff; use codex_async_utils::OrCancelExt; -use codex_execpolicy2::Policy as ExecPolicy; +use codex_execpolicy::Policy as ExecPolicy; use codex_otel::otel_event_manager::OtelEventManager; use codex_protocol::config_types::ReasoningEffort as ReasoningEffortConfig; use codex_protocol::config_types::ReasoningSummary as ReasoningSummaryConfig; @@ -2616,7 +2616,7 @@ mod tests { cwd: config.cwd.clone(), original_config_do_not_use: Arc::clone(&config), features: Features::default(), - exec_policy: Arc::new(codex_execpolicy2::Policy::empty()), + exec_policy: Arc::new(ExecPolicy::empty()), session_source: SessionSource::Exec, }; @@ -2694,7 +2694,7 @@ mod tests { cwd: config.cwd.clone(), original_config_do_not_use: Arc::clone(&config), features: Features::default(), - exec_policy: Arc::new(codex_execpolicy2::Policy::empty()), + exec_policy: Arc::new(ExecPolicy::empty()), session_source: SessionSource::Exec, }; diff --git a/codex-rs/core/src/exec_policy.rs b/codex-rs/core/src/exec_policy.rs index c55dbfed23..9525cc1acf 100644 --- a/codex-rs/core/src/exec_policy.rs +++ b/codex-rs/core/src/exec_policy.rs @@ -4,10 +4,10 @@ use std::path::PathBuf; use std::sync::Arc; use crate::command_safety::is_dangerous_command::requires_initial_appoval; -use codex_execpolicy2::Decision; -use codex_execpolicy2::Evaluation; -use codex_execpolicy2::Policy; -use codex_execpolicy2::PolicyParser; +use codex_execpolicy::Decision; +use codex_execpolicy::Evaluation; +use codex_execpolicy::Policy; +use codex_execpolicy::PolicyParser; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::SandboxPolicy; use thiserror::Error; @@ -41,7 +41,7 @@ pub enum ExecPolicyError { #[error("failed to parse execpolicy file {path}: {source}")] ParsePolicy { path: String, - source: codex_execpolicy2::Error, + source: codex_execpolicy::Error, }, } diff --git a/codex-rs/exec-server/src/posix.rs b/codex-rs/exec-server/src/posix.rs index 239eaf61f2..ea0fab1a7e 100644 --- a/codex-rs/exec-server/src/posix.rs +++ b/codex-rs/exec-server/src/posix.rs @@ -144,7 +144,7 @@ pub async fn main_execve_wrapper() -> anyhow::Result<()> { std::process::exit(exit_code); } -// TODO: replace with execpolicy2 +// TODO: replace with execpolicy fn dummy_exec_policy(file: &Path, argv: &[String], _workdir: &Path) -> ExecPolicyOutcome { if file.ends_with("rm") { diff --git a/codex-rs/execpolicy-legacy/Cargo.toml b/codex-rs/execpolicy-legacy/Cargo.toml new file mode 100644 index 0000000000..89e6e43b41 --- /dev/null +++ b/codex-rs/execpolicy-legacy/Cargo.toml @@ -0,0 +1,34 @@ +[package] +edition = "2024" +name = "codex-execpolicy-legacy" +description = "Legacy exec policy engine for validating proposed exec calls." +version = { workspace = true } + +[[bin]] +name = "codex-execpolicy-legacy" +path = "src/main.rs" + +[lib] +name = "codex_execpolicy_legacy" +path = "src/lib.rs" + +[lints] +workspace = true + +[dependencies] +allocative = { workspace = true } +anyhow = { workspace = true } +clap = { workspace = true, features = ["derive"] } +derive_more = { workspace = true, features = ["display"] } +env_logger = { workspace = true } +log = { workspace = true } +multimap = { workspace = true } +path-absolutize = { workspace = true } +regex-lite = { workspace = true } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +serde_with = { workspace = true, features = ["macros"] } +starlark = { workspace = true } + +[dev-dependencies] +tempfile = { workspace = true } diff --git a/codex-rs/execpolicy-legacy/README.md b/codex-rs/execpolicy-legacy/README.md new file mode 100644 index 0000000000..1351377e5d --- /dev/null +++ b/codex-rs/execpolicy-legacy/README.md @@ -0,0 +1,183 @@ +# codex-execpolicy-legacy + +This crate hosts the original execpolicy implementation. The newer prefix-rule +engine lives in `codex-execpolicy`. + +The goal of this library is to classify a proposed [`execv(3)`](https://linux.die.net/man/3/execv) command into one of the following states: + +- `safe` The command is safe to run (\*). +- `match` The command matched a rule in the policy, but the caller should decide whether it is safe to run based on the files it will write. +- `forbidden` The command is not allowed to be run. +- `unverified` The safety cannot be determined: make the user decide. + +(\*) Whether an `execv(3)` call should be considered "safe" often requires additional context beyond the arguments to `execv()` itself. For example, if you trust an autonomous software agent to write files in your source tree, then deciding whether `/bin/cp foo bar` is "safe" depends on `getcwd(3)` for the calling process as well as the `realpath` of `foo` and `bar` when resolved against `getcwd()`. +To that end, rather than returning a boolean, the validator returns a structured result that the client is expected to use to determine the "safety" of the proposed `execv()` call. + +For example, to check the command `ls -l foo`, the checker would be invoked as follows: + +```shell +cargo run -p codex-execpolicy-legacy -- check ls -l foo | jq +``` + +It will exit with `0` and print the following to stdout: + +```json +{ + "result": "safe", + "match": { + "program": "ls", + "flags": [ + { + "name": "-l" + } + ], + "opts": [], + "args": [ + { + "index": 1, + "type": "ReadableFile", + "value": "foo" + } + ], + "system_path": ["/bin/ls", "/usr/bin/ls"] + } +} +``` + +Of note: + +- `foo` is tagged as a `ReadableFile`, so the caller should resolve `foo` relative to `getcwd()` and `realpath` it (as it may be a symlink) to determine whether `foo` is safe to read. +- While the specified executable is `ls`, `"system_path"` offers `/bin/ls` and `/usr/bin/ls` as viable alternatives to avoid using whatever `ls` happens to appear first on the user's `$PATH`. If either exists on the host, it is recommended to use it as the first argument to `execv(3)` instead of `ls`. + +Further, "safety" in this system is not a guarantee that the command will execute successfully. As an example, `cat /Users/mbolin/code/codex/README.md` may be considered "safe" if the system has decided the agent is allowed to read anything under `/Users/mbolin/code/codex`, but it will fail at runtime if `README.md` does not exist. (Though this is "safe" in that the agent did not read any files that it was not authorized to read.) + +## Policy + +Currently, the default policy is defined in [`default.policy`](./src/default.policy) within the crate. + +The system uses [Starlark](https://bazel.build/rules/language) as the file format because, unlike something like JSON or YAML, it supports "macros" without compromising on safety or reproducibility. (Under the hood, we use [`starlark-rust`](https://github.com/facebook/starlark-rust) as the specific Starlark implementation.) + +This policy contains "rules" such as: + +```python +define_program( + program="cp", + options=[ + flag("-r"), + flag("-R"), + flag("--recursive"), + ], + args=[ARG_RFILES, ARG_WFILE], + system_path=["/bin/cp", "/usr/bin/cp"], + should_match=[ + ["foo", "bar"], + ], + should_not_match=[ + ["foo"], + ], +) +``` + +This rule means that: + +- `cp` can be used with any of the following flags (where "flag" means "an option that does not take an argument"): `-r`, `-R`, `--recursive`. +- The initial `ARG_RFILES` passed to `args` means that it expects one or more arguments that correspond to "readable files" +- The final `ARG_WFILE` passed to `args` means that it expects exactly one argument that corresponds to a "writeable file." +- As a means of a lightweight way of including a unit test alongside the definition, the `should_match` list is a list of examples of `execv(3)` args that should match the rule and `should_not_match` is a list of examples that should not match. These examples are verified when the `.policy` file is loaded. + +Note that the language of the `.policy` file is still evolving, as we have to continue to expand it so it is sufficiently expressive to accept all commands we want to consider "safe" without allowing unsafe commands to pass through. + +The integrity of `default.policy` is verified [via unit tests](./tests). + +Further, the CLI supports a `--policy` option to specify a custom `.policy` file for ad-hoc testing. + +## Output Type: `match` + +Going back to the `cp` example, because the rule matches an `ARG_WFILE`, it will return `match` instead of `safe`: + +```shell +cargo run -p codex-execpolicy-legacy -- check cp src1 src2 dest | jq +``` + +If the caller wants to consider allowing this command, it should parse the JSON to pick out the `WriteableFile` arguments and decide whether they are safe to write: + +```json +{ + "result": "match", + "match": { + "program": "cp", + "flags": [], + "opts": [], + "args": [ + { + "index": 0, + "type": "ReadableFile", + "value": "src1" + }, + { + "index": 1, + "type": "ReadableFile", + "value": "src2" + }, + { + "index": 2, + "type": "WriteableFile", + "value": "dest" + } + ], + "system_path": ["/bin/cp", "/usr/bin/cp"] + } +} +``` + +Note the exit code is still `0` for a `match` unless the `--require-safe` flag is specified, in which case the exit code is `12`. + +## Output Type: `forbidden` + +It is also possible to define a rule that, if it matches a command, should flag it as _forbidden_. For example, we do not want agents to be able to run `applied deploy` _ever_, so we define the following rule: + +```python +define_program( + program="applied", + args=["deploy"], + forbidden="Infrastructure Risk: command contains 'applied deploy'", + should_match=[ + ["deploy"], + ], + should_not_match=[ + ["lint"], + ], +) +``` + +Note that for a rule to be forbidden, the `forbidden` keyword arg must be specified as the reason the command is forbidden. This will be included in the output: + +```shell +cargo run -p codex-execpolicy-legacy -- check applied deploy | jq +``` + +```json +{ + "result": "forbidden", + "reason": "Infrastructure Risk: command contains 'applied deploy'", + "cause": { + "Exec": { + "exec": { + "program": "applied", + "flags": [], + "opts": [], + "args": [ + { + "index": 0, + "type": { + "Literal": "deploy" + }, + "value": "deploy" + } + ], + "system_path": [] + } + } + } +} +``` diff --git a/codex-rs/execpolicy/build.rs b/codex-rs/execpolicy-legacy/build.rs similarity index 100% rename from codex-rs/execpolicy/build.rs rename to codex-rs/execpolicy-legacy/build.rs diff --git a/codex-rs/execpolicy/src/arg_matcher.rs b/codex-rs/execpolicy-legacy/src/arg_matcher.rs similarity index 100% rename from codex-rs/execpolicy/src/arg_matcher.rs rename to codex-rs/execpolicy-legacy/src/arg_matcher.rs diff --git a/codex-rs/execpolicy/src/arg_resolver.rs b/codex-rs/execpolicy-legacy/src/arg_resolver.rs similarity index 100% rename from codex-rs/execpolicy/src/arg_resolver.rs rename to codex-rs/execpolicy-legacy/src/arg_resolver.rs diff --git a/codex-rs/execpolicy/src/arg_type.rs b/codex-rs/execpolicy-legacy/src/arg_type.rs similarity index 100% rename from codex-rs/execpolicy/src/arg_type.rs rename to codex-rs/execpolicy-legacy/src/arg_type.rs diff --git a/codex-rs/execpolicy/src/default.policy b/codex-rs/execpolicy-legacy/src/default.policy similarity index 100% rename from codex-rs/execpolicy/src/default.policy rename to codex-rs/execpolicy-legacy/src/default.policy diff --git a/codex-rs/execpolicy-legacy/src/error.rs b/codex-rs/execpolicy-legacy/src/error.rs new file mode 100644 index 0000000000..e6443d69dc --- /dev/null +++ b/codex-rs/execpolicy-legacy/src/error.rs @@ -0,0 +1,96 @@ +use std::path::PathBuf; + +use serde::Serialize; + +use crate::arg_matcher::ArgMatcher; +use crate::arg_resolver::PositionalArg; +use serde_with::DisplayFromStr; +use serde_with::serde_as; + +pub type Result = std::result::Result; + +#[serde_as] +#[derive(Debug, Eq, PartialEq, Serialize)] +#[serde(tag = "type")] +pub enum Error { + NoSpecForProgram { + program: String, + }, + OptionMissingValue { + program: String, + option: String, + }, + OptionFollowedByOptionInsteadOfValue { + program: String, + option: String, + value: String, + }, + UnknownOption { + program: String, + option: String, + }, + UnexpectedArguments { + program: String, + args: Vec, + }, + DoubleDashNotSupportedYet { + program: String, + }, + MultipleVarargPatterns { + program: String, + first: ArgMatcher, + second: ArgMatcher, + }, + RangeStartExceedsEnd { + start: usize, + end: usize, + }, + RangeEndOutOfBounds { + end: usize, + len: usize, + }, + PrefixOverlapsSuffix {}, + NotEnoughArgs { + program: String, + args: Vec, + arg_patterns: Vec, + }, + InternalInvariantViolation { + message: String, + }, + VarargMatcherDidNotMatchAnything { + program: String, + matcher: ArgMatcher, + }, + EmptyFileName {}, + LiteralValueDidNotMatch { + expected: String, + actual: String, + }, + InvalidPositiveInteger { + value: String, + }, + MissingRequiredOptions { + program: String, + options: Vec, + }, + SedCommandNotProvablySafe { + command: String, + }, + ReadablePathNotInReadableFolders { + file: PathBuf, + folders: Vec, + }, + WriteablePathNotInWriteableFolders { + file: PathBuf, + folders: Vec, + }, + CannotCheckRelativePath { + file: PathBuf, + }, + CannotCanonicalizePath { + file: String, + #[serde_as(as = "DisplayFromStr")] + error: std::io::ErrorKind, + }, +} diff --git a/codex-rs/execpolicy/src/exec_call.rs b/codex-rs/execpolicy-legacy/src/exec_call.rs similarity index 100% rename from codex-rs/execpolicy/src/exec_call.rs rename to codex-rs/execpolicy-legacy/src/exec_call.rs diff --git a/codex-rs/execpolicy/src/execv_checker.rs b/codex-rs/execpolicy-legacy/src/execv_checker.rs similarity index 100% rename from codex-rs/execpolicy/src/execv_checker.rs rename to codex-rs/execpolicy-legacy/src/execv_checker.rs diff --git a/codex-rs/execpolicy-legacy/src/lib.rs b/codex-rs/execpolicy-legacy/src/lib.rs new file mode 100644 index 0000000000..6f12225981 --- /dev/null +++ b/codex-rs/execpolicy-legacy/src/lib.rs @@ -0,0 +1,45 @@ +#![allow(clippy::type_complexity)] +#![allow(clippy::too_many_arguments)] +#[macro_use] +extern crate starlark; + +mod arg_matcher; +mod arg_resolver; +mod arg_type; +mod error; +mod exec_call; +mod execv_checker; +mod opt; +mod policy; +mod policy_parser; +mod program; +mod sed_command; +mod valid_exec; + +pub use arg_matcher::ArgMatcher; +pub use arg_resolver::PositionalArg; +pub use arg_type::ArgType; +pub use error::Error; +pub use error::Result; +pub use exec_call::ExecCall; +pub use execv_checker::ExecvChecker; +pub use opt::Opt; +pub use policy::Policy; +pub use policy_parser::PolicyParser; +pub use program::Forbidden; +pub use program::MatchedExec; +pub use program::NegativeExamplePassedCheck; +pub use program::PositiveExampleFailedCheck; +pub use program::ProgramSpec; +pub use sed_command::parse_sed_command; +pub use valid_exec::MatchedArg; +pub use valid_exec::MatchedFlag; +pub use valid_exec::MatchedOpt; +pub use valid_exec::ValidExec; + +const DEFAULT_POLICY: &str = include_str!("default.policy"); + +pub fn get_default_policy() -> starlark::Result { + let parser = PolicyParser::new("#default", DEFAULT_POLICY); + parser.parse() +} diff --git a/codex-rs/execpolicy-legacy/src/main.rs b/codex-rs/execpolicy-legacy/src/main.rs new file mode 100644 index 0000000000..f5b66dfe50 --- /dev/null +++ b/codex-rs/execpolicy-legacy/src/main.rs @@ -0,0 +1,169 @@ +use anyhow::Result; +use clap::Parser; +use clap::Subcommand; +use codex_execpolicy_legacy::ExecCall; +use codex_execpolicy_legacy::MatchedExec; +use codex_execpolicy_legacy::Policy; +use codex_execpolicy_legacy::PolicyParser; +use codex_execpolicy_legacy::ValidExec; +use codex_execpolicy_legacy::get_default_policy; +use serde::Deserialize; +use serde::Serialize; +use serde::de; +use starlark::Error as StarlarkError; +use std::path::PathBuf; +use std::str::FromStr; + +const MATCHED_BUT_WRITES_FILES_EXIT_CODE: i32 = 12; +const MIGHT_BE_SAFE_EXIT_CODE: i32 = 13; +const FORBIDDEN_EXIT_CODE: i32 = 14; + +#[derive(Parser, Deserialize, Debug)] +#[command(version, about, long_about = None)] +pub struct Args { + /// If the command fails the policy, exit with 13, but print parseable JSON + /// to stdout. + #[clap(long)] + pub require_safe: bool, + + /// Path to the policy file. + #[clap(long, short = 'p')] + pub policy: Option, + + #[command(subcommand)] + pub command: Command, +} + +#[derive(Clone, Debug, Deserialize, Subcommand)] +pub enum Command { + /// Checks the command as if the arguments were the inputs to execv(3). + Check { + #[arg(trailing_var_arg = true)] + command: Vec, + }, + + /// Checks the command encoded as a JSON object. + #[clap(name = "check-json")] + CheckJson { + /// JSON object with "program" (str) and "args" (list[str]) fields. + #[serde(deserialize_with = "deserialize_from_json")] + exec: ExecArg, + }, +} + +#[derive(Clone, Debug, Deserialize)] +pub struct ExecArg { + pub program: String, + + #[serde(default)] + pub args: Vec, +} + +fn main() -> Result<()> { + env_logger::init(); + + let args = Args::parse(); + let policy = match args.policy { + Some(policy) => { + let policy_source = policy.to_string_lossy().to_string(); + let unparsed_policy = std::fs::read_to_string(policy)?; + let parser = PolicyParser::new(&policy_source, &unparsed_policy); + parser.parse() + } + None => get_default_policy(), + }; + let policy = policy.map_err(StarlarkError::into_anyhow)?; + + let exec = match args.command { + Command::Check { command } => match command.split_first() { + Some((first, rest)) => ExecArg { + program: first.to_string(), + args: rest.to_vec(), + }, + None => { + eprintln!("no command provided"); + std::process::exit(1); + } + }, + Command::CheckJson { exec } => exec, + }; + + let (output, exit_code) = check_command(&policy, exec, args.require_safe); + let json = serde_json::to_string(&output)?; + println!("{json}"); + std::process::exit(exit_code); +} + +fn check_command( + policy: &Policy, + ExecArg { program, args }: ExecArg, + check: bool, +) -> (Output, i32) { + let exec_call = ExecCall { program, args }; + match policy.check(&exec_call) { + Ok(MatchedExec::Match { exec }) => { + if exec.might_write_files() { + let exit_code = if check { + MATCHED_BUT_WRITES_FILES_EXIT_CODE + } else { + 0 + }; + (Output::Match { r#match: exec }, exit_code) + } else { + (Output::Safe { r#match: exec }, 0) + } + } + Ok(MatchedExec::Forbidden { reason, cause }) => { + let exit_code = if check { FORBIDDEN_EXIT_CODE } else { 0 }; + (Output::Forbidden { reason, cause }, exit_code) + } + Err(err) => { + let exit_code = if check { MIGHT_BE_SAFE_EXIT_CODE } else { 0 }; + (Output::Unverified { error: err }, exit_code) + } + } +} + +#[derive(Debug, Serialize)] +#[serde(tag = "result")] +pub enum Output { + /// The command is verified as safe. + #[serde(rename = "safe")] + Safe { r#match: ValidExec }, + + /// The command has matched a rule in the policy, but the caller should + /// decide whether it is "safe" given the files it wants to write. + #[serde(rename = "match")] + Match { r#match: ValidExec }, + + /// The user is forbidden from running the command. + #[serde(rename = "forbidden")] + Forbidden { + reason: String, + cause: codex_execpolicy_legacy::Forbidden, + }, + + /// The safety of the command could not be verified. + #[serde(rename = "unverified")] + Unverified { + error: codex_execpolicy_legacy::Error, + }, +} + +fn deserialize_from_json<'de, D>(deserializer: D) -> Result +where + D: de::Deserializer<'de>, +{ + let s = String::deserialize(deserializer)?; + let decoded = serde_json::from_str(&s) + .map_err(|e| serde::de::Error::custom(format!("JSON parse error: {e}")))?; + Ok(decoded) +} + +impl FromStr for ExecArg { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + serde_json::from_str(s).map_err(Into::into) + } +} diff --git a/codex-rs/execpolicy/src/opt.rs b/codex-rs/execpolicy-legacy/src/opt.rs similarity index 100% rename from codex-rs/execpolicy/src/opt.rs rename to codex-rs/execpolicy-legacy/src/opt.rs diff --git a/codex-rs/execpolicy-legacy/src/policy.rs b/codex-rs/execpolicy-legacy/src/policy.rs new file mode 100644 index 0000000000..825d6164a5 --- /dev/null +++ b/codex-rs/execpolicy-legacy/src/policy.rs @@ -0,0 +1,103 @@ +use multimap::MultiMap; +use regex_lite::Error as RegexError; +use regex_lite::Regex; + +use crate::ExecCall; +use crate::Forbidden; +use crate::MatchedExec; +use crate::NegativeExamplePassedCheck; +use crate::ProgramSpec; +use crate::error::Error; +use crate::error::Result; +use crate::policy_parser::ForbiddenProgramRegex; +use crate::program::PositiveExampleFailedCheck; + +pub struct Policy { + programs: MultiMap, + forbidden_program_regexes: Vec, + forbidden_substrings_pattern: Option, +} + +impl Policy { + pub fn new( + programs: MultiMap, + forbidden_program_regexes: Vec, + forbidden_substrings: Vec, + ) -> std::result::Result { + let forbidden_substrings_pattern = if forbidden_substrings.is_empty() { + None + } else { + let escaped_substrings = forbidden_substrings + .iter() + .map(|s| regex_lite::escape(s)) + .collect::>() + .join("|"); + Some(Regex::new(&format!("({escaped_substrings})"))?) + }; + Ok(Self { + programs, + forbidden_program_regexes, + forbidden_substrings_pattern, + }) + } + + pub fn check(&self, exec_call: &ExecCall) -> Result { + let ExecCall { program, args } = &exec_call; + for ForbiddenProgramRegex { regex, reason } in &self.forbidden_program_regexes { + if regex.is_match(program) { + return Ok(MatchedExec::Forbidden { + cause: Forbidden::Program { + program: program.clone(), + exec_call: exec_call.clone(), + }, + reason: reason.clone(), + }); + } + } + + for arg in args { + if let Some(regex) = &self.forbidden_substrings_pattern + && regex.is_match(arg) + { + return Ok(MatchedExec::Forbidden { + cause: Forbidden::Arg { + arg: arg.clone(), + exec_call: exec_call.clone(), + }, + reason: format!("arg `{arg}` contains forbidden substring"), + }); + } + } + + let mut last_err = Err(Error::NoSpecForProgram { + program: program.clone(), + }); + if let Some(spec_list) = self.programs.get_vec(program) { + for spec in spec_list { + match spec.check(exec_call) { + Ok(matched_exec) => return Ok(matched_exec), + Err(err) => { + last_err = Err(err); + } + } + } + } + last_err + } + + pub fn check_each_good_list_individually(&self) -> Vec { + let mut violations = Vec::new(); + for (_program, spec) in self.programs.flat_iter() { + violations.extend(spec.verify_should_match_list()); + } + violations + } + + pub fn check_each_bad_list_individually(&self) -> Vec { + let mut violations = Vec::new(); + for (_program, spec) in self.programs.flat_iter() { + violations.extend(spec.verify_should_not_match_list()); + } + violations + } +} diff --git a/codex-rs/execpolicy/src/policy_parser.rs b/codex-rs/execpolicy-legacy/src/policy_parser.rs similarity index 100% rename from codex-rs/execpolicy/src/policy_parser.rs rename to codex-rs/execpolicy-legacy/src/policy_parser.rs diff --git a/codex-rs/execpolicy/src/program.rs b/codex-rs/execpolicy-legacy/src/program.rs similarity index 100% rename from codex-rs/execpolicy/src/program.rs rename to codex-rs/execpolicy-legacy/src/program.rs diff --git a/codex-rs/execpolicy/src/sed_command.rs b/codex-rs/execpolicy-legacy/src/sed_command.rs similarity index 100% rename from codex-rs/execpolicy/src/sed_command.rs rename to codex-rs/execpolicy-legacy/src/sed_command.rs diff --git a/codex-rs/execpolicy/src/valid_exec.rs b/codex-rs/execpolicy-legacy/src/valid_exec.rs similarity index 100% rename from codex-rs/execpolicy/src/valid_exec.rs rename to codex-rs/execpolicy-legacy/src/valid_exec.rs diff --git a/codex-rs/execpolicy/tests/all.rs b/codex-rs/execpolicy-legacy/tests/all.rs similarity index 100% rename from codex-rs/execpolicy/tests/all.rs rename to codex-rs/execpolicy-legacy/tests/all.rs diff --git a/codex-rs/execpolicy/tests/suite/bad.rs b/codex-rs/execpolicy-legacy/tests/suite/bad.rs similarity index 72% rename from codex-rs/execpolicy/tests/suite/bad.rs rename to codex-rs/execpolicy-legacy/tests/suite/bad.rs index 8b6e195fb0..e1f8675330 100644 --- a/codex-rs/execpolicy/tests/suite/bad.rs +++ b/codex-rs/execpolicy-legacy/tests/suite/bad.rs @@ -1,5 +1,5 @@ -use codex_execpolicy::NegativeExamplePassedCheck; -use codex_execpolicy::get_default_policy; +use codex_execpolicy_legacy::NegativeExamplePassedCheck; +use codex_execpolicy_legacy::get_default_policy; #[test] fn verify_everything_in_bad_list_is_rejected() { diff --git a/codex-rs/execpolicy/tests/suite/cp.rs b/codex-rs/execpolicy-legacy/tests/suite/cp.rs similarity index 81% rename from codex-rs/execpolicy/tests/suite/cp.rs rename to codex-rs/execpolicy-legacy/tests/suite/cp.rs index aa19f0b5d5..3cfc9ac5c1 100644 --- a/codex-rs/execpolicy/tests/suite/cp.rs +++ b/codex-rs/execpolicy-legacy/tests/suite/cp.rs @@ -1,15 +1,15 @@ -extern crate codex_execpolicy; +extern crate codex_execpolicy_legacy; -use codex_execpolicy::ArgMatcher; -use codex_execpolicy::ArgType; -use codex_execpolicy::Error; -use codex_execpolicy::ExecCall; -use codex_execpolicy::MatchedArg; -use codex_execpolicy::MatchedExec; -use codex_execpolicy::Policy; -use codex_execpolicy::Result; -use codex_execpolicy::ValidExec; -use codex_execpolicy::get_default_policy; +use codex_execpolicy_legacy::ArgMatcher; +use codex_execpolicy_legacy::ArgType; +use codex_execpolicy_legacy::Error; +use codex_execpolicy_legacy::ExecCall; +use codex_execpolicy_legacy::MatchedArg; +use codex_execpolicy_legacy::MatchedExec; +use codex_execpolicy_legacy::Policy; +use codex_execpolicy_legacy::Result; +use codex_execpolicy_legacy::ValidExec; +use codex_execpolicy_legacy::get_default_policy; #[expect(clippy::expect_used)] fn setup() -> Policy { diff --git a/codex-rs/execpolicy/tests/suite/good.rs b/codex-rs/execpolicy-legacy/tests/suite/good.rs similarity index 72% rename from codex-rs/execpolicy/tests/suite/good.rs rename to codex-rs/execpolicy-legacy/tests/suite/good.rs index 3b7313a335..3c86c7acb8 100644 --- a/codex-rs/execpolicy/tests/suite/good.rs +++ b/codex-rs/execpolicy-legacy/tests/suite/good.rs @@ -1,5 +1,5 @@ -use codex_execpolicy::PositiveExampleFailedCheck; -use codex_execpolicy::get_default_policy; +use codex_execpolicy_legacy::PositiveExampleFailedCheck; +use codex_execpolicy_legacy::get_default_policy; #[test] fn verify_everything_in_good_list_is_allowed() { diff --git a/codex-rs/execpolicy/tests/suite/head.rs b/codex-rs/execpolicy-legacy/tests/suite/head.rs similarity index 87% rename from codex-rs/execpolicy/tests/suite/head.rs rename to codex-rs/execpolicy-legacy/tests/suite/head.rs index 3c32ccfbfe..390b4ddb3e 100644 --- a/codex-rs/execpolicy/tests/suite/head.rs +++ b/codex-rs/execpolicy-legacy/tests/suite/head.rs @@ -1,16 +1,16 @@ -use codex_execpolicy::ArgMatcher; -use codex_execpolicy::ArgType; -use codex_execpolicy::Error; -use codex_execpolicy::ExecCall; -use codex_execpolicy::MatchedArg; -use codex_execpolicy::MatchedExec; -use codex_execpolicy::MatchedOpt; -use codex_execpolicy::Policy; -use codex_execpolicy::Result; -use codex_execpolicy::ValidExec; -use codex_execpolicy::get_default_policy; +use codex_execpolicy_legacy::ArgMatcher; +use codex_execpolicy_legacy::ArgType; +use codex_execpolicy_legacy::Error; +use codex_execpolicy_legacy::ExecCall; +use codex_execpolicy_legacy::MatchedArg; +use codex_execpolicy_legacy::MatchedExec; +use codex_execpolicy_legacy::MatchedOpt; +use codex_execpolicy_legacy::Policy; +use codex_execpolicy_legacy::Result; +use codex_execpolicy_legacy::ValidExec; +use codex_execpolicy_legacy::get_default_policy; -extern crate codex_execpolicy; +extern crate codex_execpolicy_legacy; #[expect(clippy::expect_used)] fn setup() -> Policy { diff --git a/codex-rs/execpolicy/tests/suite/literal.rs b/codex-rs/execpolicy-legacy/tests/suite/literal.rs similarity index 78% rename from codex-rs/execpolicy/tests/suite/literal.rs rename to codex-rs/execpolicy-legacy/tests/suite/literal.rs index d849371e3b..296206dbc5 100644 --- a/codex-rs/execpolicy/tests/suite/literal.rs +++ b/codex-rs/execpolicy-legacy/tests/suite/literal.rs @@ -1,13 +1,13 @@ -use codex_execpolicy::ArgType; -use codex_execpolicy::Error; -use codex_execpolicy::ExecCall; -use codex_execpolicy::MatchedArg; -use codex_execpolicy::MatchedExec; -use codex_execpolicy::PolicyParser; -use codex_execpolicy::Result; -use codex_execpolicy::ValidExec; +use codex_execpolicy_legacy::ArgType; +use codex_execpolicy_legacy::Error; +use codex_execpolicy_legacy::ExecCall; +use codex_execpolicy_legacy::MatchedArg; +use codex_execpolicy_legacy::MatchedExec; +use codex_execpolicy_legacy::PolicyParser; +use codex_execpolicy_legacy::Result; +use codex_execpolicy_legacy::ValidExec; -extern crate codex_execpolicy; +extern crate codex_execpolicy_legacy; #[test] fn test_invalid_subcommand() -> Result<()> { diff --git a/codex-rs/execpolicy/tests/suite/ls.rs b/codex-rs/execpolicy-legacy/tests/suite/ls.rs similarity index 91% rename from codex-rs/execpolicy/tests/suite/ls.rs rename to codex-rs/execpolicy-legacy/tests/suite/ls.rs index e52316c06e..b4b27b0e22 100644 --- a/codex-rs/execpolicy/tests/suite/ls.rs +++ b/codex-rs/execpolicy-legacy/tests/suite/ls.rs @@ -1,15 +1,15 @@ -extern crate codex_execpolicy; +extern crate codex_execpolicy_legacy; -use codex_execpolicy::ArgType; -use codex_execpolicy::Error; -use codex_execpolicy::ExecCall; -use codex_execpolicy::MatchedArg; -use codex_execpolicy::MatchedExec; -use codex_execpolicy::MatchedFlag; -use codex_execpolicy::Policy; -use codex_execpolicy::Result; -use codex_execpolicy::ValidExec; -use codex_execpolicy::get_default_policy; +use codex_execpolicy_legacy::ArgType; +use codex_execpolicy_legacy::Error; +use codex_execpolicy_legacy::ExecCall; +use codex_execpolicy_legacy::MatchedArg; +use codex_execpolicy_legacy::MatchedExec; +use codex_execpolicy_legacy::MatchedFlag; +use codex_execpolicy_legacy::Policy; +use codex_execpolicy_legacy::Result; +use codex_execpolicy_legacy::ValidExec; +use codex_execpolicy_legacy::get_default_policy; #[expect(clippy::expect_used)] fn setup() -> Policy { diff --git a/codex-rs/execpolicy/tests/suite/mod.rs b/codex-rs/execpolicy-legacy/tests/suite/mod.rs similarity index 100% rename from codex-rs/execpolicy/tests/suite/mod.rs rename to codex-rs/execpolicy-legacy/tests/suite/mod.rs diff --git a/codex-rs/execpolicy/tests/suite/parse_sed_command.rs b/codex-rs/execpolicy-legacy/tests/suite/parse_sed_command.rs similarity index 84% rename from codex-rs/execpolicy/tests/suite/parse_sed_command.rs rename to codex-rs/execpolicy-legacy/tests/suite/parse_sed_command.rs index 20f5bbf301..f1da55d641 100644 --- a/codex-rs/execpolicy/tests/suite/parse_sed_command.rs +++ b/codex-rs/execpolicy-legacy/tests/suite/parse_sed_command.rs @@ -1,5 +1,5 @@ -use codex_execpolicy::Error; -use codex_execpolicy::parse_sed_command; +use codex_execpolicy_legacy::Error; +use codex_execpolicy_legacy::parse_sed_command; #[test] fn parses_simple_print_command() { diff --git a/codex-rs/execpolicy/tests/suite/pwd.rs b/codex-rs/execpolicy-legacy/tests/suite/pwd.rs similarity index 82% rename from codex-rs/execpolicy/tests/suite/pwd.rs rename to codex-rs/execpolicy-legacy/tests/suite/pwd.rs index fdf5a4f1a5..73d1caada8 100644 --- a/codex-rs/execpolicy/tests/suite/pwd.rs +++ b/codex-rs/execpolicy-legacy/tests/suite/pwd.rs @@ -1,15 +1,15 @@ -extern crate codex_execpolicy; +extern crate codex_execpolicy_legacy; use std::vec; -use codex_execpolicy::Error; -use codex_execpolicy::ExecCall; -use codex_execpolicy::MatchedExec; -use codex_execpolicy::MatchedFlag; -use codex_execpolicy::Policy; -use codex_execpolicy::PositionalArg; -use codex_execpolicy::ValidExec; -use codex_execpolicy::get_default_policy; +use codex_execpolicy_legacy::Error; +use codex_execpolicy_legacy::ExecCall; +use codex_execpolicy_legacy::MatchedExec; +use codex_execpolicy_legacy::MatchedFlag; +use codex_execpolicy_legacy::Policy; +use codex_execpolicy_legacy::PositionalArg; +use codex_execpolicy_legacy::ValidExec; +use codex_execpolicy_legacy::get_default_policy; #[expect(clippy::expect_used)] fn setup() -> Policy { diff --git a/codex-rs/execpolicy/tests/suite/sed.rs b/codex-rs/execpolicy-legacy/tests/suite/sed.rs similarity index 82% rename from codex-rs/execpolicy/tests/suite/sed.rs rename to codex-rs/execpolicy-legacy/tests/suite/sed.rs index bf35bf6d46..d732cc7c9c 100644 --- a/codex-rs/execpolicy/tests/suite/sed.rs +++ b/codex-rs/execpolicy-legacy/tests/suite/sed.rs @@ -1,16 +1,16 @@ -extern crate codex_execpolicy; +extern crate codex_execpolicy_legacy; -use codex_execpolicy::ArgType; -use codex_execpolicy::Error; -use codex_execpolicy::ExecCall; -use codex_execpolicy::MatchedArg; -use codex_execpolicy::MatchedExec; -use codex_execpolicy::MatchedFlag; -use codex_execpolicy::MatchedOpt; -use codex_execpolicy::Policy; -use codex_execpolicy::Result; -use codex_execpolicy::ValidExec; -use codex_execpolicy::get_default_policy; +use codex_execpolicy_legacy::ArgType; +use codex_execpolicy_legacy::Error; +use codex_execpolicy_legacy::ExecCall; +use codex_execpolicy_legacy::MatchedArg; +use codex_execpolicy_legacy::MatchedExec; +use codex_execpolicy_legacy::MatchedFlag; +use codex_execpolicy_legacy::MatchedOpt; +use codex_execpolicy_legacy::Policy; +use codex_execpolicy_legacy::Result; +use codex_execpolicy_legacy::ValidExec; +use codex_execpolicy_legacy::get_default_policy; #[expect(clippy::expect_used)] fn setup() -> Policy { diff --git a/codex-rs/execpolicy/Cargo.toml b/codex-rs/execpolicy/Cargo.toml index 0fe7cd486a..bececed4b2 100644 --- a/codex-rs/execpolicy/Cargo.toml +++ b/codex-rs/execpolicy/Cargo.toml @@ -1,33 +1,29 @@ [package] -edition = "2024" name = "codex-execpolicy" version = { workspace = true } - -[[bin]] -name = "codex-execpolicy" -path = "src/main.rs" +edition = "2024" +description = "Codex exec policy: prefix-based Starlark rules for command decisions." [lib] name = "codex_execpolicy" path = "src/lib.rs" +[[bin]] +name = "codex-execpolicy" +path = "src/main.rs" + [lints] workspace = true [dependencies] -allocative = { workspace = true } anyhow = { workspace = true } clap = { workspace = true, features = ["derive"] } -derive_more = { workspace = true, features = ["display"] } -env_logger = { workspace = true } -log = { workspace = true } multimap = { workspace = true } -path-absolutize = { workspace = true } -regex-lite = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } -serde_with = { workspace = true, features = ["macros"] } +shlex = { workspace = true } starlark = { workspace = true } +thiserror = { workspace = true } [dev-dependencies] -tempfile = { workspace = true } +pretty_assertions = { workspace = true } diff --git a/codex-rs/execpolicy/README.md b/codex-rs/execpolicy/README.md index ca95829440..f508c8ca64 100644 --- a/codex-rs/execpolicy/README.md +++ b/codex-rs/execpolicy/README.md @@ -1,180 +1,60 @@ -# codex_execpolicy +# codex-execpolicy -The goal of this library is to classify a proposed [`execv(3)`](https://linux.die.net/man/3/execv) command into one of the following states: +## Overview +- Policy engine and CLI built around `prefix_rule(pattern=[...], decision?, match?, not_match?)`. +- This release covers the prefix-rule subset of the execpolicy language; a richer language will follow. +- Tokens are matched in order; any `pattern` element may be a list to denote alternatives. `decision` defaults to `allow`; valid values: `allow`, `prompt`, `forbidden`. +- `match` / `not_match` supply example invocations that are validated at load time (think of them as unit tests); examples can be token arrays or strings (strings are tokenized with `shlex`). +- The CLI always prints the JSON serialization of the evaluation result. +- The legacy rule matcher lives in `codex-execpolicy-legacy`. -- `safe` The command is safe to run (\*). -- `match` The command matched a rule in the policy, but the caller should decide whether it is safe to run based on the files it will write. -- `forbidden` The command is not allowed to be run. -- `unverified` The safety cannot be determined: make the user decide. - -(\*) Whether an `execv(3)` call should be considered "safe" often requires additional context beyond the arguments to `execv()` itself. For example, if you trust an autonomous software agent to write files in your source tree, then deciding whether `/bin/cp foo bar` is "safe" depends on `getcwd(3)` for the calling process as well as the `realpath` of `foo` and `bar` when resolved against `getcwd()`. -To that end, rather than returning a boolean, the validator returns a structured result that the client is expected to use to determine the "safety" of the proposed `execv()` call. - -For example, to check the command `ls -l foo`, the checker would be invoked as follows: - -```shell -cargo run -- check ls -l foo | jq -``` - -It will exit with `0` and print the following to stdout: - -```json -{ - "result": "safe", - "match": { - "program": "ls", - "flags": [ - { - "name": "-l" - } - ], - "opts": [], - "args": [ - { - "index": 1, - "type": "ReadableFile", - "value": "foo" - } - ], - "system_path": ["/bin/ls", "/usr/bin/ls"] - } -} -``` - -Of note: - -- `foo` is tagged as a `ReadableFile`, so the caller should resolve `foo` relative to `getcwd()` and `realpath` it (as it may be a symlink) to determine whether `foo` is safe to read. -- While the specified executable is `ls`, `"system_path"` offers `/bin/ls` and `/usr/bin/ls` as viable alternatives to avoid using whatever `ls` happens to appear first on the user's `$PATH`. If either exists on the host, it is recommended to use it as the first argument to `execv(3)` instead of `ls`. - -Further, "safety" in this system is not a guarantee that the command will execute successfully. As an example, `cat /Users/mbolin/code/codex/README.md` may be considered "safe" if the system has decided the agent is allowed to read anything under `/Users/mbolin/code/codex`, but it will fail at runtime if `README.md` does not exist. (Though this is "safe" in that the agent did not read any files that it was not authorized to read.) - -## Policy - -Currently, the default policy is defined in [`default.policy`](./src/default.policy) within the crate. - -The system uses [Starlark](https://bazel.build/rules/language) as the file format because, unlike something like JSON or YAML, it supports "macros" without compromising on safety or reproducibility. (Under the hood, we use [`starlark-rust`](https://github.com/facebook/starlark-rust) as the specific Starlark implementation.) - -This policy contains "rules" such as: - -```python -define_program( - program="cp", - options=[ - flag("-r"), - flag("-R"), - flag("--recursive"), - ], - args=[ARG_RFILES, ARG_WFILE], - system_path=["/bin/cp", "/usr/bin/cp"], - should_match=[ - ["foo", "bar"], - ], - should_not_match=[ - ["foo"], - ], +## Policy shapes +- Prefix rules use Starlark syntax: +```starlark +prefix_rule( + pattern = ["cmd", ["alt1", "alt2"]], # ordered tokens; list entries denote alternatives + decision = "prompt", # allow | prompt | forbidden; defaults to allow + match = [["cmd", "alt1"], "cmd alt2"], # examples that must match this rule + not_match = [["cmd", "oops"], "cmd alt3"], # examples that must not match this rule ) ``` -This rule means that: - -- `cp` can be used with any of the following flags (where "flag" means "an option that does not take an argument"): `-r`, `-R`, `--recursive`. -- The initial `ARG_RFILES` passed to `args` means that it expects one or more arguments that correspond to "readable files" -- The final `ARG_WFILE` passed to `args` means that it expects exactly one argument that corresponds to a "writeable file." -- As a means of a lightweight way of including a unit test alongside the definition, the `should_match` list is a list of examples of `execv(3)` args that should match the rule and `should_not_match` is a list of examples that should not match. These examples are verified when the `.policy` file is loaded. - -Note that the language of the `.policy` file is still evolving, as we have to continue to expand it so it is sufficiently expressive to accept all commands we want to consider "safe" without allowing unsafe commands to pass through. - -The integrity of `default.policy` is verified [via unit tests](./tests). - -Further, the CLI supports a `--policy` option to specify a custom `.policy` file for ad-hoc testing. - -## Output Type: `match` - -Going back to the `cp` example, because the rule matches an `ARG_WFILE`, it will return `match` instead of `safe`: - -```shell -cargo run -- check cp src1 src2 dest | jq +## CLI +- Provide one or more policy files (for example `src/default.codexpolicy`) to check a command: +```bash +cargo run -p codex-execpolicy -- check --policy path/to/policy.codexpolicy git status ``` +- Pass multiple `--policy` flags to merge rules, evaluated in the order provided: +```bash +cargo run -p codex-execpolicy -- check --policy base.codexpolicy --policy overrides.codexpolicy git status +``` +- Output is JSON by default; pass `--pretty` for pretty-printed JSON +- Example outcomes: + - Match: `{"match": { ... "decision": "allow" ... }}` + - No match: `"noMatch"` -If the caller wants to consider allowing this command, it should parse the JSON to pick out the `WriteableFile` arguments and decide whether they are safe to write: - +## Response shapes +- Match: ```json { - "result": "match", "match": { - "program": "cp", - "flags": [], - "opts": [], - "args": [ + "decision": "allow|prompt|forbidden", + "matchedRules": [ { - "index": 0, - "type": "ReadableFile", - "value": "src1" - }, - { - "index": 1, - "type": "ReadableFile", - "value": "src2" - }, - { - "index": 2, - "type": "WriteableFile", - "value": "dest" + "prefixRuleMatch": { + "matchedPrefix": ["", "..."], + "decision": "allow|prompt|forbidden" + } } - ], - "system_path": ["/bin/cp", "/usr/bin/cp"] + ] } } ``` -Note the exit code is still `0` for a `match` unless the `--require-safe` flag is specified, in which case the exit code is `12`. - -## Output Type: `forbidden` - -It is also possible to define a rule that, if it matches a command, should flag it as _forbidden_. For example, we do not want agents to be able to run `applied deploy` _ever_, so we define the following rule: - -```python -define_program( - program="applied", - args=["deploy"], - forbidden="Infrastructure Risk: command contains 'applied deploy'", - should_match=[ - ["deploy"], - ], - should_not_match=[ - ["lint"], - ], -) -``` - -Note that for a rule to be forbidden, the `forbidden` keyword arg must be specified as the reason the command is forbidden. This will be included in the output: - -```shell -cargo run -- check applied deploy | jq -``` - +- No match: ```json -{ - "result": "forbidden", - "reason": "Infrastructure Risk: command contains 'applied deploy'", - "cause": { - "Exec": { - "exec": { - "program": "applied", - "flags": [], - "opts": [], - "args": [ - { - "index": 0, - "type": { - "Literal": "deploy" - }, - "value": "deploy" - } - ], - "system_path": [] - } - } - } -} +"noMatch" ``` + +- `matchedRules` lists every rule whose prefix matched the command; `matchedPrefix` is the exact prefix that matched. +- The effective `decision` is the strictest severity across all matches (`forbidden` > `prompt` > `allow`). diff --git a/codex-rs/execpolicy2/examples/example.codexpolicy b/codex-rs/execpolicy/examples/example.codexpolicy similarity index 100% rename from codex-rs/execpolicy2/examples/example.codexpolicy rename to codex-rs/execpolicy/examples/example.codexpolicy diff --git a/codex-rs/execpolicy2/src/decision.rs b/codex-rs/execpolicy/src/decision.rs similarity index 100% rename from codex-rs/execpolicy2/src/decision.rs rename to codex-rs/execpolicy/src/decision.rs diff --git a/codex-rs/execpolicy/src/error.rs b/codex-rs/execpolicy/src/error.rs index e6443d69dc..2f168a027e 100644 --- a/codex-rs/execpolicy/src/error.rs +++ b/codex-rs/execpolicy/src/error.rs @@ -1,96 +1,26 @@ -use std::path::PathBuf; - -use serde::Serialize; - -use crate::arg_matcher::ArgMatcher; -use crate::arg_resolver::PositionalArg; -use serde_with::DisplayFromStr; -use serde_with::serde_as; +use starlark::Error as StarlarkError; +use thiserror::Error; pub type Result = std::result::Result; -#[serde_as] -#[derive(Debug, Eq, PartialEq, Serialize)] -#[serde(tag = "type")] +#[derive(Debug, Error)] pub enum Error { - NoSpecForProgram { - program: String, - }, - OptionMissingValue { - program: String, - option: String, - }, - OptionFollowedByOptionInsteadOfValue { - program: String, - option: String, - value: String, - }, - UnknownOption { - program: String, - option: String, - }, - UnexpectedArguments { - program: String, - args: Vec, - }, - DoubleDashNotSupportedYet { - program: String, - }, - MultipleVarargPatterns { - program: String, - first: ArgMatcher, - second: ArgMatcher, - }, - RangeStartExceedsEnd { - start: usize, - end: usize, - }, - RangeEndOutOfBounds { - end: usize, - len: usize, - }, - PrefixOverlapsSuffix {}, - NotEnoughArgs { - program: String, - args: Vec, - arg_patterns: Vec, - }, - InternalInvariantViolation { - message: String, - }, - VarargMatcherDidNotMatchAnything { - program: String, - matcher: ArgMatcher, - }, - EmptyFileName {}, - LiteralValueDidNotMatch { - expected: String, - actual: String, - }, - InvalidPositiveInteger { - value: String, - }, - MissingRequiredOptions { - program: String, - options: Vec, - }, - SedCommandNotProvablySafe { - command: String, - }, - ReadablePathNotInReadableFolders { - file: PathBuf, - folders: Vec, - }, - WriteablePathNotInWriteableFolders { - file: PathBuf, - folders: Vec, - }, - CannotCheckRelativePath { - file: PathBuf, - }, - CannotCanonicalizePath { - file: String, - #[serde_as(as = "DisplayFromStr")] - error: std::io::ErrorKind, + #[error("invalid decision: {0}")] + InvalidDecision(String), + #[error("invalid pattern element: {0}")] + InvalidPattern(String), + #[error("invalid example: {0}")] + InvalidExample(String), + #[error( + "expected every example to match at least one rule. rules: {rules:?}; unmatched examples: \ + {examples:?}" + )] + ExampleDidNotMatch { + rules: Vec, + examples: Vec, }, + #[error("expected example to not match rule `{rule}`: {example}")] + ExampleDidMatch { rule: String, example: String }, + #[error("starlark error: {0}")] + Starlark(StarlarkError), } diff --git a/codex-rs/execpolicy/src/lib.rs b/codex-rs/execpolicy/src/lib.rs index 6f12225981..1b789fd862 100644 --- a/codex-rs/execpolicy/src/lib.rs +++ b/codex-rs/execpolicy/src/lib.rs @@ -1,45 +1,15 @@ -#![allow(clippy::type_complexity)] -#![allow(clippy::too_many_arguments)] -#[macro_use] -extern crate starlark; +pub mod decision; +pub mod error; +pub mod parser; +pub mod policy; +pub mod rule; -mod arg_matcher; -mod arg_resolver; -mod arg_type; -mod error; -mod exec_call; -mod execv_checker; -mod opt; -mod policy; -mod policy_parser; -mod program; -mod sed_command; -mod valid_exec; - -pub use arg_matcher::ArgMatcher; -pub use arg_resolver::PositionalArg; -pub use arg_type::ArgType; +pub use decision::Decision; pub use error::Error; pub use error::Result; -pub use exec_call::ExecCall; -pub use execv_checker::ExecvChecker; -pub use opt::Opt; +pub use parser::PolicyParser; +pub use policy::Evaluation; pub use policy::Policy; -pub use policy_parser::PolicyParser; -pub use program::Forbidden; -pub use program::MatchedExec; -pub use program::NegativeExamplePassedCheck; -pub use program::PositiveExampleFailedCheck; -pub use program::ProgramSpec; -pub use sed_command::parse_sed_command; -pub use valid_exec::MatchedArg; -pub use valid_exec::MatchedFlag; -pub use valid_exec::MatchedOpt; -pub use valid_exec::ValidExec; - -const DEFAULT_POLICY: &str = include_str!("default.policy"); - -pub fn get_default_policy() -> starlark::Result { - let parser = PolicyParser::new("#default", DEFAULT_POLICY); - parser.parse() -} +pub use rule::Rule; +pub use rule::RuleMatch; +pub use rule::RuleRef; diff --git a/codex-rs/execpolicy/src/main.rs b/codex-rs/execpolicy/src/main.rs index 68a72c04ff..318abb5cae 100644 --- a/codex-rs/execpolicy/src/main.rs +++ b/codex-rs/execpolicy/src/main.rs @@ -1,167 +1,66 @@ +use std::fs; +use std::path::PathBuf; + +use anyhow::Context; use anyhow::Result; use clap::Parser; -use clap::Subcommand; -use codex_execpolicy::ExecCall; -use codex_execpolicy::MatchedExec; -use codex_execpolicy::Policy; use codex_execpolicy::PolicyParser; -use codex_execpolicy::ValidExec; -use codex_execpolicy::get_default_policy; -use serde::Deserialize; -use serde::Serialize; -use serde::de; -use starlark::Error as StarlarkError; -use std::path::PathBuf; -use std::str::FromStr; -const MATCHED_BUT_WRITES_FILES_EXIT_CODE: i32 = 12; -const MIGHT_BE_SAFE_EXIT_CODE: i32 = 13; -const FORBIDDEN_EXIT_CODE: i32 = 14; - -#[derive(Parser, Deserialize, Debug)] -#[command(version, about, long_about = None)] -pub struct Args { - /// If the command fails the policy, exit with 13, but print parseable JSON - /// to stdout. - #[clap(long)] - pub require_safe: bool, - - /// Path to the policy file. - #[clap(long, short = 'p')] - pub policy: Option, - - #[command(subcommand)] - pub command: Command, -} - -#[derive(Clone, Debug, Deserialize, Subcommand)] -pub enum Command { - /// Checks the command as if the arguments were the inputs to execv(3). +/// CLI for evaluating exec policies +#[derive(Parser)] +#[command(name = "codex-execpolicy")] +enum Cli { + /// Evaluate a command against a policy. Check { - #[arg(trailing_var_arg = true)] + #[arg(short, long = "policy", value_name = "PATH", required = true)] + policies: Vec, + + /// Pretty-print the JSON output. + #[arg(long)] + pretty: bool, + + /// Command tokens to check. + #[arg( + value_name = "COMMAND", + required = true, + trailing_var_arg = true, + allow_hyphen_values = true + )] command: Vec, }, - - /// Checks the command encoded as a JSON object. - #[clap(name = "check-json")] - CheckJson { - /// JSON object with "program" (str) and "args" (list[str]) fields. - #[serde(deserialize_with = "deserialize_from_json")] - exec: ExecArg, - }, -} - -#[derive(Clone, Debug, Deserialize)] -pub struct ExecArg { - pub program: String, - - #[serde(default)] - pub args: Vec, } fn main() -> Result<()> { - env_logger::init(); + let cli = Cli::parse(); + match cli { + Cli::Check { + policies, + command, + pretty, + } => cmd_check(policies, command, pretty), + } +} - let args = Args::parse(); - let policy = match args.policy { - Some(policy) => { - let policy_source = policy.to_string_lossy().to_string(); - let unparsed_policy = std::fs::read_to_string(policy)?; - let parser = PolicyParser::new(&policy_source, &unparsed_policy); - parser.parse() - } - None => get_default_policy(), +fn cmd_check(policy_paths: Vec, args: Vec, pretty: bool) -> Result<()> { + let policy = load_policies(&policy_paths)?; + + let eval = policy.check(&args); + let json = if pretty { + serde_json::to_string_pretty(&eval)? + } else { + serde_json::to_string(&eval)? }; - let policy = policy.map_err(StarlarkError::into_anyhow)?; - - let exec = match args.command { - Command::Check { command } => match command.split_first() { - Some((first, rest)) => ExecArg { - program: first.to_string(), - args: rest.to_vec(), - }, - None => { - eprintln!("no command provided"); - std::process::exit(1); - } - }, - Command::CheckJson { exec } => exec, - }; - - let (output, exit_code) = check_command(&policy, exec, args.require_safe); - let json = serde_json::to_string(&output)?; println!("{json}"); - std::process::exit(exit_code); + Ok(()) } -fn check_command( - policy: &Policy, - ExecArg { program, args }: ExecArg, - check: bool, -) -> (Output, i32) { - let exec_call = ExecCall { program, args }; - match policy.check(&exec_call) { - Ok(MatchedExec::Match { exec }) => { - if exec.might_write_files() { - let exit_code = if check { - MATCHED_BUT_WRITES_FILES_EXIT_CODE - } else { - 0 - }; - (Output::Match { r#match: exec }, exit_code) - } else { - (Output::Safe { r#match: exec }, 0) - } - } - Ok(MatchedExec::Forbidden { reason, cause }) => { - let exit_code = if check { FORBIDDEN_EXIT_CODE } else { 0 }; - (Output::Forbidden { reason, cause }, exit_code) - } - Err(err) => { - let exit_code = if check { MIGHT_BE_SAFE_EXIT_CODE } else { 0 }; - (Output::Unverified { error: err }, exit_code) - } - } -} - -#[derive(Debug, Serialize)] -#[serde(tag = "result")] -pub enum Output { - /// The command is verified as safe. - #[serde(rename = "safe")] - Safe { r#match: ValidExec }, - - /// The command has matched a rule in the policy, but the caller should - /// decide whether it is "safe" given the files it wants to write. - #[serde(rename = "match")] - Match { r#match: ValidExec }, - - /// The user is forbidden from running the command. - #[serde(rename = "forbidden")] - Forbidden { - reason: String, - cause: codex_execpolicy::Forbidden, - }, - - /// The safety of the command could not be verified. - #[serde(rename = "unverified")] - Unverified { error: codex_execpolicy::Error }, -} - -fn deserialize_from_json<'de, D>(deserializer: D) -> Result -where - D: de::Deserializer<'de>, -{ - let s = String::deserialize(deserializer)?; - let decoded = serde_json::from_str(&s) - .map_err(|e| serde::de::Error::custom(format!("JSON parse error: {e}")))?; - Ok(decoded) -} - -impl FromStr for ExecArg { - type Err = anyhow::Error; - - fn from_str(s: &str) -> Result { - serde_json::from_str(s).map_err(Into::into) +fn load_policies(policy_paths: &[PathBuf]) -> Result { + let mut parser = PolicyParser::new(); + for policy_path in policy_paths { + let policy_file_contents = fs::read_to_string(policy_path) + .with_context(|| format!("failed to read policy at {}", policy_path.display()))?; + let policy_identifier = policy_path.to_string_lossy().to_string(); + parser.parse(&policy_identifier, &policy_file_contents)?; } + Ok(parser.build()) } diff --git a/codex-rs/execpolicy2/src/parser.rs b/codex-rs/execpolicy/src/parser.rs similarity index 100% rename from codex-rs/execpolicy2/src/parser.rs rename to codex-rs/execpolicy/src/parser.rs diff --git a/codex-rs/execpolicy/src/policy.rs b/codex-rs/execpolicy/src/policy.rs index 825d6164a5..a543e5c4b0 100644 --- a/codex-rs/execpolicy/src/policy.rs +++ b/codex-rs/execpolicy/src/policy.rs @@ -1,103 +1,84 @@ +use crate::decision::Decision; +use crate::rule::RuleMatch; +use crate::rule::RuleRef; use multimap::MultiMap; -use regex_lite::Error as RegexError; -use regex_lite::Regex; - -use crate::ExecCall; -use crate::Forbidden; -use crate::MatchedExec; -use crate::NegativeExamplePassedCheck; -use crate::ProgramSpec; -use crate::error::Error; -use crate::error::Result; -use crate::policy_parser::ForbiddenProgramRegex; -use crate::program::PositiveExampleFailedCheck; +use serde::Deserialize; +use serde::Serialize; +#[derive(Clone, Debug)] pub struct Policy { - programs: MultiMap, - forbidden_program_regexes: Vec, - forbidden_substrings_pattern: Option, + rules_by_program: MultiMap, } impl Policy { - pub fn new( - programs: MultiMap, - forbidden_program_regexes: Vec, - forbidden_substrings: Vec, - ) -> std::result::Result { - let forbidden_substrings_pattern = if forbidden_substrings.is_empty() { - None - } else { - let escaped_substrings = forbidden_substrings - .iter() - .map(|s| regex_lite::escape(s)) - .collect::>() - .join("|"); - Some(Regex::new(&format!("({escaped_substrings})"))?) + pub fn new(rules_by_program: MultiMap) -> Self { + Self { rules_by_program } + } + + pub fn empty() -> Self { + Self::new(MultiMap::new()) + } + + pub fn rules(&self) -> &MultiMap { + &self.rules_by_program + } + + pub fn check(&self, cmd: &[String]) -> Evaluation { + let rules = match cmd.first() { + Some(first) => match self.rules_by_program.get_vec(first) { + Some(rules) => rules, + None => return Evaluation::NoMatch, + }, + None => return Evaluation::NoMatch, }; - Ok(Self { - programs, - forbidden_program_regexes, - forbidden_substrings_pattern, - }) + + let matched_rules: Vec = + rules.iter().filter_map(|rule| rule.matches(cmd)).collect(); + match matched_rules.iter().map(RuleMatch::decision).max() { + Some(decision) => Evaluation::Match { + decision, + matched_rules, + }, + None => Evaluation::NoMatch, + } } - pub fn check(&self, exec_call: &ExecCall) -> Result { - let ExecCall { program, args } = &exec_call; - for ForbiddenProgramRegex { regex, reason } in &self.forbidden_program_regexes { - if regex.is_match(program) { - return Ok(MatchedExec::Forbidden { - cause: Forbidden::Program { - program: program.clone(), - exec_call: exec_call.clone(), - }, - reason: reason.clone(), - }); - } - } + pub fn check_multiple(&self, commands: Commands) -> Evaluation + where + Commands: IntoIterator, + Commands::Item: AsRef<[String]>, + { + let matched_rules: Vec = commands + .into_iter() + .flat_map(|command| match self.check(command.as_ref()) { + Evaluation::Match { matched_rules, .. } => matched_rules, + Evaluation::NoMatch => Vec::new(), + }) + .collect(); - for arg in args { - if let Some(regex) = &self.forbidden_substrings_pattern - && regex.is_match(arg) - { - return Ok(MatchedExec::Forbidden { - cause: Forbidden::Arg { - arg: arg.clone(), - exec_call: exec_call.clone(), - }, - reason: format!("arg `{arg}` contains forbidden substring"), - }); - } + match matched_rules.iter().map(RuleMatch::decision).max() { + Some(decision) => Evaluation::Match { + decision, + matched_rules, + }, + None => Evaluation::NoMatch, } - - let mut last_err = Err(Error::NoSpecForProgram { - program: program.clone(), - }); - if let Some(spec_list) = self.programs.get_vec(program) { - for spec in spec_list { - match spec.check(exec_call) { - Ok(matched_exec) => return Ok(matched_exec), - Err(err) => { - last_err = Err(err); - } - } - } - } - last_err - } - - pub fn check_each_good_list_individually(&self) -> Vec { - let mut violations = Vec::new(); - for (_program, spec) in self.programs.flat_iter() { - violations.extend(spec.verify_should_match_list()); - } - violations - } - - pub fn check_each_bad_list_individually(&self) -> Vec { - let mut violations = Vec::new(); - for (_program, spec) in self.programs.flat_iter() { - violations.extend(spec.verify_should_not_match_list()); - } - violations + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum Evaluation { + NoMatch, + Match { + decision: Decision, + #[serde(rename = "matchedRules")] + matched_rules: Vec, + }, +} + +impl Evaluation { + pub fn is_match(&self) -> bool { + matches!(self, Self::Match { .. }) } } diff --git a/codex-rs/execpolicy2/src/rule.rs b/codex-rs/execpolicy/src/rule.rs similarity index 100% rename from codex-rs/execpolicy2/src/rule.rs rename to codex-rs/execpolicy/src/rule.rs diff --git a/codex-rs/execpolicy2/tests/basic.rs b/codex-rs/execpolicy/tests/basic.rs similarity index 97% rename from codex-rs/execpolicy2/tests/basic.rs rename to codex-rs/execpolicy/tests/basic.rs index 75921ecd57..301f962616 100644 --- a/codex-rs/execpolicy2/tests/basic.rs +++ b/codex-rs/execpolicy/tests/basic.rs @@ -1,14 +1,14 @@ use std::any::Any; use std::sync::Arc; -use codex_execpolicy2::Decision; -use codex_execpolicy2::Evaluation; -use codex_execpolicy2::PolicyParser; -use codex_execpolicy2::RuleMatch; -use codex_execpolicy2::RuleRef; -use codex_execpolicy2::rule::PatternToken; -use codex_execpolicy2::rule::PrefixPattern; -use codex_execpolicy2::rule::PrefixRule; +use codex_execpolicy::Decision; +use codex_execpolicy::Evaluation; +use codex_execpolicy::PolicyParser; +use codex_execpolicy::RuleMatch; +use codex_execpolicy::RuleRef; +use codex_execpolicy::rule::PatternToken; +use codex_execpolicy::rule::PrefixPattern; +use codex_execpolicy::rule::PrefixRule; use pretty_assertions::assert_eq; fn tokens(cmd: &[&str]) -> Vec { diff --git a/codex-rs/execpolicy2/Cargo.toml b/codex-rs/execpolicy2/Cargo.toml deleted file mode 100644 index c86fb7ff7a..0000000000 --- a/codex-rs/execpolicy2/Cargo.toml +++ /dev/null @@ -1,29 +0,0 @@ -[package] -name = "codex-execpolicy2" -version = { workspace = true } -edition = "2024" -description = "Codex exec policy v2: prefix-based Starlark rules for command decisions." - -[lib] -name = "codex_execpolicy2" -path = "src/lib.rs" - -[[bin]] -name = "codex-execpolicy2" -path = "src/main.rs" - -[lints] -workspace = true - -[dependencies] -anyhow = { workspace = true } -clap = { workspace = true, features = ["derive"] } -multimap = { workspace = true } -serde = { workspace = true, features = ["derive"] } -serde_json = { workspace = true } -shlex = { workspace = true } -starlark = { workspace = true } -thiserror = { workspace = true } - -[dev-dependencies] -pretty_assertions = { workspace = true } diff --git a/codex-rs/execpolicy2/README.md b/codex-rs/execpolicy2/README.md deleted file mode 100644 index d77ec3838d..0000000000 --- a/codex-rs/execpolicy2/README.md +++ /dev/null @@ -1,59 +0,0 @@ -# codex-execpolicy2 - -## Overview -- Policy engine and CLI built around `prefix_rule(pattern=[...], decision?, match?, not_match?)`. -- This release covers only the prefix-rule subset of the planned execpolicy v2 language; a richer language will follow. -- Tokens are matched in order; any `pattern` element may be a list to denote alternatives. `decision` defaults to `allow`; valid values: `allow`, `prompt`, `forbidden`. -- `match` / `not_match` supply example invocations that are validated at load time (think of them as unit tests); examples can be token arrays or strings (strings are tokenized with `shlex`). -- The CLI always prints the JSON serialization of the evaluation result. - -## Policy shapes -- Prefix rules use Starlark syntax: -```starlark -prefix_rule( - pattern = ["cmd", ["alt1", "alt2"]], # ordered tokens; list entries denote alternatives - decision = "prompt", # allow | prompt | forbidden; defaults to allow - match = [["cmd", "alt1"], "cmd alt2"], # examples that must match this rule - not_match = [["cmd", "oops"], "cmd alt3"], # examples that must not match this rule -) -``` - -## CLI -- Provide one or more policy files (for example `src/default.codexpolicy`) to check a command: -```bash -cargo run -p codex-execpolicy2 -- check --policy path/to/policy.codexpolicy git status -``` -- Pass multiple `--policy` flags to merge rules, evaluated in the order provided: -```bash -cargo run -p codex-execpolicy2 -- check --policy base.codexpolicy --policy overrides.codexpolicy git status -``` -- Output is JSON by default; pass `--pretty` for pretty-printed JSON -- Example outcomes: - - Match: `{"match": { ... "decision": "allow" ... }}` - - No match: `"noMatch"` - -## Response shapes -- Match: -```json -{ - "match": { - "decision": "allow|prompt|forbidden", - "matchedRules": [ - { - "prefixRuleMatch": { - "matchedPrefix": ["", "..."], - "decision": "allow|prompt|forbidden" - } - } - ] - } -} -``` - -- No match: -```json -"noMatch" -``` - -- `matchedRules` lists every rule whose prefix matched the command; `matchedPrefix` is the exact prefix that matched. -- The effective `decision` is the strictest severity across all matches (`forbidden` > `prompt` > `allow`). diff --git a/codex-rs/execpolicy2/src/error.rs b/codex-rs/execpolicy2/src/error.rs deleted file mode 100644 index 2f168a027e..0000000000 --- a/codex-rs/execpolicy2/src/error.rs +++ /dev/null @@ -1,26 +0,0 @@ -use starlark::Error as StarlarkError; -use thiserror::Error; - -pub type Result = std::result::Result; - -#[derive(Debug, Error)] -pub enum Error { - #[error("invalid decision: {0}")] - InvalidDecision(String), - #[error("invalid pattern element: {0}")] - InvalidPattern(String), - #[error("invalid example: {0}")] - InvalidExample(String), - #[error( - "expected every example to match at least one rule. rules: {rules:?}; unmatched examples: \ - {examples:?}" - )] - ExampleDidNotMatch { - rules: Vec, - examples: Vec, - }, - #[error("expected example to not match rule `{rule}`: {example}")] - ExampleDidMatch { rule: String, example: String }, - #[error("starlark error: {0}")] - Starlark(StarlarkError), -} diff --git a/codex-rs/execpolicy2/src/lib.rs b/codex-rs/execpolicy2/src/lib.rs deleted file mode 100644 index 1b789fd862..0000000000 --- a/codex-rs/execpolicy2/src/lib.rs +++ /dev/null @@ -1,15 +0,0 @@ -pub mod decision; -pub mod error; -pub mod parser; -pub mod policy; -pub mod rule; - -pub use decision::Decision; -pub use error::Error; -pub use error::Result; -pub use parser::PolicyParser; -pub use policy::Evaluation; -pub use policy::Policy; -pub use rule::Rule; -pub use rule::RuleMatch; -pub use rule::RuleRef; diff --git a/codex-rs/execpolicy2/src/main.rs b/codex-rs/execpolicy2/src/main.rs deleted file mode 100644 index f654c5f84a..0000000000 --- a/codex-rs/execpolicy2/src/main.rs +++ /dev/null @@ -1,66 +0,0 @@ -use std::fs; -use std::path::PathBuf; - -use anyhow::Context; -use anyhow::Result; -use clap::Parser; -use codex_execpolicy2::PolicyParser; - -/// CLI for evaluating exec policies -#[derive(Parser)] -#[command(name = "codex-execpolicy2")] -enum Cli { - /// Evaluate a command against a policy. - Check { - #[arg(short, long = "policy", value_name = "PATH", required = true)] - policies: Vec, - - /// Pretty-print the JSON output. - #[arg(long)] - pretty: bool, - - /// Command tokens to check. - #[arg( - value_name = "COMMAND", - required = true, - trailing_var_arg = true, - allow_hyphen_values = true - )] - command: Vec, - }, -} - -fn main() -> Result<()> { - let cli = Cli::parse(); - match cli { - Cli::Check { - policies, - command, - pretty, - } => cmd_check(policies, command, pretty), - } -} - -fn cmd_check(policy_paths: Vec, args: Vec, pretty: bool) -> Result<()> { - let policy = load_policies(&policy_paths)?; - - let eval = policy.check(&args); - let json = if pretty { - serde_json::to_string_pretty(&eval)? - } else { - serde_json::to_string(&eval)? - }; - println!("{json}"); - Ok(()) -} - -fn load_policies(policy_paths: &[PathBuf]) -> Result { - let mut parser = PolicyParser::new(); - for policy_path in policy_paths { - let policy_file_contents = fs::read_to_string(policy_path) - .with_context(|| format!("failed to read policy at {}", policy_path.display()))?; - let policy_identifier = policy_path.to_string_lossy().to_string(); - parser.parse(&policy_identifier, &policy_file_contents)?; - } - Ok(parser.build()) -} diff --git a/codex-rs/execpolicy2/src/policy.rs b/codex-rs/execpolicy2/src/policy.rs deleted file mode 100644 index a543e5c4b0..0000000000 --- a/codex-rs/execpolicy2/src/policy.rs +++ /dev/null @@ -1,84 +0,0 @@ -use crate::decision::Decision; -use crate::rule::RuleMatch; -use crate::rule::RuleRef; -use multimap::MultiMap; -use serde::Deserialize; -use serde::Serialize; - -#[derive(Clone, Debug)] -pub struct Policy { - rules_by_program: MultiMap, -} - -impl Policy { - pub fn new(rules_by_program: MultiMap) -> Self { - Self { rules_by_program } - } - - pub fn empty() -> Self { - Self::new(MultiMap::new()) - } - - pub fn rules(&self) -> &MultiMap { - &self.rules_by_program - } - - pub fn check(&self, cmd: &[String]) -> Evaluation { - let rules = match cmd.first() { - Some(first) => match self.rules_by_program.get_vec(first) { - Some(rules) => rules, - None => return Evaluation::NoMatch, - }, - None => return Evaluation::NoMatch, - }; - - let matched_rules: Vec = - rules.iter().filter_map(|rule| rule.matches(cmd)).collect(); - match matched_rules.iter().map(RuleMatch::decision).max() { - Some(decision) => Evaluation::Match { - decision, - matched_rules, - }, - None => Evaluation::NoMatch, - } - } - - pub fn check_multiple(&self, commands: Commands) -> Evaluation - where - Commands: IntoIterator, - Commands::Item: AsRef<[String]>, - { - let matched_rules: Vec = commands - .into_iter() - .flat_map(|command| match self.check(command.as_ref()) { - Evaluation::Match { matched_rules, .. } => matched_rules, - Evaluation::NoMatch => Vec::new(), - }) - .collect(); - - match matched_rules.iter().map(RuleMatch::decision).max() { - Some(decision) => Evaluation::Match { - decision, - matched_rules, - }, - None => Evaluation::NoMatch, - } - } -} - -#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum Evaluation { - NoMatch, - Match { - decision: Decision, - #[serde(rename = "matchedRules")] - matched_rules: Vec, - }, -} - -impl Evaluation { - pub fn is_match(&self) -> bool { - matches!(self, Self::Match { .. }) - } -} From d6c30ed25eb9667bb14eb8af20e7b712b25aee64 Mon Sep 17 00:00:00 2001 From: Owen Lin Date: Wed, 19 Nov 2025 20:13:31 -0800 Subject: [PATCH 14/42] [app-server] feat: v2 apply_patch approval flow (#6760) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR adds the API V2 version of the apply_patch approval flow, which centers around `ThreadItem::FileChange`. This PR wires the new RPC (`item/fileChange/requestApproval`, V2 only) and related events (`item/started`, `item/completed` for `ThreadItem::FileChange`, which are emitted in both V1 and V2) through the app-server protocol. The new approval RPC is only sent when the user initiates a turn with the new `turn/start` API so we don't break backwards compatibility with VSCE. Similar to https://github.com/openai/codex/pull/6758, the approach I took was to make as few changes to the Codex core as possible, leveraging existing `EventMsg` core events, and translating those in app-server. I did have to add a few additional fields to `EventMsg::PatchApplyBegin` and `EventMsg::PatchApplyEnd`, but those were fairly lightweight. However, the `EventMsg`s emitted by core are the following: ``` 1) Auto-approved (no request for approval)
 - EventMsg::PatchApplyBegin - EventMsg::PatchApplyEnd 2) Approved by user - EventMsg::ApplyPatchApprovalRequest - EventMsg::PatchApplyBegin - EventMsg::PatchApplyEnd 3) Declined by user - EventMsg::ApplyPatchApprovalRequest - EventMsg::PatchApplyBegin - EventMsg::PatchApplyEnd ``` For a request triggering an approval, this would result in: ``` item/fileChange/requestApproval item/started item/completed ``` which is different from the `ThreadItem::CommandExecution` flow introduced in https://github.com/openai/codex/pull/6758, which does the below and is preferable: ``` item/started item/commandExecution/requestApproval item/completed ``` To fix this, we leverage `TurnSummaryStore` on codex_message_processor to store a little bit of state, allowing us to fire `item/started` and `item/fileChange/requestApproval` whenever we receive the underlying `EventMsg::ApplyPatchApprovalRequest`, and no-oping when we receive the `EventMsg::PatchApplyBegin` later. This is much less invasive than modifying the order of EventMsg within core (I tried). The resulting payloads: ``` { "method": "item/started", "params": { "item": { "changes": [ { "diff": "Hello from Codex!\n", "kind": "add", "path": "/Users/owen/repos/codex/codex-rs/APPROVAL_DEMO.txt" } ], "id": "call_Nxnwj7B3YXigfV6Mwh03d686", "status": "inProgress", "type": "fileChange" } } } ``` ``` { "id": 0, "method": "item/fileChange/requestApproval", "params": { "grantRoot": null, "itemId": "call_Nxnwj7B3YXigfV6Mwh03d686", "reason": null, "threadId": "019a9e11-8295-7883-a283-779e06502c6f", "turnId": "1" } } ``` ``` { "id": 0, "result": { "decision": "accept" } } ``` ``` { "method": "item/completed", "params": { "item": { "changes": [ { "diff": "Hello from Codex!\n", "kind": "add", "path": "/Users/owen/repos/codex/codex-rs/APPROVAL_DEMO.txt" } ], "id": "call_Nxnwj7B3YXigfV6Mwh03d686", "status": "completed", "type": "fileChange" } } } ``` --- .../src/protocol/common.rs | 7 + .../app-server-protocol/src/protocol/v2.rs | 27 +- codex-rs/app-server-test-client/src/main.rs | 36 +++ .../app-server/src/bespoke_event_handling.rs | 261 ++++++++++++++- .../app-server/src/codex_message_processor.rs | 2 + .../app-server/tests/suite/v2/turn_start.rs | 300 ++++++++++++++++++ codex-rs/core/src/codex.rs | 1 + codex-rs/core/src/tools/events.rs | 28 +- .../src/event_processor_with_human_output.rs | 1 + codex-rs/exec/src/exec_events.rs | 1 + .../tests/event_processor_with_json_output.rs | 6 + codex-rs/mcp-server/src/codex_tool_runner.rs | 1 + codex-rs/protocol/src/approvals.rs | 4 + codex-rs/protocol/src/protocol.rs | 11 + codex-rs/tui/src/chatwidget.rs | 1 + codex-rs/tui/src/chatwidget/tests.rs | 28 ++ 16 files changed, 693 insertions(+), 22 deletions(-) diff --git a/codex-rs/app-server-protocol/src/protocol/common.rs b/codex-rs/app-server-protocol/src/protocol/common.rs index 027c2561b0..650b6f31a6 100644 --- a/codex-rs/app-server-protocol/src/protocol/common.rs +++ b/codex-rs/app-server-protocol/src/protocol/common.rs @@ -438,6 +438,13 @@ server_request_definitions! { response: v2::CommandExecutionRequestApprovalResponse, }, + /// Sent when approval is requested for a specific file change. + /// This request is used for Turns started via turn/start. + FileChangeRequestApproval => "item/fileChange/requestApproval" { + params: v2::FileChangeRequestApprovalParams, + response: v2::FileChangeRequestApprovalResponse, + }, + /// DEPRECATED APIs below /// Request to approve a patch. /// This request is used for Turns started via the legacy APIs (i.e. SendUserTurn, SendUserMessage). diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index 3c82d908d3..6b61245bc6 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -794,20 +794,23 @@ pub struct FileUpdateChange { } #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] -#[serde(rename_all = "camelCase")] +#[serde(tag = "type", rename_all = "camelCase")] +#[ts(tag = "type")] #[ts(export_to = "v2/")] pub enum PatchChangeKind { Add, Delete, - Update, + Update { move_path: Option }, } #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] pub enum PatchApplyStatus { + InProgress, Completed, Failed, + Declined, } #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] @@ -982,6 +985,26 @@ pub struct CommandExecutionRequestApprovalResponse { pub accept_settings: Option, } +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct FileChangeRequestApprovalParams { + pub thread_id: String, + pub turn_id: String, + pub item_id: String, + /// Optional explanatory reason (e.g. request for extra write access). + pub reason: Option, + /// [UNSTABLE] When set, the agent is asking the user to allow writes under this root + /// for the remainder of the session (unclear if this is honored today). + pub grant_root: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[ts(export_to = "v2/")] +pub struct FileChangeRequestApprovalResponse { + pub decision: ApprovalDecision, +} + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] diff --git a/codex-rs/app-server-test-client/src/main.rs b/codex-rs/app-server-test-client/src/main.rs index 6130b2d511..7f2b75bd21 100644 --- a/codex-rs/app-server-test-client/src/main.rs +++ b/codex-rs/app-server-test-client/src/main.rs @@ -24,6 +24,8 @@ use codex_app_server_protocol::ClientRequest; use codex_app_server_protocol::CommandExecutionRequestAcceptSettings; use codex_app_server_protocol::CommandExecutionRequestApprovalParams; use codex_app_server_protocol::CommandExecutionRequestApprovalResponse; +use codex_app_server_protocol::FileChangeRequestApprovalParams; +use codex_app_server_protocol::FileChangeRequestApprovalResponse; use codex_app_server_protocol::GetAccountRateLimitsResponse; use codex_app_server_protocol::InitializeParams; use codex_app_server_protocol::InitializeResponse; @@ -677,6 +679,9 @@ impl CodexClient { ServerRequest::CommandExecutionRequestApproval { request_id, params } => { self.handle_command_execution_request_approval(request_id, params)?; } + ServerRequest::FileChangeRequestApproval { request_id, params } => { + self.approve_file_change_request(request_id, params)?; + } other => { bail!("received unsupported server request: {other:?}"); } @@ -717,6 +722,37 @@ impl CodexClient { Ok(()) } + fn approve_file_change_request( + &mut self, + request_id: RequestId, + params: FileChangeRequestApprovalParams, + ) -> Result<()> { + let FileChangeRequestApprovalParams { + thread_id, + turn_id, + item_id, + reason, + grant_root, + } = params; + + println!( + "\n< fileChange approval requested for thread {thread_id}, turn {turn_id}, item {item_id}" + ); + if let Some(reason) = reason.as_deref() { + println!("< reason: {reason}"); + } + if let Some(grant_root) = grant_root.as_deref() { + println!("< grant root: {}", grant_root.display()); + } + + let response = FileChangeRequestApprovalResponse { + decision: ApprovalDecision::Accept, + }; + self.send_server_request_response(request_id, &response)?; + println!("< approved fileChange request for item {item_id}"); + Ok(()) + } + fn send_server_request_response(&mut self, request_id: RequestId, response: &T) -> Result<()> where T: Serialize, diff --git a/codex-rs/app-server/src/bespoke_event_handling.rs b/codex-rs/app-server/src/bespoke_event_handling.rs index b4fdbe3e03..d4b00e67f9 100644 --- a/codex-rs/app-server/src/bespoke_event_handling.rs +++ b/codex-rs/app-server/src/bespoke_event_handling.rs @@ -15,12 +15,17 @@ use codex_app_server_protocol::CommandExecutionRequestApprovalResponse; use codex_app_server_protocol::CommandExecutionStatus; use codex_app_server_protocol::ExecCommandApprovalParams; use codex_app_server_protocol::ExecCommandApprovalResponse; +use codex_app_server_protocol::FileChangeRequestApprovalParams; +use codex_app_server_protocol::FileChangeRequestApprovalResponse; +use codex_app_server_protocol::FileUpdateChange; use codex_app_server_protocol::InterruptConversationResponse; use codex_app_server_protocol::ItemCompletedNotification; use codex_app_server_protocol::ItemStartedNotification; use codex_app_server_protocol::McpToolCallError; use codex_app_server_protocol::McpToolCallResult; use codex_app_server_protocol::McpToolCallStatus; +use codex_app_server_protocol::PatchApplyStatus; +use codex_app_server_protocol::PatchChangeKind as V2PatchChangeKind; use codex_app_server_protocol::ReasoningSummaryPartAddedNotification; use codex_app_server_protocol::ReasoningSummaryTextDeltaNotification; use codex_app_server_protocol::ReasoningTextDeltaNotification; @@ -40,6 +45,7 @@ use codex_core::protocol::Event; use codex_core::protocol::EventMsg; use codex_core::protocol::ExecApprovalRequestEvent; use codex_core::protocol::ExecCommandEndEvent; +use codex_core::protocol::FileChange as CoreFileChange; use codex_core::protocol::McpToolCallBeginEvent; use codex_core::protocol::McpToolCallEndEvent; use codex_core::protocol::Op; @@ -47,7 +53,9 @@ use codex_core::protocol::ReviewDecision; use codex_core::review_format::format_review_findings_block; use codex_protocol::ConversationId; use codex_protocol::protocol::ReviewOutputEvent; +use std::collections::HashMap; use std::convert::TryFrom; +use std::path::PathBuf; use std::sync::Arc; use tokio::sync::oneshot; use tracing::error; @@ -70,24 +78,74 @@ pub(crate) async fn apply_bespoke_event_handling( } EventMsg::ApplyPatchApprovalRequest(ApplyPatchApprovalRequestEvent { call_id, + turn_id, changes, reason, grant_root, - }) => { - let params = ApplyPatchApprovalParams { - conversation_id, - call_id, - file_changes: changes, - reason, - grant_root, - }; - let rx = outgoing - .send_request(ServerRequestPayload::ApplyPatchApproval(params)) - .await; - tokio::spawn(async move { - on_patch_approval_response(event_id, rx, conversation).await; - }); - } + }) => match api_version { + ApiVersion::V1 => { + let params = ApplyPatchApprovalParams { + conversation_id, + call_id, + file_changes: changes.clone(), + reason, + grant_root, + }; + let rx = outgoing + .send_request(ServerRequestPayload::ApplyPatchApproval(params)) + .await; + tokio::spawn(async move { + on_patch_approval_response(event_id, rx, conversation).await; + }); + } + ApiVersion::V2 => { + // Until we migrate the core to be aware of a first class FileChangeItem + // and emit the corresponding EventMsg, we repurpose the call_id as the item_id. + let item_id = call_id.clone(); + let patch_changes = convert_patch_changes(&changes); + + let first_start = { + let mut map = turn_summary_store.lock().await; + let summary = map.entry(conversation_id).or_default(); + summary.file_change_started.insert(item_id.clone()) + }; + if first_start { + let item = ThreadItem::FileChange { + id: item_id.clone(), + changes: patch_changes.clone(), + status: PatchApplyStatus::InProgress, + }; + let notification = ItemStartedNotification { item }; + outgoing + .send_server_notification(ServerNotification::ItemStarted(notification)) + .await; + } + + let params = FileChangeRequestApprovalParams { + thread_id: conversation_id.to_string(), + turn_id: turn_id.clone(), + item_id: item_id.clone(), + reason, + grant_root, + }; + let rx = outgoing + .send_request(ServerRequestPayload::FileChangeRequestApproval(params)) + .await; + tokio::spawn(async move { + on_file_change_request_approval_response( + event_id, + conversation_id, + item_id, + patch_changes, + rx, + conversation, + outgoing, + turn_summary_store, + ) + .await; + }); + } + }, EventMsg::ExecApprovalRequest(ExecApprovalRequestEvent { call_id, turn_id, @@ -244,6 +302,49 @@ pub(crate) async fn apply_bespoke_event_handling( .send_server_notification(ServerNotification::ItemCompleted(notification)) .await; } + EventMsg::PatchApplyBegin(patch_begin_event) => { + // Until we migrate the core to be aware of a first class FileChangeItem + // and emit the corresponding EventMsg, we repurpose the call_id as the item_id. + let item_id = patch_begin_event.call_id.clone(); + + let first_start = { + let mut map = turn_summary_store.lock().await; + let summary = map.entry(conversation_id).or_default(); + summary.file_change_started.insert(item_id.clone()) + }; + if first_start { + let item = ThreadItem::FileChange { + id: item_id.clone(), + changes: convert_patch_changes(&patch_begin_event.changes), + status: PatchApplyStatus::InProgress, + }; + let notification = ItemStartedNotification { item }; + outgoing + .send_server_notification(ServerNotification::ItemStarted(notification)) + .await; + } + } + EventMsg::PatchApplyEnd(patch_end_event) => { + // Until we migrate the core to be aware of a first class FileChangeItem + // and emit the corresponding EventMsg, we repurpose the call_id as the item_id. + let item_id = patch_end_event.call_id.clone(); + + let status = if patch_end_event.success { + PatchApplyStatus::Completed + } else { + PatchApplyStatus::Failed + }; + let changes = convert_patch_changes(&patch_end_event.changes); + complete_file_change_item( + conversation_id, + item_id, + changes, + status, + outgoing.as_ref(), + &turn_summary_store, + ) + .await; + } EventMsg::ExecCommandBegin(exec_command_begin_event) => { let item = ThreadItem::CommandExecution { id: exec_command_begin_event.call_id.clone(), @@ -365,6 +466,32 @@ async fn emit_turn_completed_with_status( .await; } +async fn complete_file_change_item( + conversation_id: ConversationId, + item_id: String, + changes: Vec, + status: PatchApplyStatus, + outgoing: &OutgoingMessageSender, + turn_summary_store: &TurnSummaryStore, +) { + { + let mut map = turn_summary_store.lock().await; + if let Some(summary) = map.get_mut(&conversation_id) { + summary.file_change_started.remove(&item_id); + } + } + + let item = ThreadItem::FileChange { + id: item_id, + changes, + status, + }; + let notification = ItemCompletedNotification { item }; + outgoing + .send_server_notification(ServerNotification::ItemCompleted(notification)) + .await; +} + async fn find_and_remove_turn_summary( conversation_id: ConversationId, turn_summary_store: &TurnSummaryStore, @@ -512,6 +639,110 @@ fn render_review_output_text(output: &ReviewOutputEvent) -> String { } } +fn convert_patch_changes(changes: &HashMap) -> Vec { + let mut converted: Vec = changes + .iter() + .map(|(path, change)| FileUpdateChange { + path: path.to_string_lossy().into_owned(), + kind: map_patch_change_kind(change), + diff: format_file_change_diff(change), + }) + .collect(); + converted.sort_by(|a, b| a.path.cmp(&b.path)); + converted +} + +fn map_patch_change_kind(change: &CoreFileChange) -> V2PatchChangeKind { + match change { + CoreFileChange::Add { .. } => V2PatchChangeKind::Add, + CoreFileChange::Delete { .. } => V2PatchChangeKind::Delete, + CoreFileChange::Update { move_path, .. } => V2PatchChangeKind::Update { + move_path: move_path.clone(), + }, + } +} + +fn format_file_change_diff(change: &CoreFileChange) -> String { + match change { + CoreFileChange::Add { content } => content.clone(), + CoreFileChange::Delete { content } => content.clone(), + CoreFileChange::Update { + unified_diff, + move_path, + } => { + if let Some(path) = move_path { + format!("{unified_diff}\n\nMoved to: {}", path.display()) + } else { + unified_diff.clone() + } + } + } +} + +#[allow(clippy::too_many_arguments)] +async fn on_file_change_request_approval_response( + event_id: String, + conversation_id: ConversationId, + item_id: String, + changes: Vec, + receiver: oneshot::Receiver, + codex: Arc, + outgoing: Arc, + turn_summary_store: TurnSummaryStore, +) { + let response = receiver.await; + let (decision, completion_status) = match response { + Ok(value) => { + let response = serde_json::from_value::(value) + .unwrap_or_else(|err| { + error!("failed to deserialize FileChangeRequestApprovalResponse: {err}"); + FileChangeRequestApprovalResponse { + decision: ApprovalDecision::Decline, + } + }); + + let (decision, completion_status) = match response.decision { + ApprovalDecision::Accept => (ReviewDecision::Approved, None), + ApprovalDecision::Decline => { + (ReviewDecision::Denied, Some(PatchApplyStatus::Declined)) + } + ApprovalDecision::Cancel => { + (ReviewDecision::Abort, Some(PatchApplyStatus::Declined)) + } + }; + // Allow EventMsg::PatchApplyEnd to emit ItemCompleted for accepted patches. + // Only short-circuit on declines/cancels/failures. + (decision, completion_status) + } + Err(err) => { + error!("request failed: {err:?}"); + (ReviewDecision::Denied, Some(PatchApplyStatus::Failed)) + } + }; + + if let Some(status) = completion_status { + complete_file_change_item( + conversation_id, + item_id, + changes, + status, + outgoing.as_ref(), + &turn_summary_store, + ) + .await; + } + + if let Err(err) = codex + .submit(Op::PatchApproval { + id: event_id, + decision, + }) + .await + { + error!("failed to submit PatchApproval: {err}"); + } +} + async fn on_command_execution_request_approval_response( event_id: String, receiver: oneshot::Receiver, diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index 8b6539eb61..bf9f0b9403 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -139,6 +139,7 @@ use codex_protocol::protocol::USER_MESSAGE_BEGIN; use codex_protocol::user_input::UserInput as CoreInputItem; use codex_utils_json_to_toml::json_to_toml; use std::collections::HashMap; +use std::collections::HashSet; use std::ffi::OsStr; use std::io::Error as IoError; use std::path::Path; @@ -162,6 +163,7 @@ pub(crate) type PendingInterrupts = Arc, + pub(crate) file_change_started: HashSet, } pub(crate) type TurnSummaryStore = Arc>>; diff --git a/codex-rs/app-server/tests/suite/v2/turn_start.rs b/codex-rs/app-server/tests/suite/v2/turn_start.rs index 8c4cf4e677..b644315d97 100644 --- a/codex-rs/app-server/tests/suite/v2/turn_start.rs +++ b/codex-rs/app-server/tests/suite/v2/turn_start.rs @@ -1,14 +1,20 @@ use anyhow::Result; use app_test_support::McpProcess; +use app_test_support::create_apply_patch_sse_response; use app_test_support::create_final_assistant_message_sse_response; use app_test_support::create_mock_chat_completions_server; use app_test_support::create_mock_chat_completions_server_unchecked; use app_test_support::create_shell_sse_response; use app_test_support::to_response; +use codex_app_server_protocol::ApprovalDecision; use codex_app_server_protocol::CommandExecutionStatus; +use codex_app_server_protocol::FileChangeRequestApprovalResponse; +use codex_app_server_protocol::ItemCompletedNotification; use codex_app_server_protocol::ItemStartedNotification; use codex_app_server_protocol::JSONRPCNotification; use codex_app_server_protocol::JSONRPCResponse; +use codex_app_server_protocol::PatchApplyStatus; +use codex_app_server_protocol::PatchChangeKind; use codex_app_server_protocol::RequestId; use codex_app_server_protocol::ServerRequest; use codex_app_server_protocol::ThreadItem; @@ -471,6 +477,300 @@ async fn turn_start_updates_sandbox_and_cwd_between_turns_v2() -> Result<()> { Ok(()) } +#[tokio::test] +async fn turn_start_file_change_approval_v2() -> Result<()> { + skip_if_no_network!(Ok(())); + + let tmp = TempDir::new()?; + let codex_home = tmp.path().join("codex_home"); + std::fs::create_dir(&codex_home)?; + let workspace = tmp.path().join("workspace"); + std::fs::create_dir(&workspace)?; + + let patch = r#"*** Begin Patch +*** Add File: README.md ++new line +*** End Patch +"#; + let responses = vec![ + create_apply_patch_sse_response(patch, "patch-call")?, + create_final_assistant_message_sse_response("patch applied")?, + ]; + let server = create_mock_chat_completions_server(responses).await; + create_config_toml(&codex_home, &server.uri(), "untrusted")?; + + let mut mcp = McpProcess::new(&codex_home).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let start_req = mcp + .send_thread_start_request(ThreadStartParams { + model: Some("mock-model".to_string()), + cwd: Some(workspace.to_string_lossy().into_owned()), + ..Default::default() + }) + .await?; + let start_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(start_req)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response::(start_resp)?; + + let turn_req = mcp + .send_turn_start_request(TurnStartParams { + thread_id: thread.id.clone(), + input: vec![V2UserInput::Text { + text: "apply patch".into(), + }], + cwd: Some(workspace.clone()), + ..Default::default() + }) + .await?; + let turn_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(turn_req)), + ) + .await??; + let TurnStartResponse { turn } = to_response::(turn_resp)?; + + let started_file_change = timeout(DEFAULT_READ_TIMEOUT, async { + loop { + let started_notif = mcp + .read_stream_until_notification_message("item/started") + .await?; + let started: ItemStartedNotification = + serde_json::from_value(started_notif.params.clone().expect("item/started params"))?; + if let ThreadItem::FileChange { .. } = started.item { + return Ok::(started.item); + } + } + }) + .await??; + let ThreadItem::FileChange { + ref id, + status, + ref changes, + } = started_file_change + else { + unreachable!("loop ensures we break on file change items"); + }; + assert_eq!(id, "patch-call"); + assert_eq!(status, PatchApplyStatus::InProgress); + let started_changes = changes.clone(); + + let server_req = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_request_message(), + ) + .await??; + let ServerRequest::FileChangeRequestApproval { request_id, params } = server_req else { + panic!("expected FileChangeRequestApproval request") + }; + assert_eq!(params.item_id, "patch-call"); + assert_eq!(params.thread_id, thread.id); + assert_eq!(params.turn_id, turn.id); + let expected_readme_path = workspace.join("README.md"); + let expected_readme_path = expected_readme_path.to_string_lossy().into_owned(); + pretty_assertions::assert_eq!( + started_changes, + vec![codex_app_server_protocol::FileUpdateChange { + path: expected_readme_path.clone(), + kind: PatchChangeKind::Add, + diff: "new line\n".to_string(), + }] + ); + + mcp.send_response( + request_id, + serde_json::to_value(FileChangeRequestApprovalResponse { + decision: ApprovalDecision::Accept, + })?, + ) + .await?; + + let completed_file_change = timeout(DEFAULT_READ_TIMEOUT, async { + loop { + let completed_notif = mcp + .read_stream_until_notification_message("item/completed") + .await?; + let completed: ItemCompletedNotification = serde_json::from_value( + completed_notif + .params + .clone() + .expect("item/completed params"), + )?; + if let ThreadItem::FileChange { .. } = completed.item { + return Ok::(completed.item); + } + } + }) + .await??; + let ThreadItem::FileChange { ref id, status, .. } = completed_file_change else { + unreachable!("loop ensures we break on file change items"); + }; + assert_eq!(id, "patch-call"); + assert_eq!(status, PatchApplyStatus::Completed); + + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("codex/event/task_complete"), + ) + .await??; + + let readme_contents = std::fs::read_to_string(expected_readme_path)?; + assert_eq!(readme_contents, "new line\n"); + + Ok(()) +} + +#[tokio::test] +async fn turn_start_file_change_approval_decline_v2() -> Result<()> { + skip_if_no_network!(Ok(())); + + let tmp = TempDir::new()?; + let codex_home = tmp.path().join("codex_home"); + std::fs::create_dir(&codex_home)?; + let workspace = tmp.path().join("workspace"); + std::fs::create_dir(&workspace)?; + + let patch = r#"*** Begin Patch +*** Add File: README.md ++new line +*** End Patch +"#; + let responses = vec![ + create_apply_patch_sse_response(patch, "patch-call")?, + create_final_assistant_message_sse_response("patch declined")?, + ]; + let server = create_mock_chat_completions_server(responses).await; + create_config_toml(&codex_home, &server.uri(), "untrusted")?; + + let mut mcp = McpProcess::new(&codex_home).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let start_req = mcp + .send_thread_start_request(ThreadStartParams { + model: Some("mock-model".to_string()), + cwd: Some(workspace.to_string_lossy().into_owned()), + ..Default::default() + }) + .await?; + let start_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(start_req)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response::(start_resp)?; + + let turn_req = mcp + .send_turn_start_request(TurnStartParams { + thread_id: thread.id.clone(), + input: vec![V2UserInput::Text { + text: "apply patch".into(), + }], + cwd: Some(workspace.clone()), + ..Default::default() + }) + .await?; + let turn_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(turn_req)), + ) + .await??; + let TurnStartResponse { turn } = to_response::(turn_resp)?; + + let started_file_change = timeout(DEFAULT_READ_TIMEOUT, async { + loop { + let started_notif = mcp + .read_stream_until_notification_message("item/started") + .await?; + let started: ItemStartedNotification = + serde_json::from_value(started_notif.params.clone().expect("item/started params"))?; + if let ThreadItem::FileChange { .. } = started.item { + return Ok::(started.item); + } + } + }) + .await??; + let ThreadItem::FileChange { + ref id, + status, + ref changes, + } = started_file_change + else { + unreachable!("loop ensures we break on file change items"); + }; + assert_eq!(id, "patch-call"); + assert_eq!(status, PatchApplyStatus::InProgress); + let started_changes = changes.clone(); + + let server_req = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_request_message(), + ) + .await??; + let ServerRequest::FileChangeRequestApproval { request_id, params } = server_req else { + panic!("expected FileChangeRequestApproval request") + }; + assert_eq!(params.item_id, "patch-call"); + assert_eq!(params.thread_id, thread.id); + assert_eq!(params.turn_id, turn.id); + let expected_readme_path = workspace.join("README.md"); + let expected_readme_path_str = expected_readme_path.to_string_lossy().into_owned(); + pretty_assertions::assert_eq!( + started_changes, + vec![codex_app_server_protocol::FileUpdateChange { + path: expected_readme_path_str.clone(), + kind: PatchChangeKind::Add, + diff: "new line\n".to_string(), + }] + ); + + mcp.send_response( + request_id, + serde_json::to_value(FileChangeRequestApprovalResponse { + decision: ApprovalDecision::Decline, + })?, + ) + .await?; + + let completed_file_change = timeout(DEFAULT_READ_TIMEOUT, async { + loop { + let completed_notif = mcp + .read_stream_until_notification_message("item/completed") + .await?; + let completed: ItemCompletedNotification = serde_json::from_value( + completed_notif + .params + .clone() + .expect("item/completed params"), + )?; + if let ThreadItem::FileChange { .. } = completed.item { + return Ok::(completed.item); + } + } + }) + .await??; + let ThreadItem::FileChange { ref id, status, .. } = completed_file_change else { + unreachable!("loop ensures we break on file change items"); + }; + assert_eq!(id, "patch-call"); + assert_eq!(status, PatchApplyStatus::Declined); + + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("codex/event/task_complete"), + ) + .await??; + + assert!( + !expected_readme_path.exists(), + "declined patch should not be applied" + ); + + Ok(()) +} + // Helper to create a config.toml pointing at the mock model server. fn create_config_toml( codex_home: &Path, diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 182ca67f41..4d9285dd24 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -919,6 +919,7 @@ impl Session { let event = EventMsg::ApplyPatchApprovalRequest(ApplyPatchApprovalRequestEvent { call_id, + turn_id: turn_context.sub_id.clone(), changes, reason, grant_root, diff --git a/codex-rs/core/src/tools/events.rs b/codex-rs/core/src/tools/events.rs index 131b8f698d..37df12d4b0 100644 --- a/codex-rs/core/src/tools/events.rs +++ b/codex-rs/core/src/tools/events.rs @@ -179,15 +179,17 @@ impl ToolEmitter { ctx.turn, EventMsg::PatchApplyBegin(PatchApplyBeginEvent { call_id: ctx.call_id.to_string(), + turn_id: ctx.turn.sub_id.clone(), auto_approved: *auto_approved, changes: changes.clone(), }), ) .await; } - (Self::ApplyPatch { .. }, ToolEventStage::Success(output)) => { + (Self::ApplyPatch { changes, .. }, ToolEventStage::Success(output)) => { emit_patch_end( ctx, + changes.clone(), output.stdout.text.clone(), output.stderr.text.clone(), output.exit_code == 0, @@ -195,11 +197,12 @@ impl ToolEmitter { .await; } ( - Self::ApplyPatch { .. }, + Self::ApplyPatch { changes, .. }, ToolEventStage::Failure(ToolEventFailure::Output(output)), ) => { emit_patch_end( ctx, + changes.clone(), output.stdout.text.clone(), output.stderr.text.clone(), output.exit_code == 0, @@ -207,10 +210,17 @@ impl ToolEmitter { .await; } ( - Self::ApplyPatch { .. }, + Self::ApplyPatch { changes, .. }, ToolEventStage::Failure(ToolEventFailure::Message(message)), ) => { - emit_patch_end(ctx, String::new(), (*message).to_string(), false).await; + emit_patch_end( + ctx, + changes.clone(), + String::new(), + (*message).to_string(), + false, + ) + .await; } ( Self::UnifiedExec { @@ -409,15 +419,23 @@ async fn emit_exec_end( .await; } -async fn emit_patch_end(ctx: ToolEventCtx<'_>, stdout: String, stderr: String, success: bool) { +async fn emit_patch_end( + ctx: ToolEventCtx<'_>, + changes: HashMap, + stdout: String, + stderr: String, + success: bool, +) { ctx.session .send_event( ctx.turn, EventMsg::PatchApplyEnd(PatchApplyEndEvent { call_id: ctx.call_id.to_string(), + turn_id: ctx.turn.sub_id.clone(), stdout, stderr, success, + changes, }), ) .await; diff --git a/codex-rs/exec/src/event_processor_with_human_output.rs b/codex-rs/exec/src/event_processor_with_human_output.rs index 2d550fea46..1f007bbe0e 100644 --- a/codex-rs/exec/src/event_processor_with_human_output.rs +++ b/codex-rs/exec/src/event_processor_with_human_output.rs @@ -346,6 +346,7 @@ impl EventProcessor for EventProcessorWithHumanOutput { call_id, auto_approved, changes, + .. }) => { // Store metadata so we can calculate duration later when we // receive the corresponding PatchApplyEnd event. diff --git a/codex-rs/exec/src/exec_events.rs b/codex-rs/exec/src/exec_events.rs index 64288e1388..23f471e224 100644 --- a/codex-rs/exec/src/exec_events.rs +++ b/codex-rs/exec/src/exec_events.rs @@ -166,6 +166,7 @@ pub struct FileUpdateChange { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, TS)] #[serde(rename_all = "snake_case")] pub enum PatchApplyStatus { + InProgress, Completed, Failed, } diff --git a/codex-rs/exec/tests/event_processor_with_json_output.rs b/codex-rs/exec/tests/event_processor_with_json_output.rs index 5053f61921..cbcd5458bc 100644 --- a/codex-rs/exec/tests/event_processor_with_json_output.rs +++ b/codex-rs/exec/tests/event_processor_with_json_output.rs @@ -822,6 +822,7 @@ fn patch_apply_success_produces_item_completed_patchapply() { "p1", EventMsg::PatchApplyBegin(PatchApplyBeginEvent { call_id: "call-1".to_string(), + turn_id: "turn-1".to_string(), auto_approved: true, changes: changes.clone(), }), @@ -834,9 +835,11 @@ fn patch_apply_success_produces_item_completed_patchapply() { "p2", EventMsg::PatchApplyEnd(PatchApplyEndEvent { call_id: "call-1".to_string(), + turn_id: "turn-1".to_string(), stdout: "applied 3 changes".to_string(), stderr: String::new(), success: true, + changes: changes.clone(), }), ); let out_end = ep.collect_thread_events(&end); @@ -891,6 +894,7 @@ fn patch_apply_failure_produces_item_completed_patchapply_failed() { "p1", EventMsg::PatchApplyBegin(PatchApplyBeginEvent { call_id: "call-2".to_string(), + turn_id: "turn-2".to_string(), auto_approved: false, changes: changes.clone(), }), @@ -902,9 +906,11 @@ fn patch_apply_failure_produces_item_completed_patchapply_failed() { "p2", EventMsg::PatchApplyEnd(PatchApplyEndEvent { call_id: "call-2".to_string(), + turn_id: "turn-2".to_string(), stdout: String::new(), stderr: "failed to apply".to_string(), success: false, + changes: changes.clone(), }), ); let out_end = ep.collect_thread_events(&end); diff --git a/codex-rs/mcp-server/src/codex_tool_runner.rs b/codex-rs/mcp-server/src/codex_tool_runner.rs index 93dc7764dc..8dccb51250 100644 --- a/codex-rs/mcp-server/src/codex_tool_runner.rs +++ b/codex-rs/mcp-server/src/codex_tool_runner.rs @@ -210,6 +210,7 @@ async fn run_codex_tool_session_inner( } EventMsg::ApplyPatchApprovalRequest(ApplyPatchApprovalRequestEvent { call_id, + turn_id: _, reason, grant_root, changes, diff --git a/codex-rs/protocol/src/approvals.rs b/codex-rs/protocol/src/approvals.rs index f7c5fc6049..25f5e90e9e 100644 --- a/codex-rs/protocol/src/approvals.rs +++ b/codex-rs/protocol/src/approvals.rs @@ -57,6 +57,10 @@ pub struct ExecApprovalRequestEvent { pub struct ApplyPatchApprovalRequestEvent { /// Responses API call id for the associated patch apply call, if available. pub call_id: String, + /// Turn ID that this patch belongs to. + /// Uses `#[serde(default)]` for backwards compatibility with older senders. + #[serde(default)] + pub turn_id: String, pub changes: HashMap, /// Optional explanatory reason (e.g. request for extra write access). #[serde(skip_serializing_if = "Option::is_none")] diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index 1825d7636b..f20e412831 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -1374,6 +1374,10 @@ pub struct StreamInfoEvent { pub struct PatchApplyBeginEvent { /// Identifier so this can be paired with the PatchApplyEnd event. pub call_id: String, + /// Turn ID that this patch belongs to. + /// Uses `#[serde(default)]` for backwards compatibility. + #[serde(default)] + pub turn_id: String, /// If true, there was no ApplyPatchApprovalRequest for this patch. pub auto_approved: bool, /// The changes to be applied. @@ -1384,12 +1388,19 @@ pub struct PatchApplyBeginEvent { pub struct PatchApplyEndEvent { /// Identifier for the PatchApplyBegin that finished. pub call_id: String, + /// Turn ID that this patch belongs to. + /// Uses `#[serde(default)]` for backwards compatibility. + #[serde(default)] + pub turn_id: String, /// Captured stdout (summary printed by apply_patch). pub stdout: String, /// Captured stderr (parser errors, IO failures, etc.). pub stderr: String, /// Whether the patch was applied successfully. pub success: bool, + /// The changes that were applied (mirrors PatchApplyBeginEvent::changes). + #[serde(default)] + pub changes: HashMap, } #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, TS)] diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 987a956fe6..a5728ab14f 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -1464,6 +1464,7 @@ impl ChatWidget { // }), msg: EventMsg::ApplyPatchApprovalRequest(ApplyPatchApprovalRequestEvent { call_id: "1".to_string(), + turn_id: "turn-1".to_string(), changes: HashMap::from([ ( PathBuf::from("/tmp/test.txt"), diff --git a/codex-rs/tui/src/chatwidget/tests.rs b/codex-rs/tui/src/chatwidget/tests.rs index d60a20b8f1..1393e52886 100644 --- a/codex-rs/tui/src/chatwidget/tests.rs +++ b/codex-rs/tui/src/chatwidget/tests.rs @@ -1912,6 +1912,7 @@ fn approval_modal_patch_snapshot() { ); let ev = ApplyPatchApprovalRequestEvent { call_id: "call-approve-patch".into(), + turn_id: "turn-approve-patch".into(), changes, reason: Some("The model wants to apply changes".into()), grant_root: Some(PathBuf::from("/tmp")), @@ -2164,6 +2165,7 @@ fn apply_patch_events_emit_history_cells() { ); let ev = ApplyPatchApprovalRequestEvent { call_id: "c1".into(), + turn_id: "turn-c1".into(), changes, reason: None, grant_root: None, @@ -2204,6 +2206,7 @@ fn apply_patch_events_emit_history_cells() { ); let begin = PatchApplyBeginEvent { call_id: "c1".into(), + turn_id: "turn-c1".into(), auto_approved: true, changes: changes2, }; @@ -2220,11 +2223,20 @@ fn apply_patch_events_emit_history_cells() { ); // 3) End apply success -> success cell + let mut end_changes = HashMap::new(); + end_changes.insert( + PathBuf::from("foo.txt"), + FileChange::Add { + content: "hello\n".to_string(), + }, + ); let end = PatchApplyEndEvent { call_id: "c1".into(), + turn_id: "turn-c1".into(), stdout: "ok\n".into(), stderr: String::new(), success: true, + changes: end_changes, }; chat.handle_codex_event(Event { id: "s1".into(), @@ -2252,6 +2264,7 @@ fn apply_patch_manual_approval_adjusts_header() { id: "s1".into(), msg: EventMsg::ApplyPatchApprovalRequest(ApplyPatchApprovalRequestEvent { call_id: "c1".into(), + turn_id: "turn-c1".into(), changes: proposed_changes, reason: None, grant_root: None, @@ -2270,6 +2283,7 @@ fn apply_patch_manual_approval_adjusts_header() { id: "s1".into(), msg: EventMsg::PatchApplyBegin(PatchApplyBeginEvent { call_id: "c1".into(), + turn_id: "turn-c1".into(), auto_approved: false, changes: apply_changes, }), @@ -2299,6 +2313,7 @@ fn apply_patch_manual_flow_snapshot() { id: "s1".into(), msg: EventMsg::ApplyPatchApprovalRequest(ApplyPatchApprovalRequestEvent { call_id: "c1".into(), + turn_id: "turn-c1".into(), changes: proposed_changes, reason: Some("Manual review required".into()), grant_root: None, @@ -2321,6 +2336,7 @@ fn apply_patch_manual_flow_snapshot() { id: "s1".into(), msg: EventMsg::PatchApplyBegin(PatchApplyBeginEvent { call_id: "c1".into(), + turn_id: "turn-c1".into(), auto_approved: false, changes: apply_changes, }), @@ -2348,6 +2364,7 @@ fn apply_patch_approval_sends_op_with_submission_id() { ); let ev = ApplyPatchApprovalRequestEvent { call_id: "call-999".into(), + turn_id: "turn-999".into(), changes, reason: None, grant_root: None, @@ -2387,6 +2404,7 @@ fn apply_patch_full_flow_integration_like() { id: "sub-xyz".into(), msg: EventMsg::ApplyPatchApprovalRequest(ApplyPatchApprovalRequestEvent { call_id: "call-1".into(), + turn_id: "turn-call-1".into(), changes, reason: None, grant_root: None, @@ -2427,17 +2445,25 @@ fn apply_patch_full_flow_integration_like() { id: "sub-xyz".into(), msg: EventMsg::PatchApplyBegin(PatchApplyBeginEvent { call_id: "call-1".into(), + turn_id: "turn-call-1".into(), auto_approved: false, changes: changes2, }), }); + let mut end_changes = HashMap::new(); + end_changes.insert( + PathBuf::from("pkg.rs"), + FileChange::Add { content: "".into() }, + ); chat.handle_codex_event(Event { id: "sub-xyz".into(), msg: EventMsg::PatchApplyEnd(PatchApplyEndEvent { call_id: "call-1".into(), + turn_id: "turn-call-1".into(), stdout: String::from("ok"), stderr: String::new(), success: true, + changes: end_changes, }), }); } @@ -2458,6 +2484,7 @@ fn apply_patch_untrusted_shows_approval_modal() { id: "sub-1".into(), msg: EventMsg::ApplyPatchApprovalRequest(ApplyPatchApprovalRequestEvent { call_id: "call-1".into(), + turn_id: "turn-call-1".into(), changes, reason: None, grant_root: None, @@ -2506,6 +2533,7 @@ fn apply_patch_request_shows_diff_summary() { id: "sub-apply".into(), msg: EventMsg::ApplyPatchApprovalRequest(ApplyPatchApprovalRequestEvent { call_id: "call-apply".into(), + turn_id: "turn-apply".into(), changes, reason: None, grant_root: None, From e8af41de8a71d8dd653adb0ac29e1c12013615d3 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Wed, 19 Nov 2025 20:59:17 -0800 Subject: [PATCH 15/42] fix: clean up elicitation used by exec-server (#6958) Using appropriate message/title fields, I think this looks better now: image Though note that in the current version of the Inspector (`0.17.2`), you cannot hit **Submit** until you fill out the field. I believe this is a bug in the Inspector, as it does not properly handle the case when all fields are optional. I put up a fix: https://github.com/modelcontextprotocol/inspector/pull/926 --- .../src/posix/mcp_escalation_policy.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/codex-rs/exec-server/src/posix/mcp_escalation_policy.rs b/codex-rs/exec-server/src/posix/mcp_escalation_policy.rs index 24f42fe91b..39958bb4f9 100644 --- a/codex-rs/exec-server/src/posix/mcp_escalation_policy.rs +++ b/codex-rs/exec-server/src/posix/mcp_escalation_policy.rs @@ -6,8 +6,6 @@ use rmcp::model::CreateElicitationRequestParam; use rmcp::model::CreateElicitationResult; use rmcp::model::ElicitationAction; use rmcp::model::ElicitationSchema; -use rmcp::model::PrimitiveSchema; -use rmcp::model::StringSchema; use rmcp::service::RequestContext; use crate::posix::escalate_protocol::EscalateAction; @@ -54,12 +52,19 @@ impl McpEscalationPolicy { context .peer .create_elicitation(CreateElicitationRequestParam { - message: format!("Allow Codex to run `{command:?}` in `{workdir:?}`?"), - #[allow(clippy::expect_used)] + message: format!("Allow agent to run `{command}` in `{}`?", workdir.display()), requested_schema: ElicitationSchema::builder() - .property("dummy", PrimitiveSchema::String(StringSchema::new())) + .title("Execution Permission Request") + .optional_string_with("reason", |schema| { + schema.description("Optional reason for allowing or denying execution") + }) .build() - .expect("failed to build elicitation schema"), + .map_err(|e| { + McpError::internal_error( + format!("failed to build elicitation schema: {e}"), + None, + ) + })?, }) .await .map_err(|e| McpError::internal_error(e.to_string(), None)) From 54e6e4ac325c2fd207a95bccb35f454433553a8b Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Wed, 19 Nov 2025 22:42:58 -0800 Subject: [PATCH 16/42] fix: when displaying execv, show file instead of arg0 (#6966) After merging https://github.com/openai/codex/pull/6958, I realized that the `command` I was displaying was not quite right. Since we know it, we should show the _exact_ program being executed (the first arg to `execve(3)`) rather than `arg0` to be more precise. Below is the same command I used to test https://github.com/openai/codex/pull/6958, but now you can see it shows `/Users/mbolin/.openai/bin/git` instead of just `git`. image --- codex-rs/exec-server/src/posix/mcp_escalation_policy.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/codex-rs/exec-server/src/posix/mcp_escalation_policy.rs b/codex-rs/exec-server/src/posix/mcp_escalation_policy.rs index 39958bb4f9..069948ea06 100644 --- a/codex-rs/exec-server/src/posix/mcp_escalation_policy.rs +++ b/codex-rs/exec-server/src/posix/mcp_escalation_policy.rs @@ -43,12 +43,17 @@ impl McpEscalationPolicy { async fn prompt( &self, - _file: &Path, + file: &Path, argv: &[String], workdir: &Path, context: RequestContext, ) -> Result { - let command = shlex::try_join(argv.iter().map(String::as_str)).unwrap_or_default(); + let args = shlex::try_join(argv.iter().skip(1).map(String::as_str)).unwrap_or_default(); + let command = if args.is_empty() { + file.display().to_string() + } else { + format!("{} {}", file.display(), args) + }; context .peer .create_elicitation(CreateElicitationRequestParam { From b5dd1890671d58dd92c29dc4ffa6432ecb48d54c Mon Sep 17 00:00:00 2001 From: hanson-openai Date: Thu, 20 Nov 2025 04:34:41 -0800 Subject: [PATCH 17/42] Allow unified_exec to early exit (if the process terminates before yield_time_ms) (#6867) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thread through an `exit_notify` tokio `Notify` through to the `UnifiedExecSession` so that we can return early if the command terminates before `yield_time_ms`. As Codex review correctly pointed out below 🙌 we also need a `exit_signaled` flag so that commands which finish before we start waiting can also exit early. Since the default `yield_time_ms` is now 10s, this means that we don't have to wait 10s for trivial commands like ls, sed, etc (which are the majority of agent commands 😅) --------- Co-authored-by: jif-oai --- codex-rs/core/src/unified_exec/session.rs | 42 +++++-- .../core/src/unified_exec/session_manager.rs | 116 +++++++++++------- codex-rs/core/tests/suite/unified_exec.rs | 92 ++++++++++++++ 3 files changed, 198 insertions(+), 52 deletions(-) diff --git a/codex-rs/core/src/unified_exec/session.rs b/codex-rs/core/src/unified_exec/session.rs index b37a9cdb55..710334c806 100644 --- a/codex-rs/core/src/unified_exec/session.rs +++ b/codex-rs/core/src/unified_exec/session.rs @@ -2,13 +2,13 @@ use std::collections::VecDeque; use std::sync::Arc; - use tokio::sync::Mutex; use tokio::sync::Notify; use tokio::sync::mpsc; use tokio::sync::oneshot::error::TryRecvError; use tokio::task::JoinHandle; use tokio::time::Duration; +use tokio_util::sync::CancellationToken; use crate::exec::ExecToolCallOutput; use crate::exec::SandboxType; @@ -67,13 +67,18 @@ impl OutputBufferState { } pub(crate) type OutputBuffer = Arc>; -pub(crate) type OutputHandles = (OutputBuffer, Arc); +pub(crate) struct OutputHandles { + pub(crate) output_buffer: OutputBuffer, + pub(crate) output_notify: Arc, + pub(crate) cancellation_token: CancellationToken, +} #[derive(Debug)] pub(crate) struct UnifiedExecSession { session: ExecCommandSession, output_buffer: OutputBuffer, output_notify: Arc, + cancellation_token: CancellationToken, output_task: JoinHandle<()>, sandbox_type: SandboxType, } @@ -86,9 +91,11 @@ impl UnifiedExecSession { ) -> Self { let output_buffer = Arc::new(Mutex::new(OutputBufferState::default())); let output_notify = Arc::new(Notify::new()); + let cancellation_token = CancellationToken::new(); let mut receiver = initial_output_rx; let buffer_clone = Arc::clone(&output_buffer); let notify_clone = Arc::clone(&output_notify); + let cancellation_token_clone = cancellation_token.clone(); let output_task = tokio::spawn(async move { loop { match receiver.recv().await { @@ -99,7 +106,10 @@ impl UnifiedExecSession { notify_clone.notify_waiters(); } Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => continue, - Err(tokio::sync::broadcast::error::RecvError::Closed) => break, + Err(tokio::sync::broadcast::error::RecvError::Closed) => { + cancellation_token_clone.cancel(); + break; + } } } }); @@ -108,6 +118,7 @@ impl UnifiedExecSession { session, output_buffer, output_notify, + cancellation_token, output_task, sandbox_type, } @@ -118,10 +129,11 @@ impl UnifiedExecSession { } pub(super) fn output_handles(&self) -> OutputHandles { - ( - Arc::clone(&self.output_buffer), - Arc::clone(&self.output_notify), - ) + OutputHandles { + output_buffer: Arc::clone(&self.output_buffer), + output_notify: Arc::clone(&self.output_notify), + cancellation_token: self.cancellation_token.clone(), + } } pub(super) fn has_exited(&self) -> bool { @@ -199,20 +211,34 @@ impl UnifiedExecSession { }; if exit_ready { + managed.signal_exit(); managed.check_for_sandbox_denial().await?; return Ok(managed); } - tokio::pin!(exit_rx); if tokio::time::timeout(Duration::from_millis(50), &mut exit_rx) .await .is_ok() { + managed.signal_exit(); managed.check_for_sandbox_denial().await?; + return Ok(managed); } + tokio::spawn({ + let cancellation_token = managed.cancellation_token.clone(); + async move { + let _ = exit_rx.await; + cancellation_token.cancel(); + } + }); + Ok(managed) } + + fn signal_exit(&self) { + self.cancellation_token.cancel(); + } } impl Drop for UnifiedExecSession { diff --git a/codex-rs/core/src/unified_exec/session_manager.rs b/codex-rs/core/src/unified_exec/session_manager.rs index 93340bb2d4..d9f99b9eaf 100644 --- a/codex-rs/core/src/unified_exec/session_manager.rs +++ b/codex-rs/core/src/unified_exec/session_manager.rs @@ -5,6 +5,7 @@ use tokio::sync::Notify; use tokio::sync::mpsc; use tokio::time::Duration; use tokio::time::Instant; +use tokio_util::sync::CancellationToken; use crate::codex::Session; use crate::codex::TurnContext; @@ -40,8 +41,20 @@ use super::clamp_yield_time; use super::generate_chunk_id; use super::resolve_max_tokens; use super::session::OutputBuffer; +use super::session::OutputHandles; use super::session::UnifiedExecSession; +struct PreparedSessionHandles { + writer_tx: mpsc::Sender>, + output_buffer: OutputBuffer, + output_notify: Arc, + cancellation_token: CancellationToken, + session_ref: Arc, + turn_ref: Arc, + command: Vec, + cwd: PathBuf, +} + impl UnifiedExecSessionManager { pub(crate) async fn exec_command( &self, @@ -67,10 +80,19 @@ impl UnifiedExecSessionManager { let yield_time_ms = clamp_yield_time(request.yield_time_ms); let start = Instant::now(); - let (output_buffer, output_notify) = session.output_handles(); + let OutputHandles { + output_buffer, + output_notify, + cancellation_token, + } = session.output_handles(); let deadline = start + Duration::from_millis(yield_time_ms); - let collected = - Self::collect_output_until_deadline(&output_buffer, &output_notify, deadline).await; + let collected = Self::collect_output_until_deadline( + &output_buffer, + &output_notify, + &cancellation_token, + deadline, + ) + .await; let wall_time = Instant::now().saturating_duration_since(start); let text = String::from_utf8_lossy(&collected).to_string(); @@ -129,15 +151,16 @@ impl UnifiedExecSessionManager { ) -> Result { let session_id = request.session_id; - let ( + let PreparedSessionHandles { writer_tx, output_buffer, output_notify, + cancellation_token, session_ref, turn_ref, - session_command, - session_cwd, - ) = self.prepare_session_handles(session_id).await?; + command: session_command, + cwd: session_cwd, + } = self.prepare_session_handles(session_id).await?; let interaction_emitter = ToolEmitter::unified_exec( &session_command, @@ -176,8 +199,13 @@ impl UnifiedExecSessionManager { let yield_time_ms = clamp_yield_time(request.yield_time_ms); let start = Instant::now(); let deadline = start + Duration::from_millis(yield_time_ms); - let collected = - Self::collect_output_until_deadline(&output_buffer, &output_notify, deadline).await; + let collected = Self::collect_output_until_deadline( + &output_buffer, + &output_notify, + &cancellation_token, + deadline, + ) + .await; let wall_time = Instant::now().saturating_duration_since(start); let text = String::from_utf8_lossy(&collected).to_string(); @@ -265,44 +293,27 @@ impl UnifiedExecSessionManager { async fn prepare_session_handles( &self, session_id: i32, - ) -> Result< - ( - mpsc::Sender>, - OutputBuffer, - Arc, - Arc, - Arc, - Vec, - PathBuf, - ), - UnifiedExecError, - > { + ) -> Result { let sessions = self.sessions.lock().await; - let (output_buffer, output_notify, writer_tx, session, turn, command, cwd) = - if let Some(entry) = sessions.get(&session_id) { - let (buffer, notify) = entry.session.output_handles(); - ( - buffer, - notify, - entry.session.writer_sender(), - Arc::clone(&entry.session_ref), - Arc::clone(&entry.turn_ref), - entry.command.clone(), - entry.cwd.clone(), - ) - } else { - return Err(UnifiedExecError::UnknownSessionId { session_id }); - }; - - Ok(( - writer_tx, + let entry = sessions + .get(&session_id) + .ok_or(UnifiedExecError::UnknownSessionId { session_id })?; + let OutputHandles { output_buffer, output_notify, - session, - turn, - command, - cwd, - )) + cancellation_token, + } = entry.session.output_handles(); + + Ok(PreparedSessionHandles { + writer_tx: entry.session.writer_sender(), + output_buffer, + output_notify, + cancellation_token, + session_ref: Arc::clone(&entry.session_ref), + turn_ref: Arc::clone(&entry.turn_ref), + command: entry.command.clone(), + cwd: entry.cwd.clone(), + }) } async fn send_input( @@ -480,9 +491,13 @@ impl UnifiedExecSessionManager { pub(super) async fn collect_output_until_deadline( output_buffer: &OutputBuffer, output_notify: &Arc, + cancellation_token: &CancellationToken, deadline: Instant, ) -> Vec { + const POST_EXIT_OUTPUT_GRACE: Duration = Duration::from_millis(25); + let mut collected: Vec = Vec::with_capacity(4096); + let mut exit_signal_received = cancellation_token.is_cancelled(); loop { let drained_chunks; let mut wait_for_output = None; @@ -495,15 +510,27 @@ impl UnifiedExecSessionManager { } if drained_chunks.is_empty() { + exit_signal_received |= cancellation_token.is_cancelled(); let remaining = deadline.saturating_duration_since(Instant::now()); if remaining == Duration::ZERO { break; } let notified = wait_for_output.unwrap_or_else(|| output_notify.notified()); + if exit_signal_received { + let grace = remaining.min(POST_EXIT_OUTPUT_GRACE); + if tokio::time::timeout(grace, notified).await.is_err() { + break; + } + continue; + } + tokio::pin!(notified); + let exit_notified = cancellation_token.cancelled(); + tokio::pin!(exit_notified); tokio::select! { _ = &mut notified => {} + _ = &mut exit_notified => exit_signal_received = true, _ = tokio::time::sleep(remaining) => break, } continue; @@ -513,6 +540,7 @@ impl UnifiedExecSessionManager { collected.extend_from_slice(&chunk); } + exit_signal_received |= cancellation_token.is_cancelled(); if Instant::now() >= deadline { break; } diff --git a/codex-rs/core/tests/suite/unified_exec.rs b/codex-rs/core/tests/suite/unified_exec.rs index 07a0b21a7f..d3059aba42 100644 --- a/codex-rs/core/tests/suite/unified_exec.rs +++ b/codex-rs/core/tests/suite/unified_exec.rs @@ -904,6 +904,98 @@ async fn exec_command_reports_chunk_and_exit_metadata() -> Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn unified_exec_respects_early_exit_notifications() -> Result<()> { + skip_if_no_network!(Ok(())); + skip_if_sandbox!(Ok(())); + + let server = start_mock_server().await; + + let mut builder = test_codex().with_config(|config| { + config.features.enable(Feature::UnifiedExec); + }); + let TestCodex { + codex, + cwd, + session_configured, + .. + } = builder.build(&server).await?; + + let call_id = "uexec-early-exit"; + let args = serde_json::json!({ + "cmd": "sleep 0.05", + "yield_time_ms": 31415, + }); + + let responses = vec![ + sse(vec![ + ev_response_created("resp-1"), + ev_function_call(call_id, "exec_command", &serde_json::to_string(&args)?), + ev_completed("resp-1"), + ]), + sse(vec![ + ev_assistant_message("msg-1", "done"), + ev_completed("resp-2"), + ]), + ]; + mount_sse_sequence(&server, responses).await; + + let session_model = session_configured.model.clone(); + + codex + .submit(Op::UserTurn { + items: vec![UserInput::Text { + text: "watch early exit timing".into(), + }], + final_output_json_schema: None, + cwd: cwd.path().to_path_buf(), + approval_policy: AskForApproval::Never, + sandbox_policy: SandboxPolicy::DangerFullAccess, + model: session_model, + effort: None, + summary: ReasoningSummary::Auto, + }) + .await?; + + wait_for_event(&codex, |event| matches!(event, EventMsg::TaskComplete(_))).await; + + let requests = server.received_requests().await.expect("recorded requests"); + assert!(!requests.is_empty(), "expected at least one POST request"); + + let bodies = requests + .iter() + .map(|req| req.body_json::().expect("request json")) + .collect::>(); + + let outputs = collect_tool_outputs(&bodies)?; + let output = outputs + .get(call_id) + .expect("missing early exit unified_exec output"); + + assert!( + output.session_id.is_none(), + "short-lived process should not keep a session alive" + ); + assert_eq!( + output.exit_code, + Some(0), + "short-lived process should exit successfully" + ); + + let wall_time = output.wall_time_seconds; + assert!( + wall_time < 0.75, + "wall_time should reflect early exit rather than the full yield time; got {wall_time}" + ); + assert!( + output.output.is_empty(), + "sleep command should not emit output, got {:?}", + output.output + ); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn write_stdin_returns_exit_metadata_and_clears_session() -> Result<()> { skip_if_no_network!(Ok(())); From 888c6dd9e76564bafb6109d8b6f407d1ef07a78f Mon Sep 17 00:00:00 2001 From: jif-oai Date: Thu, 20 Nov 2025 17:29:15 +0100 Subject: [PATCH 18/42] fix: command formatting for user commands (#7002) --- ...atwidget__tests__user_shell_ls_output.snap | 7 ++++++ codex-rs/tui/src/chatwidget/tests.rs | 22 +++++++++++++++++++ codex-rs/tui/src/exec_cell/model.rs | 3 ++- 3 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__user_shell_ls_output.snap diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__user_shell_ls_output.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__user_shell_ls_output.snap new file mode 100644 index 0000000000..c67cd637d7 --- /dev/null +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__user_shell_ls_output.snap @@ -0,0 +1,7 @@ +--- +source: tui/src/chatwidget/tests.rs +expression: blob +--- +• You ran ls + └ file1 + file2 diff --git a/codex-rs/tui/src/chatwidget/tests.rs b/codex-rs/tui/src/chatwidget/tests.rs index 1393e52886..f07d7a51d2 100644 --- a/codex-rs/tui/src/chatwidget/tests.rs +++ b/codex-rs/tui/src/chatwidget/tests.rs @@ -1790,6 +1790,28 @@ fn exec_history_extends_previous_when_consecutive() { assert_snapshot!("exploring_step6_finish_cat_bar", active_blob(&chat)); } +#[test] +fn user_shell_command_renders_output_not_exploring() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(); + + let begin_ls = begin_exec_with_source( + &mut chat, + "user-shell-ls", + "ls", + ExecCommandSource::UserShell, + ); + end_exec(&mut chat, begin_ls, "file1\nfile2\n", "", 0); + + let cells = drain_insert_history(&mut rx); + assert_eq!( + cells.len(), + 1, + "expected a single history cell for the user command" + ); + let blob = lines_to_single_string(cells.first().unwrap()); + assert_snapshot!("user_shell_ls_output", blob); +} + #[test] fn disabled_slash_command_while_task_running_snapshot() { // Build a chat widget and simulate an active task diff --git a/codex-rs/tui/src/exec_cell/model.rs b/codex-rs/tui/src/exec_cell/model.rs index 943fb83654..3fddee1562 100644 --- a/codex-rs/tui/src/exec_cell/model.rs +++ b/codex-rs/tui/src/exec_cell/model.rs @@ -117,7 +117,8 @@ impl ExecCell { } pub(super) fn is_exploring_call(call: &ExecCall) -> bool { - !call.parsed.is_empty() + !matches!(call.source, ExecCommandSource::UserShell) + && !call.parsed.is_empty() && call.parsed.iter().all(|p| { matches!( p, From d909048a857aa93ae4dd542328ae9118f6e952ba Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Thu, 20 Nov 2025 12:40:08 -0600 Subject: [PATCH 19/42] Added feature switch to disable animations in TUI (#6870) This PR adds support for a new feature flag `tui.animations`. By default, the TUI uses animations in its welcome screen, "working" spinners, and "shimmer" effects. This animations can interfere with screen readers, so it's good to provide a way to disable them. This change is inspired by [a PR](https://github.com/openai/codex/pull/4014) contributed by @Orinks. That PR has faltered a bit, but I think the core idea is sound. This version incorporates feedback from @aibrahim-oai. In particular: 1. It uses a feature flag (`tui.animations`) rather than the unqualified CLI key `no-animations`. Feature flags are the preferred way to expose boolean switches. They are also exposed via CLI command switches. 2. It includes more complete documentation. 3. It disables a few animations that the other PR omitted. --- codex-rs/core/src/config/mod.rs | 8 + codex-rs/core/src/config/types.rs | 9 + codex-rs/tui/src/bottom_pane/mod.rs | 36 +- codex-rs/tui/src/chatwidget.rs | 11 +- codex-rs/tui/src/chatwidget/tests.rs | 1 + codex-rs/tui/src/exec_cell/model.rs | 13 +- codex-rs/tui/src/exec_cell/render.rs | 33 +- codex-rs/tui/src/history_cell.rs | 327 ++++++++++-------- codex-rs/tui/src/onboarding/auth.rs | 14 +- .../tui/src/onboarding/onboarding_screen.rs | 2 + codex-rs/tui/src/onboarding/welcome.rs | 22 +- codex-rs/tui/src/pager_overlay.rs | 1 + codex-rs/tui/src/status_indicator_widget.rs | 23 +- docs/config.md | 5 + docs/example-config.md | 9 +- 15 files changed, 326 insertions(+), 188 deletions(-) diff --git a/codex-rs/core/src/config/mod.rs b/codex-rs/core/src/config/mod.rs index 4cdb1b0b03..bf7203a9bb 100644 --- a/codex-rs/core/src/config/mod.rs +++ b/codex-rs/core/src/config/mod.rs @@ -160,6 +160,9 @@ pub struct Config { /// and turn completions when not focused. pub tui_notifications: Notifications, + /// Enable ASCII animations and shimmer effects in the TUI. + pub animations: bool, + /// The directory that should be treated as the current working directory /// for the session. All relative paths inside the business-logic layer are /// resolved against this path. @@ -1253,6 +1256,7 @@ impl Config { .as_ref() .map(|t| t.notifications.clone()) .unwrap_or_default(), + animations: cfg.tui.as_ref().map(|t| t.animations).unwrap_or(true), otel: { let t: OtelConfigToml = cfg.otel.unwrap_or_default(); let log_user_prompt = t.log_user_prompt.unwrap_or(false); @@ -3003,6 +3007,7 @@ model_verbosity = "high" notices: Default::default(), disable_paste_burst: false, tui_notifications: Default::default(), + animations: true, otel: OtelConfig::default(), }, o3_profile_config @@ -3075,6 +3080,7 @@ model_verbosity = "high" notices: Default::default(), disable_paste_burst: false, tui_notifications: Default::default(), + animations: true, otel: OtelConfig::default(), }; @@ -3162,6 +3168,7 @@ model_verbosity = "high" notices: Default::default(), disable_paste_burst: false, tui_notifications: Default::default(), + animations: true, otel: OtelConfig::default(), }; @@ -3235,6 +3242,7 @@ model_verbosity = "high" notices: Default::default(), disable_paste_burst: false, tui_notifications: Default::default(), + animations: true, otel: OtelConfig::default(), }; diff --git a/codex-rs/core/src/config/types.rs b/codex-rs/core/src/config/types.rs index 8f5077410e..869ec82973 100644 --- a/codex-rs/core/src/config/types.rs +++ b/codex-rs/core/src/config/types.rs @@ -363,6 +363,15 @@ pub struct Tui { /// Defaults to `true`. #[serde(default)] pub notifications: Notifications, + + /// Enable animations (welcome screen, shimmer effects, spinners). + /// Defaults to `true`. + #[serde(default = "default_true")] + pub animations: bool, +} + +const fn default_true() -> bool { + true } /// Settings for notices we display to users via the tui and app-server clients diff --git a/codex-rs/tui/src/bottom_pane/mod.rs b/codex-rs/tui/src/bottom_pane/mod.rs index 6738d7672d..5dbfb210b2 100644 --- a/codex-rs/tui/src/bottom_pane/mod.rs +++ b/codex-rs/tui/src/bottom_pane/mod.rs @@ -69,6 +69,7 @@ pub(crate) struct BottomPane { is_task_running: bool, ctrl_c_quit_hint: bool, esc_backtrack_hint: bool, + animations_enabled: bool, /// Inline status indicator shown above the composer while a task is running. status: Option, @@ -84,28 +85,38 @@ pub(crate) struct BottomPaneParams { pub(crate) enhanced_keys_supported: bool, pub(crate) placeholder_text: String, pub(crate) disable_paste_burst: bool, + pub(crate) animations_enabled: bool, } impl BottomPane { pub fn new(params: BottomPaneParams) -> Self { - let enhanced_keys_supported = params.enhanced_keys_supported; + let BottomPaneParams { + app_event_tx, + frame_requester, + has_input_focus, + enhanced_keys_supported, + placeholder_text, + disable_paste_burst, + animations_enabled, + } = params; Self { composer: ChatComposer::new( - params.has_input_focus, - params.app_event_tx.clone(), + has_input_focus, + app_event_tx.clone(), enhanced_keys_supported, - params.placeholder_text, - params.disable_paste_burst, + placeholder_text, + disable_paste_burst, ), view_stack: Vec::new(), - app_event_tx: params.app_event_tx, - frame_requester: params.frame_requester, - has_input_focus: params.has_input_focus, + app_event_tx, + frame_requester, + has_input_focus, is_task_running: false, ctrl_c_quit_hint: false, status: None, queued_user_messages: QueuedUserMessages::new(), esc_backtrack_hint: false, + animations_enabled, context_window_percent: None, } } @@ -294,6 +305,7 @@ impl BottomPane { self.status = Some(StatusIndicatorWidget::new( self.app_event_tx.clone(), self.frame_requester.clone(), + self.animations_enabled, )); } if let Some(status) = self.status.as_mut() { @@ -319,6 +331,7 @@ impl BottomPane { self.status = Some(StatusIndicatorWidget::new( self.app_event_tx.clone(), self.frame_requester.clone(), + self.animations_enabled, )); self.request_redraw(); } @@ -554,6 +567,7 @@ mod tests { enhanced_keys_supported: false, placeholder_text: "Ask Codex to do anything".to_string(), disable_paste_burst: false, + animations_enabled: true, }); pane.push_approval_request(exec_request()); assert_eq!(CancellationEvent::Handled, pane.on_ctrl_c()); @@ -574,6 +588,7 @@ mod tests { enhanced_keys_supported: false, placeholder_text: "Ask Codex to do anything".to_string(), disable_paste_burst: false, + animations_enabled: true, }); // Create an approval modal (active view). @@ -605,6 +620,7 @@ mod tests { enhanced_keys_supported: false, placeholder_text: "Ask Codex to do anything".to_string(), disable_paste_burst: false, + animations_enabled: true, }); // Start a running task so the status indicator is active above the composer. @@ -670,6 +686,7 @@ mod tests { enhanced_keys_supported: false, placeholder_text: "Ask Codex to do anything".to_string(), disable_paste_burst: false, + animations_enabled: true, }); // Begin a task: show initial status. @@ -695,6 +712,7 @@ mod tests { enhanced_keys_supported: false, placeholder_text: "Ask Codex to do anything".to_string(), disable_paste_burst: false, + animations_enabled: true, }); // Activate spinner (status view replaces composer) with no live ring. @@ -724,6 +742,7 @@ mod tests { enhanced_keys_supported: false, placeholder_text: "Ask Codex to do anything".to_string(), disable_paste_burst: false, + animations_enabled: true, }); pane.set_task_running(true); @@ -750,6 +769,7 @@ mod tests { enhanced_keys_supported: false, placeholder_text: "Ask Codex to do anything".to_string(), disable_paste_burst: false, + animations_enabled: true, }); pane.set_task_running(true); diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index a5728ab14f..501515aec0 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -962,6 +962,7 @@ impl ChatWidget { parsed, source, None, + self.config.animations, ))); } @@ -1071,6 +1072,7 @@ impl ChatWidget { ev.parsed_cmd, ev.source, interaction_input, + self.config.animations, ))); } @@ -1083,6 +1085,7 @@ impl ChatWidget { self.active_cell = Some(Box::new(history_cell::new_active_mcp_tool_call( ev.call_id, ev.invocation, + self.config.animations, ))); self.request_redraw(); } @@ -1104,7 +1107,11 @@ impl ChatWidget { Some(cell) if cell.call_id() == call_id => cell.complete(duration, result), _ => { self.flush_active_cell(); - let mut cell = history_cell::new_active_mcp_tool_call(call_id, invocation); + let mut cell = history_cell::new_active_mcp_tool_call( + call_id, + invocation, + self.config.animations, + ); let extra_cell = cell.complete(duration, result); self.active_cell = Some(Box::new(cell)); extra_cell @@ -1146,6 +1153,7 @@ impl ChatWidget { enhanced_keys_supported, placeholder_text: placeholder, disable_paste_burst: config.disable_paste_burst, + animations_enabled: config.animations, }), active_cell: None, config: config.clone(), @@ -1220,6 +1228,7 @@ impl ChatWidget { enhanced_keys_supported, placeholder_text: placeholder, disable_paste_burst: config.disable_paste_burst, + animations_enabled: config.animations, }), active_cell: None, config: config.clone(), diff --git a/codex-rs/tui/src/chatwidget/tests.rs b/codex-rs/tui/src/chatwidget/tests.rs index f07d7a51d2..6d3426cbea 100644 --- a/codex-rs/tui/src/chatwidget/tests.rs +++ b/codex-rs/tui/src/chatwidget/tests.rs @@ -338,6 +338,7 @@ fn make_chatwidget_manual() -> ( enhanced_keys_supported: false, placeholder_text: "Ask Codex to do anything".to_string(), disable_paste_burst: false, + animations_enabled: cfg.animations, }); let auth_manager = AuthManager::from_auth_for_testing(CodexAuth::from_api_key("test")); let widget = ChatWidget { diff --git a/codex-rs/tui/src/exec_cell/model.rs b/codex-rs/tui/src/exec_cell/model.rs index 3fddee1562..76316968c6 100644 --- a/codex-rs/tui/src/exec_cell/model.rs +++ b/codex-rs/tui/src/exec_cell/model.rs @@ -28,11 +28,15 @@ pub(crate) struct ExecCall { #[derive(Debug)] pub(crate) struct ExecCell { pub(crate) calls: Vec, + animations_enabled: bool, } impl ExecCell { - pub(crate) fn new(call: ExecCall) -> Self { - Self { calls: vec![call] } + pub(crate) fn new(call: ExecCall, animations_enabled: bool) -> Self { + Self { + calls: vec![call], + animations_enabled, + } } pub(crate) fn with_added_call( @@ -56,6 +60,7 @@ impl ExecCell { if self.is_exploring_cell() && Self::is_exploring_call(&call) { Some(Self { calls: [self.calls.clone(), vec![call]].concat(), + animations_enabled: self.animations_enabled, }) } else { None @@ -112,6 +117,10 @@ impl ExecCell { .and_then(|c| c.start_time) } + pub(crate) fn animations_enabled(&self) -> bool { + self.animations_enabled + } + pub(crate) fn iter_calls(&self) -> impl Iterator { self.calls.iter() } diff --git a/codex-rs/tui/src/exec_cell/render.rs b/codex-rs/tui/src/exec_cell/render.rs index 352a61476e..3e434138d4 100644 --- a/codex-rs/tui/src/exec_cell/render.rs +++ b/codex-rs/tui/src/exec_cell/render.rs @@ -40,17 +40,21 @@ pub(crate) fn new_active_exec_command( parsed: Vec, source: ExecCommandSource, interaction_input: Option, + animations_enabled: bool, ) -> ExecCell { - ExecCell::new(ExecCall { - call_id, - command, - parsed, - output: None, - source, - start_time: Some(Instant::now()), - duration: None, - interaction_input, - }) + ExecCell::new( + ExecCall { + call_id, + command, + parsed, + output: None, + source, + start_time: Some(Instant::now()), + duration: None, + interaction_input, + }, + animations_enabled, + ) } fn format_unified_exec_interaction(command: &[String], input: Option<&str>) -> String { @@ -168,7 +172,10 @@ pub(crate) fn output_lines( } } -pub(crate) fn spinner(start_time: Option) -> Span<'static> { +pub(crate) fn spinner(start_time: Option, animations_enabled: bool) -> Span<'static> { + if !animations_enabled { + return "•".dim(); + } let elapsed = start_time.map(|st| st.elapsed()).unwrap_or_default(); if supports_color::on_cached(supports_color::Stream::Stdout) .map(|level| level.has_16m) @@ -239,7 +246,7 @@ impl ExecCell { let mut out: Vec> = Vec::new(); out.push(Line::from(vec![ if self.is_active() { - spinner(self.active_start_time()) + spinner(self.active_start_time(), self.animations_enabled()) } else { "•".dim() }, @@ -347,7 +354,7 @@ impl ExecCell { let bullet = match success { Some(true) => "•".green().bold(), Some(false) => "•".red().bold(), - None => spinner(call.start_time), + None => spinner(call.start_time, self.animations_enabled()), }; let is_interaction = call.is_unified_exec_interaction(); let title = if is_interaction { diff --git a/codex-rs/tui/src/history_cell.rs b/codex-rs/tui/src/history_cell.rs index 394dc33919..02ab0d243b 100644 --- a/codex-rs/tui/src/history_cell.rs +++ b/codex-rs/tui/src/history_cell.rs @@ -806,16 +806,22 @@ pub(crate) struct McpToolCallCell { start_time: Instant, duration: Option, result: Option>, + animations_enabled: bool, } impl McpToolCallCell { - pub(crate) fn new(call_id: String, invocation: McpInvocation) -> Self { + pub(crate) fn new( + call_id: String, + invocation: McpInvocation, + animations_enabled: bool, + ) -> Self { Self { call_id, invocation, start_time: Instant::now(), duration: None, result: None, + animations_enabled, } } @@ -877,7 +883,7 @@ impl HistoryCell for McpToolCallCell { let bullet = match status { Some(true) => "•".green().bold(), Some(false) => "•".red().bold(), - None => spinner(Some(self.start_time)), + None => spinner(Some(self.start_time), self.animations_enabled), }; let header_text = if status.is_some() { "Called" @@ -965,8 +971,9 @@ impl HistoryCell for McpToolCallCell { pub(crate) fn new_active_mcp_tool_call( call_id: String, invocation: McpInvocation, + animations_enabled: bool, ) -> McpToolCallCell { - McpToolCallCell::new(call_id, invocation) + McpToolCallCell::new(call_id, invocation, animations_enabled) } pub(crate) fn new_web_search_call(query: String) -> PlainHistoryCell { @@ -1631,7 +1638,7 @@ mod tests { })), }; - let cell = new_active_mcp_tool_call("call-1".into(), invocation); + let cell = new_active_mcp_tool_call("call-1".into(), invocation, true); let rendered = render_lines(&cell.display_lines(80)).join("\n"); insta::assert_snapshot!(rendered); @@ -1658,7 +1665,7 @@ mod tests { structured_content: None, }; - let mut cell = new_active_mcp_tool_call("call-2".into(), invocation); + let mut cell = new_active_mcp_tool_call("call-2".into(), invocation, true); assert!( cell.complete(Duration::from_millis(1420), Ok(result)) .is_none() @@ -1680,7 +1687,7 @@ mod tests { })), }; - let mut cell = new_active_mcp_tool_call("call-3".into(), invocation); + let mut cell = new_active_mcp_tool_call("call-3".into(), invocation, true); assert!( cell.complete(Duration::from_secs(2), Err("network timeout".into())) .is_none() @@ -1724,7 +1731,7 @@ mod tests { structured_content: None, }; - let mut cell = new_active_mcp_tool_call("call-4".into(), invocation); + let mut cell = new_active_mcp_tool_call("call-4".into(), invocation, true); assert!( cell.complete(Duration::from_millis(640), Ok(result)) .is_none() @@ -1756,7 +1763,7 @@ mod tests { structured_content: None, }; - let mut cell = new_active_mcp_tool_call("call-5".into(), invocation); + let mut cell = new_active_mcp_tool_call("call-5".into(), invocation, true); assert!( cell.complete(Duration::from_millis(1280), Ok(result)) .is_none() @@ -1795,7 +1802,7 @@ mod tests { structured_content: None, }; - let mut cell = new_active_mcp_tool_call("call-6".into(), invocation); + let mut cell = new_active_mcp_tool_call("call-6".into(), invocation, true); assert!( cell.complete(Duration::from_millis(320), Ok(result)) .is_none() @@ -1853,32 +1860,35 @@ mod tests { fn coalesces_sequential_reads_within_one_call() { // Build one exec cell with a Search followed by two Reads let call_id = "c1".to_string(); - let mut cell = ExecCell::new(ExecCall { - call_id: call_id.clone(), - command: vec!["bash".into(), "-lc".into(), "echo".into()], - parsed: vec![ - ParsedCommand::Search { - query: Some("shimmer_spans".into()), - path: None, - cmd: "rg shimmer_spans".into(), - }, - ParsedCommand::Read { - name: "shimmer.rs".into(), - cmd: "cat shimmer.rs".into(), - path: "shimmer.rs".into(), - }, - ParsedCommand::Read { - name: "status_indicator_widget.rs".into(), - cmd: "cat status_indicator_widget.rs".into(), - path: "status_indicator_widget.rs".into(), - }, - ], - output: None, - source: ExecCommandSource::Agent, - start_time: Some(Instant::now()), - duration: None, - interaction_input: None, - }); + let mut cell = ExecCell::new( + ExecCall { + call_id: call_id.clone(), + command: vec!["bash".into(), "-lc".into(), "echo".into()], + parsed: vec![ + ParsedCommand::Search { + query: Some("shimmer_spans".into()), + path: None, + cmd: "rg shimmer_spans".into(), + }, + ParsedCommand::Read { + name: "shimmer.rs".into(), + cmd: "cat shimmer.rs".into(), + path: "shimmer.rs".into(), + }, + ParsedCommand::Read { + name: "status_indicator_widget.rs".into(), + cmd: "cat status_indicator_widget.rs".into(), + path: "status_indicator_widget.rs".into(), + }, + ], + output: None, + source: ExecCommandSource::Agent, + start_time: Some(Instant::now()), + duration: None, + interaction_input: None, + }, + true, + ); // Mark call complete so markers are ✓ cell.complete_call(&call_id, CommandOutput::default(), Duration::from_millis(1)); @@ -1889,20 +1899,23 @@ mod tests { #[test] fn coalesces_reads_across_multiple_calls() { - let mut cell = ExecCell::new(ExecCall { - call_id: "c1".to_string(), - command: vec!["bash".into(), "-lc".into(), "echo".into()], - parsed: vec![ParsedCommand::Search { - query: Some("shimmer_spans".into()), - path: None, - cmd: "rg shimmer_spans".into(), - }], - output: None, - source: ExecCommandSource::Agent, - start_time: Some(Instant::now()), - duration: None, - interaction_input: None, - }); + let mut cell = ExecCell::new( + ExecCall { + call_id: "c1".to_string(), + command: vec!["bash".into(), "-lc".into(), "echo".into()], + parsed: vec![ParsedCommand::Search { + query: Some("shimmer_spans".into()), + path: None, + cmd: "rg shimmer_spans".into(), + }], + output: None, + source: ExecCommandSource::Agent, + start_time: Some(Instant::now()), + duration: None, + interaction_input: None, + }, + true, + ); // Call 1: Search only cell.complete_call("c1", CommandOutput::default(), Duration::from_millis(1)); // Call 2: Read A @@ -1943,32 +1956,35 @@ mod tests { #[test] fn coalesced_reads_dedupe_names() { - let mut cell = ExecCell::new(ExecCall { - call_id: "c1".to_string(), - command: vec!["bash".into(), "-lc".into(), "echo".into()], - parsed: vec![ - ParsedCommand::Read { - name: "auth.rs".into(), - cmd: "cat auth.rs".into(), - path: "auth.rs".into(), - }, - ParsedCommand::Read { - name: "auth.rs".into(), - cmd: "cat auth.rs".into(), - path: "auth.rs".into(), - }, - ParsedCommand::Read { - name: "shimmer.rs".into(), - cmd: "cat shimmer.rs".into(), - path: "shimmer.rs".into(), - }, - ], - output: None, - source: ExecCommandSource::Agent, - start_time: Some(Instant::now()), - duration: None, - interaction_input: None, - }); + let mut cell = ExecCell::new( + ExecCall { + call_id: "c1".to_string(), + command: vec!["bash".into(), "-lc".into(), "echo".into()], + parsed: vec![ + ParsedCommand::Read { + name: "auth.rs".into(), + cmd: "cat auth.rs".into(), + path: "auth.rs".into(), + }, + ParsedCommand::Read { + name: "auth.rs".into(), + cmd: "cat auth.rs".into(), + path: "auth.rs".into(), + }, + ParsedCommand::Read { + name: "shimmer.rs".into(), + cmd: "cat shimmer.rs".into(), + path: "shimmer.rs".into(), + }, + ], + output: None, + source: ExecCommandSource::Agent, + start_time: Some(Instant::now()), + duration: None, + interaction_input: None, + }, + true, + ); cell.complete_call("c1", CommandOutput::default(), Duration::from_millis(1)); let lines = cell.display_lines(80); let rendered = render_lines(&lines).join("\n"); @@ -1980,16 +1996,19 @@ mod tests { // Create a completed exec cell with a multiline command let cmd = "set -o pipefail\ncargo test --all-features --quiet".to_string(); let call_id = "c1".to_string(); - let mut cell = ExecCell::new(ExecCall { - call_id: call_id.clone(), - command: vec!["bash".into(), "-lc".into(), cmd], - parsed: Vec::new(), - output: None, - source: ExecCommandSource::Agent, - start_time: Some(Instant::now()), - duration: None, - interaction_input: None, - }); + let mut cell = ExecCell::new( + ExecCall { + call_id: call_id.clone(), + command: vec!["bash".into(), "-lc".into(), cmd], + parsed: Vec::new(), + output: None, + source: ExecCommandSource::Agent, + start_time: Some(Instant::now()), + duration: None, + interaction_input: None, + }, + true, + ); // Mark call complete so it renders as "Ran" cell.complete_call(&call_id, CommandOutput::default(), Duration::from_millis(1)); @@ -2003,16 +2022,19 @@ mod tests { #[test] fn single_line_command_compact_when_fits() { let call_id = "c1".to_string(); - let mut cell = ExecCell::new(ExecCall { - call_id: call_id.clone(), - command: vec!["echo".into(), "ok".into()], - parsed: Vec::new(), - output: None, - source: ExecCommandSource::Agent, - start_time: Some(Instant::now()), - duration: None, - interaction_input: None, - }); + let mut cell = ExecCell::new( + ExecCall { + call_id: call_id.clone(), + command: vec!["echo".into(), "ok".into()], + parsed: Vec::new(), + output: None, + source: ExecCommandSource::Agent, + start_time: Some(Instant::now()), + duration: None, + interaction_input: None, + }, + true, + ); cell.complete_call(&call_id, CommandOutput::default(), Duration::from_millis(1)); // Wide enough that it fits inline let lines = cell.display_lines(80); @@ -2024,16 +2046,19 @@ mod tests { fn single_line_command_wraps_with_four_space_continuation() { let call_id = "c1".to_string(); let long = "a_very_long_token_without_spaces_to_force_wrapping".to_string(); - let mut cell = ExecCell::new(ExecCall { - call_id: call_id.clone(), - command: vec!["bash".into(), "-lc".into(), long], - parsed: Vec::new(), - output: None, - source: ExecCommandSource::Agent, - start_time: Some(Instant::now()), - duration: None, - interaction_input: None, - }); + let mut cell = ExecCell::new( + ExecCall { + call_id: call_id.clone(), + command: vec!["bash".into(), "-lc".into(), long], + parsed: Vec::new(), + output: None, + source: ExecCommandSource::Agent, + start_time: Some(Instant::now()), + duration: None, + interaction_input: None, + }, + true, + ); cell.complete_call(&call_id, CommandOutput::default(), Duration::from_millis(1)); let lines = cell.display_lines(24); let rendered = render_lines(&lines).join("\n"); @@ -2044,16 +2069,19 @@ mod tests { fn multiline_command_without_wrap_uses_branch_then_eight_spaces() { let call_id = "c1".to_string(); let cmd = "echo one\necho two".to_string(); - let mut cell = ExecCell::new(ExecCall { - call_id: call_id.clone(), - command: vec!["bash".into(), "-lc".into(), cmd], - parsed: Vec::new(), - output: None, - source: ExecCommandSource::Agent, - start_time: Some(Instant::now()), - duration: None, - interaction_input: None, - }); + let mut cell = ExecCell::new( + ExecCall { + call_id: call_id.clone(), + command: vec!["bash".into(), "-lc".into(), cmd], + parsed: Vec::new(), + output: None, + source: ExecCommandSource::Agent, + start_time: Some(Instant::now()), + duration: None, + interaction_input: None, + }, + true, + ); cell.complete_call(&call_id, CommandOutput::default(), Duration::from_millis(1)); let lines = cell.display_lines(80); let rendered = render_lines(&lines).join("\n"); @@ -2065,16 +2093,19 @@ mod tests { let call_id = "c1".to_string(); let cmd = "first_token_is_long_enough_to_wrap\nsecond_token_is_also_long_enough_to_wrap" .to_string(); - let mut cell = ExecCell::new(ExecCall { - call_id: call_id.clone(), - command: vec!["bash".into(), "-lc".into(), cmd], - parsed: Vec::new(), - output: None, - source: ExecCommandSource::Agent, - start_time: Some(Instant::now()), - duration: None, - interaction_input: None, - }); + let mut cell = ExecCell::new( + ExecCall { + call_id: call_id.clone(), + command: vec!["bash".into(), "-lc".into(), cmd], + parsed: Vec::new(), + output: None, + source: ExecCommandSource::Agent, + start_time: Some(Instant::now()), + duration: None, + interaction_input: None, + }, + true, + ); cell.complete_call(&call_id, CommandOutput::default(), Duration::from_millis(1)); let lines = cell.display_lines(28); let rendered = render_lines(&lines).join("\n"); @@ -2086,16 +2117,19 @@ mod tests { // Build an exec cell with a non-zero exit and 10 lines on stderr to exercise // the head/tail rendering and gutter prefixes. let call_id = "c_err".to_string(); - let mut cell = ExecCell::new(ExecCall { - call_id: call_id.clone(), - command: vec!["bash".into(), "-lc".into(), "seq 1 10 1>&2 && false".into()], - parsed: Vec::new(), - output: None, - source: ExecCommandSource::Agent, - start_time: Some(Instant::now()), - duration: None, - interaction_input: None, - }); + let mut cell = ExecCell::new( + ExecCall { + call_id: call_id.clone(), + command: vec!["bash".into(), "-lc".into(), "seq 1 10 1>&2 && false".into()], + parsed: Vec::new(), + output: None, + source: ExecCommandSource::Agent, + start_time: Some(Instant::now()), + duration: None, + interaction_input: None, + }, + true, + ); let stderr: String = (1..=10) .map(|n| n.to_string()) .collect::>() @@ -2133,16 +2167,19 @@ mod tests { let call_id = "c_wrap_err".to_string(); let long_cmd = "echo this_is_a_very_long_single_token_that_will_wrap_across_the_available_width"; - let mut cell = ExecCell::new(ExecCall { - call_id: call_id.clone(), - command: vec!["bash".into(), "-lc".into(), long_cmd.to_string()], - parsed: Vec::new(), - output: None, - source: ExecCommandSource::Agent, - start_time: Some(Instant::now()), - duration: None, - interaction_input: None, - }); + let mut cell = ExecCell::new( + ExecCall { + call_id: call_id.clone(), + command: vec!["bash".into(), "-lc".into(), long_cmd.to_string()], + parsed: Vec::new(), + output: None, + source: ExecCommandSource::Agent, + start_time: Some(Instant::now()), + duration: None, + interaction_input: None, + }, + true, + ); let stderr = "error: first line on stderr\nerror: second line on stderr".to_string(); cell.complete_call( diff --git a/codex-rs/tui/src/onboarding/auth.rs b/codex-rs/tui/src/onboarding/auth.rs index 06e1c629e3..04096ce0ec 100644 --- a/codex-rs/tui/src/onboarding/auth.rs +++ b/codex-rs/tui/src/onboarding/auth.rs @@ -155,6 +155,7 @@ pub(crate) struct AuthModeWidget { pub auth_manager: Arc, pub forced_chatgpt_workspace_id: Option, pub forced_login_method: Option, + pub animations_enabled: bool, } impl AuthModeWidget { @@ -260,10 +261,14 @@ impl AuthModeWidget { fn render_continue_in_browser(&self, area: Rect, buf: &mut Buffer) { let mut spans = vec![" ".into()]; - // Schedule a follow-up frame to keep the shimmer animation going. - self.request_frame - .schedule_frame_in(std::time::Duration::from_millis(100)); - spans.extend(shimmer_spans("Finish signing in via your browser")); + if self.animations_enabled { + // Schedule a follow-up frame to keep the shimmer animation going. + self.request_frame + .schedule_frame_in(std::time::Duration::from_millis(100)); + spans.extend(shimmer_spans("Finish signing in via your browser")); + } else { + spans.push("Finish signing in via your browser".into()); + } let mut lines = vec![spans.into(), "".into()]; let sign_in_state = self.sign_in_state.read().unwrap(); @@ -670,6 +675,7 @@ mod tests { ), forced_chatgpt_workspace_id: None, forced_login_method: Some(ForcedLoginMethod::Chatgpt), + animations_enabled: true, }; (widget, codex_home) } diff --git a/codex-rs/tui/src/onboarding/onboarding_screen.rs b/codex-rs/tui/src/onboarding/onboarding_screen.rs index b085f28888..47c7811a3b 100644 --- a/codex-rs/tui/src/onboarding/onboarding_screen.rs +++ b/codex-rs/tui/src/onboarding/onboarding_screen.rs @@ -88,6 +88,7 @@ impl OnboardingScreen { steps.push(Step::Welcome(WelcomeWidget::new( !matches!(login_status, LoginStatus::NotAuthenticated), tui.frame_requester(), + config.animations, ))); if show_login_screen { let highlighted_mode = match forced_login_method { @@ -105,6 +106,7 @@ impl OnboardingScreen { auth_manager, forced_chatgpt_workspace_id, forced_login_method, + animations_enabled: config.animations, })) } let is_git_repo = get_git_repo_root(&cwd).is_some(); diff --git a/codex-rs/tui/src/onboarding/welcome.rs b/codex-rs/tui/src/onboarding/welcome.rs index 645c86ba43..8ff5e81982 100644 --- a/codex-rs/tui/src/onboarding/welcome.rs +++ b/codex-rs/tui/src/onboarding/welcome.rs @@ -25,10 +25,14 @@ const MIN_ANIMATION_WIDTH: u16 = 60; pub(crate) struct WelcomeWidget { pub is_logged_in: bool, animation: AsciiAnimation, + animations_enabled: bool, } impl KeyboardHandler for WelcomeWidget { fn handle_key_event(&mut self, key_event: KeyEvent) { + if !self.animations_enabled { + return; + } if key_event.kind == KeyEventKind::Press && key_event.code == KeyCode::Char('.') && key_event.modifiers.contains(KeyModifiers::CONTROL) @@ -40,10 +44,15 @@ impl KeyboardHandler for WelcomeWidget { } impl WelcomeWidget { - pub(crate) fn new(is_logged_in: bool, request_frame: FrameRequester) -> Self { + pub(crate) fn new( + is_logged_in: bool, + request_frame: FrameRequester, + animations_enabled: bool, + ) -> Self { Self { is_logged_in, animation: AsciiAnimation::new(request_frame), + animations_enabled, } } } @@ -51,17 +60,17 @@ impl WelcomeWidget { impl WidgetRef for &WelcomeWidget { fn render_ref(&self, area: Rect, buf: &mut Buffer) { Clear.render(area, buf); - self.animation.schedule_next_frame(); + if self.animations_enabled { + self.animation.schedule_next_frame(); + } // Skip the animation entirely when the viewport is too small so we don't clip frames. let show_animation = area.height >= MIN_ANIMATION_HEIGHT && area.width >= MIN_ANIMATION_WIDTH; let mut lines: Vec = Vec::new(); - if show_animation { + if show_animation && self.animations_enabled { let frame = self.animation.current_frame(); - // let frame_line_count = frame.lines().count(); - // lines.reserve(frame_line_count + 2); lines.extend(frame.lines().map(Into::into)); lines.push("".into()); } @@ -99,7 +108,7 @@ mod tests { #[test] fn welcome_renders_animation_on_first_draw() { - let widget = WelcomeWidget::new(false, FrameRequester::test_dummy()); + let widget = WelcomeWidget::new(false, FrameRequester::test_dummy(), true); let area = Rect::new(0, 0, MIN_ANIMATION_WIDTH, MIN_ANIMATION_HEIGHT); let mut buf = Buffer::empty(area); (&widget).render(area, &mut buf); @@ -129,6 +138,7 @@ mod tests { let mut widget = WelcomeWidget { is_logged_in: false, animation: AsciiAnimation::with_variants(FrameRequester::test_dummy(), &VARIANTS, 0), + animations_enabled: true, }; let before = widget.animation.current_frame(); diff --git a/codex-rs/tui/src/pager_overlay.rs b/codex-rs/tui/src/pager_overlay.rs index 6393a754e5..3b47e9a70e 100644 --- a/codex-rs/tui/src/pager_overlay.rs +++ b/codex-rs/tui/src/pager_overlay.rs @@ -722,6 +722,7 @@ mod tests { vec![ParsedCommand::Unknown { cmd: "ls".into() }], ExecCommandSource::Agent, None, + true, ); exec_cell.complete_call( "exec-1", diff --git a/codex-rs/tui/src/status_indicator_widget.rs b/codex-rs/tui/src/status_indicator_widget.rs index 54979e6d68..642b9ca2b7 100644 --- a/codex-rs/tui/src/status_indicator_widget.rs +++ b/codex-rs/tui/src/status_indicator_widget.rs @@ -30,6 +30,7 @@ pub(crate) struct StatusIndicatorWidget { is_paused: bool, app_event_tx: AppEventSender, frame_requester: FrameRequester, + animations_enabled: bool, } // Format elapsed seconds into a compact human-friendly form used by the status line. @@ -50,7 +51,11 @@ pub fn fmt_elapsed_compact(elapsed_secs: u64) -> String { } impl StatusIndicatorWidget { - pub(crate) fn new(app_event_tx: AppEventSender, frame_requester: FrameRequester) -> Self { + pub(crate) fn new( + app_event_tx: AppEventSender, + frame_requester: FrameRequester, + animations_enabled: bool, + ) -> Self { Self { header: String::from("Working"), show_interrupt_hint: true, @@ -60,6 +65,7 @@ impl StatusIndicatorWidget { app_event_tx, frame_requester, + animations_enabled, } } @@ -146,9 +152,13 @@ impl Renderable for StatusIndicatorWidget { let pretty_elapsed = fmt_elapsed_compact(elapsed_duration.as_secs()); let mut spans = Vec::with_capacity(5); - spans.push(spinner(Some(self.last_resume_at))); + spans.push(spinner(Some(self.last_resume_at), self.animations_enabled)); spans.push(" ".into()); - spans.extend(shimmer_spans(&self.header)); + if self.animations_enabled { + spans.extend(shimmer_spans(&self.header)); + } else if !self.header.is_empty() { + spans.push(self.header.clone().into()); + } spans.push(" ".into()); if self.show_interrupt_hint { spans.extend(vec![ @@ -195,7 +205,7 @@ mod tests { fn renders_with_working_header() { let (tx_raw, _rx) = unbounded_channel::(); let tx = AppEventSender::new(tx_raw); - let w = StatusIndicatorWidget::new(tx, crate::tui::FrameRequester::test_dummy()); + let w = StatusIndicatorWidget::new(tx, crate::tui::FrameRequester::test_dummy(), true); // Render into a fixed-size test terminal and snapshot the backend. let mut terminal = Terminal::new(TestBackend::new(80, 2)).expect("terminal"); @@ -209,7 +219,7 @@ mod tests { fn renders_truncated() { let (tx_raw, _rx) = unbounded_channel::(); let tx = AppEventSender::new(tx_raw); - let w = StatusIndicatorWidget::new(tx, crate::tui::FrameRequester::test_dummy()); + let w = StatusIndicatorWidget::new(tx, crate::tui::FrameRequester::test_dummy(), true); // Render into a fixed-size test terminal and snapshot the backend. let mut terminal = Terminal::new(TestBackend::new(20, 2)).expect("terminal"); @@ -223,7 +233,8 @@ mod tests { fn timer_pauses_when_requested() { let (tx_raw, _rx) = unbounded_channel::(); let tx = AppEventSender::new(tx_raw); - let mut widget = StatusIndicatorWidget::new(tx, crate::tui::FrameRequester::test_dummy()); + let mut widget = + StatusIndicatorWidget::new(tx, crate::tui::FrameRequester::test_dummy(), true); let baseline = Instant::now(); widget.last_resume_at = baseline; diff --git a/docs/config.md b/docs/config.md index f47bf34643..cb756c6bf6 100644 --- a/docs/config.md +++ b/docs/config.md @@ -893,6 +893,10 @@ notifications = true # You can optionally filter to specific notification types. # Available types are "agent-turn-complete" and "approval-requested". notifications = [ "agent-turn-complete", "approval-requested" ] + +# Disable terminal animations (welcome screen, status shimmer, spinner). +# Defaults to true. +animations = false ``` > [!NOTE] @@ -950,6 +954,7 @@ Valid values: | `sandbox_workspace_write.exclude_tmpdir_env_var` | boolean | Exclude `$TMPDIR` from writable roots (default: false). | | `sandbox_workspace_write.exclude_slash_tmp` | boolean | Exclude `/tmp` from writable roots (default: false). | | `notify` | array | External program for notifications. | +| `tui.animations` | boolean | Enable terminal animations (welcome screen, shimmer, spinner). Defaults to true; set to `false` to disable visual motion. | | `instructions` | string | Currently ignored; use `experimental_instructions_file` or `AGENTS.md`. | | `features.` | boolean | See [feature flags](#feature-flags) for details | | `mcp_servers..command` | string | MCP server launcher command (stdio servers only). | diff --git a/docs/example-config.md b/docs/example-config.md index 335a826e26..f9d7955434 100644 --- a/docs/example-config.md +++ b/docs/example-config.md @@ -140,13 +140,16 @@ file_opener = "vscode" # Examples: false | ["agent-turn-complete", "approval-requested"] notifications = false -# Suppress internal reasoning events from output (default: false) +# Enables welcome/status/spinner animations. Default: true +animations = true + +# Suppress internal reasoning events from output. Default: false hide_agent_reasoning = false -# Show raw reasoning content when available (default: false) +# Show raw reasoning content when available. Default: false show_raw_agent_reasoning = false -# Disable burst-paste detection in the TUI (default: false) +# Disable burst-paste detection in the TUI. Default: false disable_paste_burst = false # Track Windows onboarding acknowledgement (Windows only). Default: false From 30ca89424cc7170fa595cea022fa0e4d9a73b7a0 Mon Sep 17 00:00:00 2001 From: pakrym-oai Date: Thu, 20 Nov 2025 10:58:46 -0800 Subject: [PATCH 20/42] Always fallback to real shell (#6953) Either cmd.exe or `/bin/sh`. --- codex-rs/core/src/bash.rs | 2 +- codex-rs/core/src/codex.rs | 17 +- codex-rs/core/src/environment_context.rs | 58 +++-- codex-rs/core/src/shell.rs | 232 ++++++++++++++------ codex-rs/core/src/tools/handlers/shell.rs | 19 +- codex-rs/core/tests/suite/prompt_caching.rs | 37 ++-- 6 files changed, 234 insertions(+), 131 deletions(-) diff --git a/codex-rs/core/src/bash.rs b/codex-rs/core/src/bash.rs index 0ffb8e785e..cb8248ec1f 100644 --- a/codex-rs/core/src/bash.rs +++ b/codex-rs/core/src/bash.rs @@ -100,7 +100,7 @@ pub fn extract_bash_command(command: &[String]) -> Option<(&str, &str)> { if !matches!(flag.as_str(), "-lc" | "-c") || !matches!( detect_shell_type(&PathBuf::from(shell)), - Some(ShellType::Zsh) | Some(ShellType::Bash) + Some(ShellType::Zsh) | Some(ShellType::Bash) | Some(ShellType::Sh) ) { return None; diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 4d9285dd24..55dfe17047 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -493,7 +493,7 @@ impl Session { // - load history metadata let rollout_fut = RolloutRecorder::new(&config, rollout_params); - let default_shell_fut = shell::default_user_shell(); + let default_shell = shell::default_user_shell(); let history_meta_fut = crate::message_history::history_metadata(&config); let auth_statuses_fut = compute_auth_statuses( config.mcp_servers.iter(), @@ -501,12 +501,8 @@ impl Session { ); // Join all independent futures. - let (rollout_recorder, default_shell, (history_log_id, history_entry_count), auth_statuses) = tokio::join!( - rollout_fut, - default_shell_fut, - history_meta_fut, - auth_statuses_fut - ); + let (rollout_recorder, (history_log_id, history_entry_count), auth_statuses) = + tokio::join!(rollout_fut, history_meta_fut, auth_statuses_fut); let rollout_recorder = rollout_recorder.map_err(|e| { error!("failed to initialize rollout recorder: {e:#}"); @@ -1057,7 +1053,7 @@ impl Session { Some(turn_context.cwd.clone()), Some(turn_context.approval_policy), Some(turn_context.sandbox_policy.clone()), - Some(self.user_shell().clone()), + self.user_shell().clone(), ))); items } @@ -2390,6 +2386,7 @@ mod tests { use crate::config::ConfigOverrides; use crate::config::ConfigToml; use crate::exec::ExecToolCallOutput; + use crate::shell::default_user_shell; use crate::tools::format_exec_output_str; use crate::protocol::CompactedItem; @@ -2629,7 +2626,7 @@ mod tests { unified_exec_manager: UnifiedExecSessionManager::default(), notifier: UserNotifier::new(None), rollout: Mutex::new(None), - user_shell: shell::Shell::Unknown, + user_shell: default_user_shell(), show_raw_agent_reasoning: config.show_raw_agent_reasoning, auth_manager: Arc::clone(&auth_manager), otel_event_manager: otel_event_manager.clone(), @@ -2707,7 +2704,7 @@ mod tests { unified_exec_manager: UnifiedExecSessionManager::default(), notifier: UserNotifier::new(None), rollout: Mutex::new(None), - user_shell: shell::Shell::Unknown, + user_shell: default_user_shell(), show_raw_agent_reasoning: config.show_raw_agent_reasoning, auth_manager: Arc::clone(&auth_manager), otel_event_manager: otel_event_manager.clone(), diff --git a/codex-rs/core/src/environment_context.rs b/codex-rs/core/src/environment_context.rs index 6db2bb8f52..56e7f6cadb 100644 --- a/codex-rs/core/src/environment_context.rs +++ b/codex-rs/core/src/environment_context.rs @@ -6,6 +6,7 @@ use crate::codex::TurnContext; use crate::protocol::AskForApproval; use crate::protocol::SandboxPolicy; use crate::shell::Shell; +use crate::shell::default_user_shell; use codex_protocol::config_types::SandboxMode; use codex_protocol::models::ContentItem; use codex_protocol::models::ResponseItem; @@ -28,7 +29,7 @@ pub(crate) struct EnvironmentContext { pub sandbox_mode: Option, pub network_access: Option, pub writable_roots: Option>, - pub shell: Option, + pub shell: Shell, } impl EnvironmentContext { @@ -36,7 +37,7 @@ impl EnvironmentContext { cwd: Option, approval_policy: Option, sandbox_policy: Option, - shell: Option, + shell: Shell, ) -> Self { Self { cwd, @@ -110,7 +111,7 @@ impl EnvironmentContext { } else { None }; - EnvironmentContext::new(cwd, approval_policy, sandbox_policy, None) + EnvironmentContext::new(cwd, approval_policy, sandbox_policy, default_user_shell()) } } @@ -121,7 +122,7 @@ impl From<&TurnContext> for EnvironmentContext { Some(turn_context.approval_policy), Some(turn_context.sandbox_policy.clone()), // Shell is not configurable from turn to turn - None, + default_user_shell(), ) } } @@ -169,11 +170,9 @@ impl EnvironmentContext { } lines.push(" ".to_string()); } - if let Some(shell) = self.shell - && let Some(shell_name) = shell.name() - { - lines.push(format!(" {shell_name}")); - } + + let shell_name = self.shell.name(); + lines.push(format!(" {shell_name}")); lines.push(ENVIRONMENT_CONTEXT_CLOSE_TAG.to_string()); lines.join("\n") } @@ -193,12 +192,18 @@ impl From for ResponseItem { #[cfg(test)] mod tests { - use crate::shell::BashShell; - use crate::shell::ZshShell; + use crate::shell::ShellType; use super::*; use pretty_assertions::assert_eq; + fn fake_shell() -> Shell { + Shell { + shell_type: ShellType::Bash, + shell_path: PathBuf::from("/bin/bash"), + } + } + fn workspace_write_policy(writable_roots: Vec<&str>, network_access: bool) -> SandboxPolicy { SandboxPolicy::WorkspaceWrite { writable_roots: writable_roots.into_iter().map(PathBuf::from).collect(), @@ -214,7 +219,7 @@ mod tests { Some(PathBuf::from("/repo")), Some(AskForApproval::OnRequest), Some(workspace_write_policy(vec!["/repo", "/tmp"], false)), - None, + fake_shell(), ); let expected = r#" @@ -226,6 +231,7 @@ mod tests { /repo /tmp + bash "#; assert_eq!(context.serialize_to_xml(), expected); @@ -237,13 +243,14 @@ mod tests { None, Some(AskForApproval::Never), Some(SandboxPolicy::ReadOnly), - None, + fake_shell(), ); let expected = r#" never read-only restricted + bash "#; assert_eq!(context.serialize_to_xml(), expected); @@ -255,13 +262,14 @@ mod tests { None, Some(AskForApproval::OnFailure), Some(SandboxPolicy::DangerFullAccess), - None, + fake_shell(), ); let expected = r#" on-failure danger-full-access enabled + bash "#; assert_eq!(context.serialize_to_xml(), expected); @@ -274,13 +282,13 @@ mod tests { Some(PathBuf::from("/repo")), Some(AskForApproval::OnRequest), Some(workspace_write_policy(vec!["/repo"], false)), - None, + fake_shell(), ); let context2 = EnvironmentContext::new( Some(PathBuf::from("/repo")), Some(AskForApproval::Never), Some(workspace_write_policy(vec!["/repo"], true)), - None, + fake_shell(), ); assert!(!context1.equals_except_shell(&context2)); } @@ -291,13 +299,13 @@ mod tests { Some(PathBuf::from("/repo")), Some(AskForApproval::OnRequest), Some(SandboxPolicy::new_read_only_policy()), - None, + fake_shell(), ); let context2 = EnvironmentContext::new( Some(PathBuf::from("/repo")), Some(AskForApproval::OnRequest), Some(SandboxPolicy::new_workspace_write_policy()), - None, + fake_shell(), ); assert!(!context1.equals_except_shell(&context2)); @@ -309,13 +317,13 @@ mod tests { Some(PathBuf::from("/repo")), Some(AskForApproval::OnRequest), Some(workspace_write_policy(vec!["/repo", "/tmp", "/var"], false)), - None, + fake_shell(), ); let context2 = EnvironmentContext::new( Some(PathBuf::from("/repo")), Some(AskForApproval::OnRequest), Some(workspace_write_policy(vec!["/repo", "/tmp"], true)), - None, + fake_shell(), ); assert!(!context1.equals_except_shell(&context2)); @@ -327,17 +335,19 @@ mod tests { Some(PathBuf::from("/repo")), Some(AskForApproval::OnRequest), Some(workspace_write_policy(vec!["/repo"], false)), - Some(Shell::Bash(BashShell { + Shell { + shell_type: ShellType::Bash, shell_path: "/bin/bash".into(), - })), + }, ); let context2 = EnvironmentContext::new( Some(PathBuf::from("/repo")), Some(AskForApproval::OnRequest), Some(workspace_write_policy(vec!["/repo"], false)), - Some(Shell::Zsh(ZshShell { + Shell { + shell_type: ShellType::Zsh, shell_path: "/bin/zsh".into(), - })), + }, ); assert!(context1.equals_except_shell(&context2)); diff --git a/codex-rs/core/src/shell.rs b/codex-rs/core/src/shell.rs index 1fb2b97dd7..a275b2e399 100644 --- a/codex-rs/core/src/shell.rs +++ b/codex-rs/core/src/shell.rs @@ -7,61 +7,41 @@ pub enum ShellType { Zsh, Bash, PowerShell, + Sh, + Cmd, } #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] -pub struct ZshShell { +pub struct Shell { + pub(crate) shell_type: ShellType, pub(crate) shell_path: PathBuf, } -#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] -pub struct BashShell { - pub(crate) shell_path: PathBuf, -} - -#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] -pub struct PowerShellConfig { - pub(crate) shell_path: PathBuf, // Executable name or path, e.g. "pwsh" or "powershell.exe". -} - -#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] -pub enum Shell { - Zsh(ZshShell), - Bash(BashShell), - PowerShell(PowerShellConfig), - Unknown, -} - impl Shell { - pub fn name(&self) -> Option { - match self { - Shell::Zsh(ZshShell { shell_path, .. }) | Shell::Bash(BashShell { shell_path, .. }) => { - std::path::Path::new(shell_path) - .file_name() - .map(|s| s.to_string_lossy().to_string()) - } - Shell::PowerShell(ps) => ps - .shell_path - .file_stem() - .map(|s| s.to_string_lossy().to_string()), - Shell::Unknown => None, + pub fn name(&self) -> &'static str { + match self.shell_type { + ShellType::Zsh => "zsh", + ShellType::Bash => "bash", + ShellType::PowerShell => "powershell", + ShellType::Sh => "sh", + ShellType::Cmd => "cmd", } } /// Takes a string of shell and returns the full list of command args to /// use with `exec()` to run the shell command. pub fn derive_exec_args(&self, command: &str, use_login_shell: bool) -> Vec { - match self { - Shell::Zsh(ZshShell { shell_path, .. }) | Shell::Bash(BashShell { shell_path, .. }) => { + match self.shell_type { + ShellType::Zsh | ShellType::Bash | ShellType::Sh => { let arg = if use_login_shell { "-lc" } else { "-c" }; vec![ - shell_path.to_string_lossy().to_string(), + self.shell_path.to_string_lossy().to_string(), arg.to_string(), command.to_string(), ] } - Shell::PowerShell(ps) => { - let mut args = vec![ps.shell_path.to_string_lossy().to_string()]; + ShellType::PowerShell => { + let mut args = vec![self.shell_path.to_string_lossy().to_string()]; if !use_login_shell { args.push("-NoProfile".to_string()); } @@ -70,7 +50,12 @@ impl Shell { args.push(command.to_string()); args } - Shell::Unknown => shlex::split(command).unwrap_or_else(|| vec![command.to_string()]), + ShellType::Cmd => { + let mut args = vec![self.shell_path.to_string_lossy().to_string()]; + args.push("/c".to_string()); + args.push(command.to_string()); + args + } } } } @@ -143,19 +128,34 @@ fn get_shell_path( None } -fn get_zsh_shell(path: Option<&PathBuf>) -> Option { +fn get_zsh_shell(path: Option<&PathBuf>) -> Option { let shell_path = get_shell_path(ShellType::Zsh, path, "zsh", vec!["/bin/zsh"]); - shell_path.map(|shell_path| ZshShell { shell_path }) + shell_path.map(|shell_path| Shell { + shell_type: ShellType::Zsh, + shell_path, + }) } -fn get_bash_shell(path: Option<&PathBuf>) -> Option { +fn get_bash_shell(path: Option<&PathBuf>) -> Option { let shell_path = get_shell_path(ShellType::Bash, path, "bash", vec!["/bin/bash"]); - shell_path.map(|shell_path| BashShell { shell_path }) + shell_path.map(|shell_path| Shell { + shell_type: ShellType::Bash, + shell_path, + }) } -fn get_powershell_shell(path: Option<&PathBuf>) -> Option { +fn get_sh_shell(path: Option<&PathBuf>) -> Option { + let shell_path = get_shell_path(ShellType::Sh, path, "sh", vec!["/bin/sh"]); + + shell_path.map(|shell_path| Shell { + shell_type: ShellType::Sh, + shell_path, + }) +} + +fn get_powershell_shell(path: Option<&PathBuf>) -> Option { let shell_path = get_shell_path( ShellType::PowerShell, path, @@ -164,26 +164,56 @@ fn get_powershell_shell(path: Option<&PathBuf>) -> Option { ) .or_else(|| get_shell_path(ShellType::PowerShell, path, "powershell", vec![])); - shell_path.map(|shell_path| PowerShellConfig { shell_path }) + shell_path.map(|shell_path| Shell { + shell_type: ShellType::PowerShell, + shell_path, + }) +} + +fn get_cmd_shell(path: Option<&PathBuf>) -> Option { + let shell_path = get_shell_path(ShellType::Cmd, path, "cmd", vec![]); + + shell_path.map(|shell_path| Shell { + shell_type: ShellType::Cmd, + shell_path, + }) +} + +fn ultimate_fallback_shell() -> Shell { + if cfg!(windows) { + Shell { + shell_type: ShellType::Cmd, + shell_path: PathBuf::from("cmd.exe"), + } + } else { + Shell { + shell_type: ShellType::Sh, + shell_path: PathBuf::from("/bin/sh"), + } + } } pub fn get_shell_by_model_provided_path(shell_path: &PathBuf) -> Shell { detect_shell_type(shell_path) .and_then(|shell_type| get_shell(shell_type, Some(shell_path))) - .unwrap_or(Shell::Unknown) + .unwrap_or(ultimate_fallback_shell()) } pub fn get_shell(shell_type: ShellType, path: Option<&PathBuf>) -> Option { match shell_type { - ShellType::Zsh => get_zsh_shell(path).map(Shell::Zsh), - ShellType::Bash => get_bash_shell(path).map(Shell::Bash), - ShellType::PowerShell => get_powershell_shell(path).map(Shell::PowerShell), + ShellType::Zsh => get_zsh_shell(path), + ShellType::Bash => get_bash_shell(path), + ShellType::PowerShell => get_powershell_shell(path), + ShellType::Sh => get_sh_shell(path), + ShellType::Cmd => get_cmd_shell(path), } } pub fn detect_shell_type(shell_path: &PathBuf) -> Option { match shell_path.as_os_str().to_str() { Some("zsh") => Some(ShellType::Zsh), + Some("sh") => Some(ShellType::Sh), + Some("cmd") => Some(ShellType::Cmd), Some("bash") => Some(ShellType::Bash), Some("pwsh") => Some(ShellType::PowerShell), Some("powershell") => Some(ShellType::PowerShell), @@ -200,11 +230,15 @@ pub fn detect_shell_type(shell_path: &PathBuf) -> Option { } } -pub async fn default_user_shell() -> Shell { +pub fn default_user_shell() -> Shell { + default_user_shell_from_path(get_user_shell_path()) +} + +fn default_user_shell_from_path(user_shell_path: Option) -> Shell { if cfg!(windows) { - get_shell(ShellType::PowerShell, None).unwrap_or(Shell::Unknown) + get_shell(ShellType::PowerShell, None).unwrap_or(ultimate_fallback_shell()) } else { - let user_default_shell = get_user_shell_path() + let user_default_shell = user_shell_path .and_then(|shell| detect_shell_type(&shell)) .and_then(|shell_type| get_shell(shell_type, None)); @@ -218,7 +252,7 @@ pub async fn default_user_shell() -> Shell { .or_else(|| get_shell(ShellType::Zsh, None)) }; - shell_with_fallback.unwrap_or(Shell::Unknown) + shell_with_fallback.unwrap_or(ultimate_fallback_shell()) } } @@ -274,6 +308,19 @@ mod detect_shell_type_tests { detect_shell_type(&PathBuf::from("/usr/local/bin/pwsh")), Some(ShellType::PowerShell) ); + assert_eq!( + detect_shell_type(&PathBuf::from("/bin/sh")), + Some(ShellType::Sh) + ); + assert_eq!(detect_shell_type(&PathBuf::from("sh")), Some(ShellType::Sh)); + assert_eq!( + detect_shell_type(&PathBuf::from("cmd")), + Some(ShellType::Cmd) + ); + assert_eq!( + detect_shell_type(&PathBuf::from("cmd.exe")), + Some(ShellType::Cmd) + ); } } @@ -289,10 +336,17 @@ mod tests { fn detects_zsh() { let zsh_shell = get_shell(ShellType::Zsh, None).unwrap(); - let ZshShell { shell_path } = match zsh_shell { - Shell::Zsh(zsh_shell) => zsh_shell, - _ => panic!("expected zsh shell"), - }; + let shell_path = zsh_shell.shell_path; + + assert_eq!(shell_path, PathBuf::from("/bin/zsh")); + } + + #[test] + #[cfg(target_os = "macos")] + fn fish_fallback_to_zsh() { + let zsh_shell = default_user_shell_from_path(Some(PathBuf::from("/bin/fish"))); + + let shell_path = zsh_shell.shell_path; assert_eq!(shell_path, PathBuf::from("/bin/zsh")); } @@ -300,10 +354,7 @@ mod tests { #[test] fn detects_bash() { let bash_shell = get_shell(ShellType::Bash, None).unwrap(); - let BashShell { shell_path } = match bash_shell { - Shell::Bash(bash_shell) => bash_shell, - _ => panic!("expected bash shell"), - }; + let shell_path = bash_shell.shell_path; assert!( shell_path == PathBuf::from("/bin/bash") @@ -312,6 +363,50 @@ mod tests { ); } + #[test] + fn detects_sh() { + let sh_shell = get_shell(ShellType::Sh, None).unwrap(); + let shell_path = sh_shell.shell_path; + assert!( + shell_path == PathBuf::from("/bin/sh") || shell_path == PathBuf::from("/usr/bin/sh"), + "shell path: {shell_path:?}", + ); + } + + #[test] + fn can_run_on_shell_test() { + let cmd = "echo \"Works\""; + if cfg!(windows) { + assert!(shell_works( + get_shell(ShellType::PowerShell, None), + "Out-String 'Works'", + true, + )); + assert!(shell_works(get_shell(ShellType::Cmd, None), cmd, true,)); + assert!(shell_works(Some(ultimate_fallback_shell()), cmd, true)); + } else { + assert!(shell_works(Some(ultimate_fallback_shell()), cmd, true)); + assert!(shell_works(get_shell(ShellType::Zsh, None), cmd, false)); + assert!(shell_works(get_shell(ShellType::Bash, None), cmd, true)); + assert!(shell_works(get_shell(ShellType::Sh, None), cmd, true)); + } + } + + fn shell_works(shell: Option, command: &str, required: bool) -> bool { + if let Some(shell) = shell { + let args = shell.derive_exec_args(command, false); + let output = Command::new(args[0].clone()) + .args(&args[1..]) + .output() + .unwrap(); + assert!(output.status.success()); + assert!(String::from_utf8_lossy(&output.stdout).contains("Works")); + true + } else { + !required + } + } + #[tokio::test] async fn test_current_shell_detects_zsh() { let shell = Command::new("sh") @@ -323,10 +418,11 @@ mod tests { let shell_path = String::from_utf8_lossy(&shell.stdout).trim().to_string(); if shell_path.ends_with("/zsh") { assert_eq!( - default_user_shell().await, - Shell::Zsh(ZshShell { + default_user_shell(), + Shell { + shell_type: ShellType::Zsh, shell_path: PathBuf::from(shell_path), - }) + } ); } } @@ -337,11 +433,8 @@ mod tests { return; } - let powershell_shell = default_user_shell().await; - let PowerShellConfig { shell_path } = match powershell_shell { - Shell::PowerShell(powershell_shell) => powershell_shell, - _ => panic!("expected powershell shell"), - }; + let powershell_shell = default_user_shell(); + let shell_path = powershell_shell.shell_path; assert!(shell_path.ends_with("pwsh.exe") || shell_path.ends_with("powershell.exe")); } @@ -353,10 +446,7 @@ mod tests { } let powershell_shell = get_shell(ShellType::PowerShell, None).unwrap(); - let PowerShellConfig { shell_path } = match powershell_shell { - Shell::PowerShell(powershell_shell) => powershell_shell, - _ => panic!("expected powershell shell"), - }; + let shell_path = powershell_shell.shell_path; assert!(shell_path.ends_with("pwsh.exe") || shell_path.ends_with("powershell.exe")); } diff --git a/codex-rs/core/src/tools/handlers/shell.rs b/codex-rs/core/src/tools/handlers/shell.rs index fcd2f5b0c9..3cdf8af57d 100644 --- a/codex-rs/core/src/tools/handlers/shell.rs +++ b/codex-rs/core/src/tools/handlers/shell.rs @@ -338,29 +338,30 @@ mod tests { use std::path::PathBuf; use crate::is_safe_command::is_known_safe_command; - use crate::shell::BashShell; - use crate::shell::PowerShellConfig; use crate::shell::Shell; - use crate::shell::ZshShell; + use crate::shell::ShellType; /// The logic for is_known_safe_command() has heuristics for known shells, /// so we must ensure the commands generated by [ShellCommandHandler] can be /// recognized as safe if the `command` is safe. #[test] fn commands_generated_by_shell_command_handler_can_be_matched_by_is_known_safe_command() { - let bash_shell = Shell::Bash(BashShell { + let bash_shell = Shell { + shell_type: ShellType::Bash, shell_path: PathBuf::from("/bin/bash"), - }); + }; assert_safe(&bash_shell, "ls -la"); - let zsh_shell = Shell::Zsh(ZshShell { + let zsh_shell = Shell { + shell_type: ShellType::Zsh, shell_path: PathBuf::from("/bin/zsh"), - }); + }; assert_safe(&zsh_shell, "ls -la"); - let powershell = Shell::PowerShell(PowerShellConfig { + let powershell = Shell { + shell_type: ShellType::PowerShell, shell_path: PathBuf::from("pwsh.exe"), - }); + }; assert_safe(&powershell, "ls -Name"); } diff --git a/codex-rs/core/tests/suite/prompt_caching.rs b/codex-rs/core/tests/suite/prompt_caching.rs index 55e8f66416..f4455fd022 100644 --- a/codex-rs/core/tests/suite/prompt_caching.rs +++ b/codex-rs/core/tests/suite/prompt_caching.rs @@ -30,18 +30,15 @@ fn text_user_input(text: String) -> serde_json::Value { } fn default_env_context_str(cwd: &str, shell: &Shell) -> String { + let shell_name = shell.name(); format!( r#" - {} + {cwd} on-request read-only restricted -{}"#, - cwd, - match shell.name() { - Some(name) => format!(" {name}\n"), - None => String::new(), - } + {shell_name} +"# ) } @@ -227,7 +224,7 @@ async fn prefixes_context_and_instructions_once_and_consistently_across_requests .await?; wait_for_event(&codex, |ev| matches!(ev, EventMsg::TaskComplete(_))).await; - let shell = default_user_shell().await; + let shell = default_user_shell(); let cwd_str = config.cwd.to_string_lossy(); let expected_env_text = default_env_context_str(&cwd_str, &shell); let expected_ui_text = format!( @@ -345,6 +342,7 @@ async fn overrides_turn_context_but_keeps_cached_prefix_and_key_constant() -> an // After overriding the turn context, the environment context should be emitted again // reflecting the new approval policy and sandbox settings. Omit cwd because it did // not change. + let shell = default_user_shell(); let expected_env_text_2 = format!( r#" never @@ -353,8 +351,10 @@ async fn overrides_turn_context_but_keeps_cached_prefix_and_key_constant() -> an {} + {} "#, - writable.path().to_string_lossy(), + writable.path().display(), + shell.name() ); let expected_env_msg_2 = serde_json::json!({ "type": "message", @@ -522,6 +522,8 @@ async fn per_turn_overrides_keep_cached_prefix_and_key_constant() -> anyhow::Res "role": "user", "content": [ { "type": "input_text", "text": "hello 2" } ] }); + let shell = default_user_shell(); + let expected_env_text_2 = format!( r#" {} @@ -531,9 +533,11 @@ async fn per_turn_overrides_keep_cached_prefix_and_key_constant() -> anyhow::Res {} + {} "#, - new_cwd.path().to_string_lossy(), - writable.path().to_string_lossy(), + new_cwd.path().display(), + writable.path().display(), + shell.name(), ); let expected_env_msg_2 = serde_json::json!({ "type": "message", @@ -610,7 +614,7 @@ async fn send_user_turn_with_no_changes_does_not_send_environment_context() -> a let body1 = req1.single_request().body_json(); let body2 = req2.single_request().body_json(); - let shell = default_user_shell().await; + let shell = default_user_shell(); let default_cwd_lossy = default_cwd.to_string_lossy(); let expected_ui_text = format!( "# AGENTS.md instructions for {default_cwd_lossy}\n\n\nbe consistent and helpful\n" @@ -697,7 +701,7 @@ async fn send_user_turn_with_changes_sends_environment_context() -> anyhow::Resu let body1 = req1.single_request().body_json(); let body2 = req2.single_request().body_json(); - let shell = default_user_shell().await; + let shell = default_user_shell(); let expected_ui_text = format!( "# AGENTS.md instructions for {}\n\n\nbe consistent and helpful\n", default_cwd.to_string_lossy() @@ -717,14 +721,15 @@ async fn send_user_turn_with_changes_sends_environment_context() -> anyhow::Resu ]); assert_eq!(body1["input"], expected_input_1); - let expected_env_msg_2 = text_user_input( + let shell_name = shell.name(); + let expected_env_msg_2 = text_user_input(format!( r#" never danger-full-access enabled + {shell_name} "# - .to_string(), - ); + )); let expected_user_message_2 = text_user_input("hello 2".to_string()); let expected_input_2 = serde_json::Value::Array(vec![ expected_ui_msg, From 397279d46e301194ffd27ca072b8d7a685459135 Mon Sep 17 00:00:00 2001 From: LIHUA <1017343802@qq.com> Date: Fri, 21 Nov 2025 03:04:11 +0800 Subject: [PATCH 21/42] Fix: Improve text encoding for shell output in VSCode preview (#6178) (#6182) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 🐛 Problem Users running commands with non-ASCII characters (like Russian text "пример") in Windows/WSL environments experience garbled text in VSCode's shell preview window, with Unicode replacement characters (�) appearing instead of the actual text. **Issue**: https://github.com/openai/codex/issues/6178 ## 🔧 Root Cause The issue was in `StreamOutput>::from_utf8_lossy()` method in `codex-rs/core/src/exec.rs`, which used `String::from_utf8_lossy()` to convert shell output bytes to strings. This function immediately replaces any invalid UTF-8 byte sequences with replacement characters, without attempting to decode using other common encodings. In Windows/WSL environments, shell output often uses encodings like: - Windows-1252 (common Windows encoding) - Latin-1/ISO-8859-1 (extended ASCII) ## 🛠️ Solution Replaced the simple `String::from_utf8_lossy()` call with intelligent encoding detection via a new `bytes_to_string_smart()` function that tries multiple encoding strategies: 1. **UTF-8** (fast path for valid UTF-8) 2. **Windows-1252** (handles Windows-specific characters in 0x80-0x9F range) 3. **Latin-1** (fallback for extended ASCII) 4. **Lossy UTF-8** (final fallback, same as before) ## 📁 Changes ### New Files - `codex-rs/core/src/text_encoding.rs` - Smart encoding detection module - `codex-rs/core/tests/suite/text_encoding_fix.rs` - Integration tests ### Modified Files - `codex-rs/core/src/lib.rs` - Added text_encoding module - `codex-rs/core/src/exec.rs` - Updated StreamOutput::from_utf8_lossy() - `codex-rs/core/tests/suite/mod.rs` - Registered new test module ## ✅ Testing - **5 unit tests** covering UTF-8, Windows-1252, Latin-1, and fallback scenarios - **2 integration tests** simulating the exact Issue #6178 scenario - **Demonstrates improvement** over the previous `String::from_utf8_lossy()` approach All tests pass: ```bash cargo test -p codex-core text_encoding cargo test -p codex-core test_shell_output_encoding_issue_6178 ``` ## 🎯 Impact - ✅ **Eliminates garbled text** in VSCode shell preview for non-ASCII content - ✅ **Supports Windows/WSL environments** with proper encoding detection - ✅ **Zero performance impact** for UTF-8 text (fast path) - ✅ **Backward compatible** - UTF-8 content works exactly as before - ✅ **Handles edge cases** with robust fallback mechanism ## 🧪 Test Scenarios The fix has been tested with: - Russian text ("пример") - Windows-1252 quotation marks (""test") - Latin-1 accented characters ("café") - Mixed encoding content - Invalid byte sequences (graceful fallback) ## 📋 Checklist - [X] Addresses the reported issue - [X] Includes comprehensive tests - [X] Maintains backward compatibility - [X] Follows project coding conventions - [X] No breaking changes --------- Co-authored-by: Josh McKinney --- codex-rs/Cargo.lock | 13 + codex-rs/Cargo.toml | 2 + codex-rs/core/Cargo.toml | 2 + codex-rs/core/src/exec.rs | 3 +- codex-rs/core/src/lib.rs | 1 + codex-rs/core/src/text_encoding.rs | 461 ++++++++++++++++++ codex-rs/core/tests/suite/mod.rs | 1 + .../core/tests/suite/text_encoding_fix.rs | 77 +++ 8 files changed, 559 insertions(+), 1 deletion(-) create mode 100644 codex-rs/core/src/text_encoding.rs create mode 100644 codex-rs/core/tests/suite/text_encoding_fix.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index b8175ee36b..01abeee6af 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -726,6 +726,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chardetng" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14b8f0b65b7b08ae3c8187e8d77174de20cb6777864c6b832d8ad365999cf1ea" +dependencies = [ + "cfg-if", + "encoding_rs", + "memchr", +] + [[package]] name = "chrono" version = "0.4.42" @@ -1081,6 +1092,7 @@ dependencies = [ "async-trait", "base64", "bytes", + "chardetng", "chrono", "codex-app-server-protocol", "codex-apply-patch", @@ -1103,6 +1115,7 @@ dependencies = [ "ctor 0.5.0", "dirs", "dunce", + "encoding_rs", "env-flags", "escargot", "eventsource-stream", diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index e2aa981381..61ab7749b0 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -111,6 +111,7 @@ axum = { version = "0.8", default-features = false } base64 = "0.22.1" bytes = "1.10.1" chrono = "0.4.42" +chardetng = "0.1.17" clap = "4" clap_complete = "4" color-eyre = "0.6.3" @@ -123,6 +124,7 @@ dotenvy = "0.15.7" dunce = "1.0.4" env-flags = "0.1.1" env_logger = "0.11.5" +encoding_rs = "0.8.35" escargot = "0.5" eventsource-stream = "0.2.3" futures = { version = "0.3", default-features = false } diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index bc7b3668d0..565b9ddd57 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -19,6 +19,7 @@ async-trait = { workspace = true } base64 = { workspace = true } bytes = { workspace = true } chrono = { workspace = true, features = ["serde"] } +chardetng = { workspace = true } codex-app-server-protocol = { workspace = true } codex-apply-patch = { workspace = true } codex-async-utils = { workspace = true } @@ -37,6 +38,7 @@ codex-windows-sandbox = { package = "codex-windows-sandbox", path = "../windows- dirs = { workspace = true } dunce = { workspace = true } env-flags = { workspace = true } +encoding_rs = { workspace = true } eventsource-stream = { workspace = true } futures = { workspace = true } http = { workspace = true } diff --git a/codex-rs/core/src/exec.rs b/codex-rs/core/src/exec.rs index 583c2233a2..f7a145663b 100644 --- a/codex-rs/core/src/exec.rs +++ b/codex-rs/core/src/exec.rs @@ -28,6 +28,7 @@ use crate::sandboxing::ExecEnv; use crate::sandboxing::SandboxManager; use crate::spawn::StdioPolicy; use crate::spawn::spawn_child_async; +use crate::text_encoding::bytes_to_string_smart; const DEFAULT_TIMEOUT_MS: u64 = 10_000; @@ -414,7 +415,7 @@ impl StreamOutput { impl StreamOutput> { pub fn from_utf8_lossy(&self) -> StreamOutput { StreamOutput { - text: String::from_utf8_lossy(&self.text).to_string(), + text: bytes_to_string_smart(&self.text), truncated_after_lines: self.truncated_after_lines, } } diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index 2ae11d79d0..6906489e7e 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -39,6 +39,7 @@ pub mod parse_command; pub mod powershell; mod response_processing; pub mod sandboxing; +mod text_encoding; pub mod token_data; mod truncate; mod unified_exec; diff --git a/codex-rs/core/src/text_encoding.rs b/codex-rs/core/src/text_encoding.rs new file mode 100644 index 0000000000..fde44c4195 --- /dev/null +++ b/codex-rs/core/src/text_encoding.rs @@ -0,0 +1,461 @@ +//! Text encoding detection and conversion utilities for shell output. +//! +//! Windows users frequently run into code pages such as CP1251 or CP866 when invoking commands +//! through VS Code. Those bytes show up as invalid UTF-8 and used to be replaced with the standard +//! Unicode replacement character. We now lean on `chardetng` and `encoding_rs` so we can +//! automatically detect and decode the vast majority of legacy encodings before falling back to +//! lossy UTF-8 decoding. + +use chardetng::EncodingDetector; +use encoding_rs::Encoding; +use encoding_rs::IBM866; +use encoding_rs::WINDOWS_1252; + +/// Attempts to convert arbitrary bytes to UTF-8 with best-effort encoding detection. +pub fn bytes_to_string_smart(bytes: &[u8]) -> String { + if bytes.is_empty() { + return String::new(); + } + + if let Ok(utf8_str) = std::str::from_utf8(bytes) { + return utf8_str.to_owned(); + } + + let encoding = detect_encoding(bytes); + decode_bytes(bytes, encoding) +} + +// Windows-1252 reassigns a handful of 0x80-0x9F slots to smart punctuation (curly quotes, dashes, +// ™). CP866 uses those *same byte values* for uppercase Cyrillic letters. When chardetng sees shell +// snippets that mix these bytes with ASCII it sometimes guesses IBM866, so “smart quotes” render as +// Cyrillic garbage (“УФЦ”) in VS Code. However, CP866 uppercase tokens are perfectly valid output +// (e.g., `ПРИ test`) so we cannot flip every 0x80-0x9F byte to Windows-1252 either. The compromise +// is to only coerce IBM866 to Windows-1252 when (a) the high bytes are exclusively the punctuation +// values listed below and (b) we spot adjacent ASCII. This targets the real failure case without +// clobbering legitimate Cyrillic text. If another code page has a similar collision, introduce a +// dedicated allowlist (like this one) plus unit tests that capture the actual shell output we want +// to preserve. Windows-1252 byte values for smart punctuation. +const WINDOWS_1252_PUNCT_BYTES: [u8; 8] = [ + 0x91, // ‘ (left single quotation mark) + 0x92, // ’ (right single quotation mark) + 0x93, // “ (left double quotation mark) + 0x94, // ” (right double quotation mark) + 0x95, // • (bullet) + 0x96, // – (en dash) + 0x97, // — (em dash) + 0x99, // ™ (trade mark sign) +]; + +fn detect_encoding(bytes: &[u8]) -> &'static Encoding { + let mut detector = EncodingDetector::new(); + detector.feed(bytes, true); + let (encoding, _is_confident) = detector.guess_assess(None, true); + + // chardetng occasionally reports IBM866 for short strings that only contain Windows-1252 “smart + // punctuation” bytes (0x80-0x9F) because that range maps to Cyrillic letters in IBM866. When + // those bytes show up alongside an ASCII word (typical shell output: `"“`test), we know the + // intent was likely CP1252 quotes/dashes. Prefer WINDOWS_1252 in that specific situation so we + // render the characters users expect instead of Cyrillic junk. References: + // - Windows-1252 reserving 0x80-0x9F for curly quotes/dashes: + // https://en.wikipedia.org/wiki/Windows-1252 + // - CP866 mapping 0x93/0x94/0x96 to Cyrillic letters, so the same bytes show up as “УФЦ” when + // mis-decoded: https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/PC/CP866.TXT + if encoding == IBM866 && looks_like_windows_1252_punctuation(bytes) { + return WINDOWS_1252; + } + + encoding +} + +fn decode_bytes(bytes: &[u8], encoding: &'static Encoding) -> String { + let (decoded, _, had_errors) = encoding.decode(bytes); + + if had_errors { + return String::from_utf8_lossy(bytes).into_owned(); + } + + decoded.into_owned() +} + +/// Detect whether the byte stream looks like Windows-1252 “smart punctuation” wrapped around +/// otherwise-ASCII text. +/// +/// Context: IBM866 and Windows-1252 share the 0x80-0x9F slot range. In IBM866 these bytes decode to +/// Cyrillic letters, whereas Windows-1252 maps them to curly quotes and dashes. chardetng can guess +/// IBM866 for short snippets that only contain those bytes, which turns shell output such as +/// `“test”` into unreadable Cyrillic. To avoid that, we treat inputs comprising a handful of bytes +/// from the problematic range plus ASCII letters as CP1252 punctuation. We deliberately do *not* +/// cap how many of those punctuation bytes we accept: VS Code frequently prints several quoted +/// phrases (e.g., `"foo" – "bar"`), and truncating the count would once again mis-decode those as +/// Cyrillic. If we discover additional encodings with overlapping byte ranges, prefer adding +/// encoding-specific byte allowlists like `WINDOWS_1252_PUNCT` and tests that exercise real-world +/// shell snippets. +fn looks_like_windows_1252_punctuation(bytes: &[u8]) -> bool { + let mut saw_extended_punctuation = false; + let mut saw_ascii_word = false; + + for &byte in bytes { + if byte >= 0xA0 { + return false; + } + if (0x80..=0x9F).contains(&byte) { + if !is_windows_1252_punct(byte) { + return false; + } + saw_extended_punctuation = true; + } + if byte.is_ascii_alphabetic() { + saw_ascii_word = true; + } + } + + saw_extended_punctuation && saw_ascii_word +} + +fn is_windows_1252_punct(byte: u8) -> bool { + WINDOWS_1252_PUNCT_BYTES.contains(&byte) +} + +#[cfg(test)] +mod tests { + use super::*; + use encoding_rs::BIG5; + use encoding_rs::EUC_KR; + use encoding_rs::GBK; + use encoding_rs::ISO_8859_2; + use encoding_rs::ISO_8859_3; + use encoding_rs::ISO_8859_4; + use encoding_rs::ISO_8859_5; + use encoding_rs::ISO_8859_6; + use encoding_rs::ISO_8859_7; + use encoding_rs::ISO_8859_8; + use encoding_rs::ISO_8859_10; + use encoding_rs::ISO_8859_13; + use encoding_rs::SHIFT_JIS; + use encoding_rs::WINDOWS_874; + use encoding_rs::WINDOWS_1250; + use encoding_rs::WINDOWS_1251; + use encoding_rs::WINDOWS_1253; + use encoding_rs::WINDOWS_1254; + use encoding_rs::WINDOWS_1255; + use encoding_rs::WINDOWS_1256; + use encoding_rs::WINDOWS_1257; + use encoding_rs::WINDOWS_1258; + use pretty_assertions::assert_eq; + + #[test] + fn test_utf8_passthrough() { + // Fast path: when UTF-8 is valid we should avoid copies and return as-is. + let utf8_text = "Hello, мир! 世界"; + let bytes = utf8_text.as_bytes(); + assert_eq!(bytes_to_string_smart(bytes), utf8_text); + } + + #[test] + fn test_cp1251_russian_text() { + // Cyrillic text emitted by PowerShell/WSL in CP1251 should decode cleanly. + let bytes = b"\xEF\xF0\xE8\xEC\xE5\xF0"; // "пример" encoded with Windows-1251 + assert_eq!(bytes_to_string_smart(bytes), "пример"); + } + + #[test] + fn test_cp1251_privet_word() { + // Regression: CP1251 words like "Привет" must not be mis-identified as Windows-1252. + let bytes = b"\xCF\xF0\xE8\xE2\xE5\xF2"; // "Привет" encoded with Windows-1251 + assert_eq!(bytes_to_string_smart(bytes), "Привет"); + } + + #[test] + fn test_koi8_r_privet_word() { + // KOI8-R output should decode to the original Cyrillic as well. + let bytes = b"\xF0\xD2\xC9\xD7\xC5\xD4"; // "Привет" encoded with KOI8-R + assert_eq!(bytes_to_string_smart(bytes), "Привет"); + } + + #[test] + fn test_cp866_russian_text() { + // Legacy consoles (cmd.exe) commonly emit CP866 bytes for Cyrillic content. + let bytes = b"\xAF\xE0\xA8\xAC\xA5\xE0"; // "пример" encoded with CP866 + assert_eq!(bytes_to_string_smart(bytes), "пример"); + } + + #[test] + fn test_cp866_uppercase_text() { + // Ensure the IBM866 heuristic still returns IBM866 for uppercase-only words. + let bytes = b"\x8F\x90\x88"; // "ПРИ" encoded with CP866 uppercase letters + assert_eq!(bytes_to_string_smart(bytes), "ПРИ"); + } + + #[test] + fn test_cp866_uppercase_followed_by_ascii() { + // Regression test: uppercase CP866 tokens next to ASCII text should not be treated as + // CP1252. + let bytes = b"\x8F\x90\x88 test"; // "ПРИ test" encoded with CP866 uppercase letters followed by ASCII + assert_eq!(bytes_to_string_smart(bytes), "ПРИ test"); + } + + #[test] + fn test_windows_1252_quotes() { + // Smart detection should map Windows-1252 punctuation into proper Unicode. + let bytes = b"\x93\x94test"; + assert_eq!(bytes_to_string_smart(bytes), "\u{201C}\u{201D}test"); + } + + #[test] + fn test_windows_1252_multiple_quotes() { + // Longer snippets of punctuation (e.g., “foo” – “bar”) should still flip to CP1252. + let bytes = b"\x93foo\x94 \x96 \x93bar\x94"; + assert_eq!( + bytes_to_string_smart(bytes), + "\u{201C}foo\u{201D} \u{2013} \u{201C}bar\u{201D}" + ); + } + + #[test] + fn test_windows_1252_privet_gibberish_is_preserved() { + // Windows-1252 cannot encode Cyrillic; if the input literally contains "ПÑ..." we should not "fix" it. + let bytes = "Привет".as_bytes(); + assert_eq!(bytes_to_string_smart(bytes), "Привет"); + } + + #[test] + fn test_iso8859_1_latin_text() { + // ISO-8859-1 (code page 28591) is the Latin segment used by LatArCyrHeb. + // encoding_rs unifies ISO-8859-1 with Windows-1252, so reuse that constant here. + let (encoded, _, had_errors) = WINDOWS_1252.encode("Hello"); + assert!(!had_errors, "failed to encode Latin sample"); + assert_eq!(bytes_to_string_smart(encoded.as_ref()), "Hello"); + } + + #[test] + fn test_iso8859_2_central_european_text() { + // ISO-8859-2 (code page 28592) covers additional Central European glyphs. + let (encoded, _, had_errors) = ISO_8859_2.encode("Příliš žluťoučký kůň"); + assert!(!had_errors, "failed to encode ISO-8859-2 sample"); + assert_eq!( + bytes_to_string_smart(encoded.as_ref()), + "Příliš žluťoučký kůň" + ); + } + + #[test] + fn test_iso8859_3_south_europe_text() { + // ISO-8859-3 (code page 28593) adds support for Maltese/Esperanto letters. + // chardetng rarely distinguishes ISO-8859-3 from neighboring Latin code pages, so we rely on + // an ASCII-only sample to ensure round-tripping still succeeds. + let (encoded, _, had_errors) = ISO_8859_3.encode("Esperanto and Maltese"); + assert!(!had_errors, "failed to encode ISO-8859-3 sample"); + assert_eq!( + bytes_to_string_smart(encoded.as_ref()), + "Esperanto and Maltese" + ); + } + + #[test] + fn test_iso8859_4_baltic_text() { + // ISO-8859-4 (code page 28594) targets the Baltic/Nordic repertoire. + let sample = "Šis ir rakstzīmju kodēšanas tests. Dažās valodās, kurās tiek \ + izmantotas latīņu valodas burti, lēmuma pieņemšanai mums ir nepieciešams \ + vairāk ieguldījuma."; + let (encoded, _, had_errors) = ISO_8859_4.encode(sample); + assert!(!had_errors, "failed to encode ISO-8859-4 sample"); + assert_eq!(bytes_to_string_smart(encoded.as_ref()), sample); + } + + #[test] + fn test_iso8859_5_cyrillic_text() { + // ISO-8859-5 (code page 28595) covers the Cyrillic portion. + let (encoded, _, had_errors) = ISO_8859_5.encode("Привет"); + assert!(!had_errors, "failed to encode Cyrillic sample"); + assert_eq!(bytes_to_string_smart(encoded.as_ref()), "Привет"); + } + + #[test] + fn test_iso8859_6_arabic_text() { + // ISO-8859-6 (code page 28596) covers the Arabic glyphs. + let (encoded, _, had_errors) = ISO_8859_6.encode("مرحبا"); + assert!(!had_errors, "failed to encode Arabic sample"); + assert_eq!(bytes_to_string_smart(encoded.as_ref()), "مرحبا"); + } + + #[test] + fn test_iso8859_7_greek_text() { + // ISO-8859-7 (code page 28597) is used for Greek locales. + let (encoded, _, had_errors) = ISO_8859_7.encode("Καλημέρα"); + assert!(!had_errors, "failed to encode ISO-8859-7 sample"); + assert_eq!(bytes_to_string_smart(encoded.as_ref()), "Καλημέρα"); + } + + #[test] + fn test_iso8859_8_hebrew_text() { + // ISO-8859-8 (code page 28598) covers the Hebrew glyphs. + let (encoded, _, had_errors) = ISO_8859_8.encode("שלום"); + assert!(!had_errors, "failed to encode Hebrew sample"); + assert_eq!(bytes_to_string_smart(encoded.as_ref()), "שלום"); + } + + #[test] + fn test_iso8859_9_turkish_text() { + // ISO-8859-9 (code page 28599) mirrors Latin-1 but inserts Turkish letters. + // encoding_rs exposes the equivalent Windows-1254 mapping. + let (encoded, _, had_errors) = WINDOWS_1254.encode("İstanbul"); + assert!(!had_errors, "failed to encode ISO-8859-9 sample"); + assert_eq!(bytes_to_string_smart(encoded.as_ref()), "İstanbul"); + } + + #[test] + fn test_iso8859_10_nordic_text() { + // ISO-8859-10 (code page 28600) adds additional Nordic letters. + let sample = "Þetta er prófun fyrir Ægir og Øystein."; + let (encoded, _, had_errors) = ISO_8859_10.encode(sample); + assert!(!had_errors, "failed to encode ISO-8859-10 sample"); + assert_eq!(bytes_to_string_smart(encoded.as_ref()), sample); + } + + #[test] + fn test_iso8859_11_thai_text() { + // ISO-8859-11 (code page 28601) mirrors TIS-620 / Windows-874 for Thai. + let sample = "ภาษาไทยสำหรับการทดสอบ ISO-8859-11"; + // encoding_rs exposes the equivalent Windows-874 encoding, so use that constant. + let (encoded, _, had_errors) = WINDOWS_874.encode(sample); + assert!(!had_errors, "failed to encode ISO-8859-11 sample"); + assert_eq!(bytes_to_string_smart(encoded.as_ref()), sample); + } + + // ISO-8859-12 was never standardized, and encodings 14–16 cannot be distinguished reliably + // without the heuristics we removed (chardetng generally reports neighboring Latin pages), so + // we intentionally omit coverage for those slots until the detector can identify them. + + #[test] + fn test_iso8859_13_baltic_text() { + // ISO-8859-13 (code page 28603) is common across Baltic languages. + let (encoded, _, had_errors) = ISO_8859_13.encode("Sveiki"); + assert!(!had_errors, "failed to encode ISO-8859-13 sample"); + assert_eq!(bytes_to_string_smart(encoded.as_ref()), "Sveiki"); + } + + #[test] + fn test_windows_1250_central_european_text() { + let (encoded, _, had_errors) = WINDOWS_1250.encode("Příliš žluťoučký kůň"); + assert!(!had_errors, "failed to encode Central European sample"); + assert_eq!( + bytes_to_string_smart(encoded.as_ref()), + "Příliš žluťoučký kůň" + ); + } + + #[test] + fn test_windows_1251_encoded_text() { + let (encoded, _, had_errors) = WINDOWS_1251.encode("Привет из Windows-1251"); + assert!(!had_errors, "failed to encode Windows-1251 sample"); + assert_eq!( + bytes_to_string_smart(encoded.as_ref()), + "Привет из Windows-1251" + ); + } + + #[test] + fn test_windows_1253_greek_text() { + let (encoded, _, had_errors) = WINDOWS_1253.encode("Γειά σου"); + assert!(!had_errors, "failed to encode Greek sample"); + assert_eq!(bytes_to_string_smart(encoded.as_ref()), "Γειά σου"); + } + + #[test] + fn test_windows_1254_turkish_text() { + let (encoded, _, had_errors) = WINDOWS_1254.encode("İstanbul"); + assert!(!had_errors, "failed to encode Turkish sample"); + assert_eq!(bytes_to_string_smart(encoded.as_ref()), "İstanbul"); + } + + #[test] + fn test_windows_1255_hebrew_text() { + let (encoded, _, had_errors) = WINDOWS_1255.encode("שלום"); + assert!(!had_errors, "failed to encode Windows-1255 Hebrew sample"); + assert_eq!(bytes_to_string_smart(encoded.as_ref()), "שלום"); + } + + #[test] + fn test_windows_1256_arabic_text() { + let (encoded, _, had_errors) = WINDOWS_1256.encode("مرحبا"); + assert!(!had_errors, "failed to encode Windows-1256 Arabic sample"); + assert_eq!(bytes_to_string_smart(encoded.as_ref()), "مرحبا"); + } + + #[test] + fn test_windows_1257_baltic_text() { + let (encoded, _, had_errors) = WINDOWS_1257.encode("Pērkons"); + assert!(!had_errors, "failed to encode Baltic sample"); + assert_eq!(bytes_to_string_smart(encoded.as_ref()), "Pērkons"); + } + + #[test] + fn test_windows_1258_vietnamese_text() { + let (encoded, _, had_errors) = WINDOWS_1258.encode("Xin chào"); + assert!(!had_errors, "failed to encode Vietnamese sample"); + assert_eq!(bytes_to_string_smart(encoded.as_ref()), "Xin chào"); + } + + #[test] + fn test_windows_874_thai_text() { + let (encoded, _, had_errors) = WINDOWS_874.encode("สวัสดีครับ นี่คือการทดสอบภาษาไทย"); + assert!(!had_errors, "failed to encode Thai sample"); + assert_eq!( + bytes_to_string_smart(encoded.as_ref()), + "สวัสดีครับ นี่คือการทดสอบภาษาไทย" + ); + } + + #[test] + fn test_windows_932_shift_jis_text() { + let (encoded, _, had_errors) = SHIFT_JIS.encode("こんにちは"); + assert!(!had_errors, "failed to encode Shift-JIS sample"); + assert_eq!(bytes_to_string_smart(encoded.as_ref()), "こんにちは"); + } + + #[test] + fn test_windows_936_gbk_text() { + let (encoded, _, had_errors) = GBK.encode("你好，世界，这是一个测试"); + assert!(!had_errors, "failed to encode GBK sample"); + assert_eq!( + bytes_to_string_smart(encoded.as_ref()), + "你好，世界，这是一个测试" + ); + } + + #[test] + fn test_windows_949_korean_text() { + let (encoded, _, had_errors) = EUC_KR.encode("안녕하세요"); + assert!(!had_errors, "failed to encode Korean sample"); + assert_eq!(bytes_to_string_smart(encoded.as_ref()), "안녕하세요"); + } + + #[test] + fn test_windows_950_big5_text() { + let (encoded, _, had_errors) = BIG5.encode("繁體"); + assert!(!had_errors, "failed to encode Big5 sample"); + assert_eq!(bytes_to_string_smart(encoded.as_ref()), "繁體"); + } + + #[test] + fn test_latin1_cafe() { + // Latin-1 bytes remain common in Western-European locales; decode them directly. + let bytes = b"caf\xE9"; // codespell:ignore caf + assert_eq!(bytes_to_string_smart(bytes), "café"); + } + + #[test] + fn test_preserves_ansi_sequences() { + // ANSI escape sequences should survive regardless of the detected encoding. + let bytes = b"\x1b[31mred\x1b[0m"; + assert_eq!(bytes_to_string_smart(bytes), "\x1b[31mred\x1b[0m"); + } + + #[test] + fn test_fallback_to_lossy() { + // Completely invalid sequences fall back to the old lossy behavior. + let invalid_bytes = [0xFF, 0xFE, 0xFD]; + let result = bytes_to_string_smart(&invalid_bytes); + assert_eq!(result, String::from_utf8_lossy(&invalid_bytes)); + } +} diff --git a/codex-rs/core/tests/suite/mod.rs b/codex-rs/core/tests/suite/mod.rs index 60b828b1c3..b877663614 100644 --- a/codex-rs/core/tests/suite/mod.rs +++ b/codex-rs/core/tests/suite/mod.rs @@ -49,6 +49,7 @@ mod seatbelt; mod shell_serialization; mod stream_error_allows_next_turn; mod stream_no_completed; +mod text_encoding_fix; mod tool_harness; mod tool_parallelism; mod tools; diff --git a/codex-rs/core/tests/suite/text_encoding_fix.rs b/codex-rs/core/tests/suite/text_encoding_fix.rs new file mode 100644 index 0000000000..ecebb1e429 --- /dev/null +++ b/codex-rs/core/tests/suite/text_encoding_fix.rs @@ -0,0 +1,77 @@ +//! Integration test for the text encoding fix for issue #6178. +//! +//! These tests simulate VSCode's shell preview on Windows/WSL where the output +//! may be encoded with a legacy code page before it reaches Codex. + +use codex_core::exec::StreamOutput; +use pretty_assertions::assert_eq; + +#[test] +fn test_utf8_shell_output() { + // Baseline: UTF-8 output should bypass the detector and remain unchanged. + assert_eq!(decode_shell_output("пример".as_bytes()), "пример"); +} + +#[test] +fn test_cp1251_shell_output() { + // VS Code shells on Windows frequently surface CP1251 bytes for Cyrillic text. + assert_eq!(decode_shell_output(b"\xEF\xF0\xE8\xEC\xE5\xF0"), "пример"); +} + +#[test] +fn test_cp866_shell_output() { + // Native cmd.exe still defaults to CP866; make sure we recognize that too. + assert_eq!(decode_shell_output(b"\xAF\xE0\xA8\xAC\xA5\xE0"), "пример"); +} + +#[test] +fn test_windows_1252_smart_decoding() { + // Smart detection should turn fancy quotes/dashes into the proper Unicode glyphs. + assert_eq!( + decode_shell_output(b"\x93\x94 test \x96 dash"), + "\u{201C}\u{201D} test \u{2013} dash" + ); +} + +#[test] +fn test_smart_decoding_improves_over_lossy_utf8() { + // Regression guard: String::from_utf8_lossy() alone used to emit replacement chars here. + let bytes = b"\x93\x94 test \x96 dash"; + assert!( + String::from_utf8_lossy(bytes).contains('\u{FFFD}'), + "lossy UTF-8 should inject replacement chars" + ); + assert_eq!( + decode_shell_output(bytes), + "\u{201C}\u{201D} test \u{2013} dash", + "smart decoding should keep curly quotes intact" + ); +} + +#[test] +fn test_mixed_ascii_and_legacy_encoding() { + // Commands tend to mix ASCII status text with Latin-1 bytes (e.g. café). + assert_eq!(decode_shell_output(b"Output: caf\xE9"), "Output: café"); // codespell:ignore caf +} + +#[test] +fn test_pure_latin1_shell_output() { + // Latin-1 by itself should still decode correctly (regression coverage for the older tests). + assert_eq!(decode_shell_output(b"caf\xE9"), "café"); // codespell:ignore caf +} + +#[test] +fn test_invalid_bytes_still_fall_back_to_lossy() { + // If detection fails, we still want the user to see replacement characters. + let bytes = b"\xFF\xFE\xFD"; + assert_eq!(decode_shell_output(bytes), String::from_utf8_lossy(bytes)); +} + +fn decode_shell_output(bytes: &[u8]) -> String { + StreamOutput { + text: bytes.to_vec(), + truncated_after_lines: None, + } + .from_utf8_lossy() + .text +} From 52d0ec4cd84893a08b0c7f3ea7133d1ecaa39be7 Mon Sep 17 00:00:00 2001 From: pakrym-oai Date: Thu, 20 Nov 2025 11:15:04 -0800 Subject: [PATCH 22/42] Delete tiktoken-rs (#7018) --- codex-rs/Cargo.lock | 51 +---- codex-rs/Cargo.toml | 4 - codex-rs/core/Cargo.toml | 1 - codex-rs/core/src/codex.rs | 4 - codex-rs/core/src/config/edit.rs | 4 - codex-rs/core/src/context_manager/history.rs | 31 ++-- codex-rs/core/tests/suite/compact.rs | 2 +- codex-rs/utils/tokenizer/Cargo.toml | 17 -- codex-rs/utils/tokenizer/src/lib.rs | 186 ------------------- 9 files changed, 17 insertions(+), 283 deletions(-) delete mode 100644 codex-rs/utils/tokenizer/Cargo.toml delete mode 100644 codex-rs/utils/tokenizer/src/lib.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 01abeee6af..fbbdb3d1f0 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -260,7 +260,7 @@ dependencies = [ "memchr", "proc-macro2", "quote", - "rustc-hash 2.1.1", + "rustc-hash", "serde", "serde_derive", "syn 2.0.104", @@ -1108,7 +1108,6 @@ dependencies = [ "codex-utils-pty", "codex-utils-readiness", "codex-utils-string", - "codex-utils-tokenizer", "codex-windows-sandbox", "core-foundation 0.9.4", "core_test_support", @@ -1628,18 +1627,6 @@ dependencies = [ name = "codex-utils-string" version = "0.0.0" -[[package]] -name = "codex-utils-tokenizer" -version = "0.0.0" -dependencies = [ - "anyhow", - "codex-utils-cache", - "pretty_assertions", - "thiserror 2.0.17", - "tiktoken-rs", - "tokio", -] - [[package]] name = "codex-windows-sandbox" version = "0.1.0" @@ -2461,17 +2448,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "fancy-regex" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2" -dependencies = [ - "bit-set", - "regex-automata", - "regex-syntax 0.8.5", -] - [[package]] name = "fastrand" version = "2.3.0" @@ -4796,7 +4772,7 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash 2.1.1", + "rustc-hash", "rustls", "socket2 0.6.0", "thiserror 2.0.17", @@ -4816,7 +4792,7 @@ dependencies = [ "lru-slab", "rand 0.9.2", "ring", - "rustc-hash 2.1.1", + "rustc-hash", "rustls", "rustls-pki-types", "slab", @@ -5161,12 +5137,6 @@ version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "989e6739f80c4ad5b13e0fd7fe89531180375b18520cc8c82080e4dc4035b84f" -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - [[package]] name = "rustc-hash" version = "2.1.1" @@ -6387,21 +6357,6 @@ dependencies = [ "zune-jpeg", ] -[[package]] -name = "tiktoken-rs" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a19830747d9034cd9da43a60eaa8e552dfda7712424aebf187b7a60126bae0d" -dependencies = [ - "anyhow", - "base64", - "bstr", - "fancy-regex", - "lazy_static", - "regex", - "rustc-hash 1.1.0", -] - [[package]] name = "time" version = "0.3.44" diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index 61ab7749b0..b34cd46b4b 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -41,7 +41,6 @@ members = [ "utils/pty", "utils/readiness", "utils/string", - "utils/tokenizer", ] resolver = "2" @@ -90,7 +89,6 @@ codex-utils-json-to-toml = { path = "utils/json-to-toml" } codex-utils-pty = { path = "utils/pty" } codex-utils-readiness = { path = "utils/readiness" } codex-utils-string = { path = "utils/string" } -codex-utils-tokenizer = { path = "utils/tokenizer" } codex-windows-sandbox = { path = "windows-sandbox-rs" } core_test_support = { path = "core/tests/common" } mcp-types = { path = "mcp-types" } @@ -190,7 +188,6 @@ tempfile = "3.23.0" test-log = "0.2.18" textwrap = "0.16.2" thiserror = "2.0.17" -tiktoken-rs = "0.9" time = "0.3" tiny_http = "0.12" tokio = "1" @@ -268,7 +265,6 @@ ignored = [ "icu_provider", "openssl-sys", "codex-utils-readiness", - "codex-utils-tokenizer", ] [profile.release] diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index 565b9ddd57..669a9a63f5 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -33,7 +33,6 @@ codex-rmcp-client = { workspace = true } codex-utils-pty = { workspace = true } codex-utils-readiness = { workspace = true } codex-utils-string = { workspace = true } -codex-utils-tokenizer = { workspace = true } codex-windows-sandbox = { package = "codex-windows-sandbox", path = "../windows-sandbox-rs" } dirs = { workspace = true } dunce = { workspace = true } diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 55dfe17047..1cf4e923cd 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -133,7 +133,6 @@ use codex_protocol::protocol::InitialHistory; use codex_protocol::user_input::UserInput; use codex_utils_readiness::Readiness; use codex_utils_readiness::ReadinessFlag; -use codex_utils_tokenizer::warm_model_cache; /// The high-level interface to the Codex system. /// It operates as a queue pair where you send submissions and receive events. @@ -555,9 +554,6 @@ impl Session { // Create the mutable state for the Session. let state = SessionState::new(session_configuration.clone()); - // Warm the tokenizer cache for the session model without blocking startup. - warm_model_cache(&session_configuration.model); - let services = SessionServices { mcp_connection_manager: Arc::new(RwLock::new(McpConnectionManager::default())), mcp_startup_cancellation_token: CancellationToken::new(), diff --git a/codex-rs/core/src/config/edit.rs b/codex-rs/core/src/config/edit.rs index bd7c16a0ae..b8862fa5c5 100644 --- a/codex-rs/core/src/config/edit.rs +++ b/codex-rs/core/src/config/edit.rs @@ -4,7 +4,6 @@ use crate::config::types::Notice; use anyhow::Context; use codex_protocol::config_types::ReasoningEffort; use codex_protocol::config_types::TrustLevel; -use codex_utils_tokenizer::warm_model_cache; use std::collections::BTreeMap; use std::path::Path; use std::path::PathBuf; @@ -231,9 +230,6 @@ impl ConfigDocument { fn apply(&mut self, edit: &ConfigEdit) -> anyhow::Result { match edit { ConfigEdit::SetModel { model, effort } => Ok({ - if let Some(model) = &model { - warm_model_cache(model) - } let mut mutated = false; mutated |= self.write_profile_value( &["model"], diff --git a/codex-rs/core/src/context_manager/history.rs b/codex-rs/core/src/context_manager/history.rs index 8575fe9f6c..8eefcbf855 100644 --- a/codex-rs/core/src/context_manager/history.rs +++ b/codex-rs/core/src/context_manager/history.rs @@ -1,13 +1,13 @@ use crate::codex::TurnContext; use crate::context_manager::normalize; use crate::truncate::TruncationPolicy; +use crate::truncate::approx_token_count; use crate::truncate::truncate_function_output_items_with_policy; use crate::truncate::truncate_text; use codex_protocol::models::FunctionCallOutputPayload; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::TokenUsage; use codex_protocol::protocol::TokenUsageInfo; -use codex_utils_tokenizer::Tokenizer; use std::ops::Deref; /// Transcript of conversation history @@ -74,26 +74,21 @@ impl ContextManager { history } - // Estimate the number of tokens in the history. Return None if no tokenizer - // is available. This does not consider the reasoning traces. - // /!\ The value is a lower bound estimate and does not represent the exact - // context length. + // Estimate token usage using byte-based heuristics from the truncation helpers. + // This is a coarse lower bound, not a tokenizer-accurate count. pub(crate) fn estimate_token_count(&self, turn_context: &TurnContext) -> Option { - let model = turn_context.client.get_model(); - let tokenizer = Tokenizer::for_model(model.as_str()).ok()?; let model_family = turn_context.client.get_model_family(); + let base_tokens = + i64::try_from(approx_token_count(model_family.base_instructions.as_str())) + .unwrap_or(i64::MAX); - Some( - self.items - .iter() - .map(|item| { - serde_json::to_string(&item) - .map(|item| tokenizer.count(&item)) - .unwrap_or_default() - }) - .sum::() - + tokenizer.count(model_family.base_instructions.as_str()), - ) + let items_tokens = self.items.iter().fold(0i64, |acc, item| { + let serialized = serde_json::to_string(item).unwrap_or_default(); + let item_tokens = i64::try_from(approx_token_count(&serialized)).unwrap_or(i64::MAX); + acc.saturating_add(item_tokens) + }); + + Some(base_tokens.saturating_add(items_tokens)) } pub(crate) fn remove_first_item(&mut self) { diff --git a/codex-rs/core/tests/suite/compact.rs b/codex-rs/core/tests/suite/compact.rs index 5359f3b1ad..1324d3edb9 100644 --- a/codex-rs/core/tests/suite/compact.rs +++ b/codex-rs/core/tests/suite/compact.rs @@ -384,7 +384,7 @@ async fn manual_compact_uses_custom_prompt() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn manual_compact_emits_estimated_token_usage_event() { +async fn manual_compact_emits_api_and_local_token_usage_events() { skip_if_no_network!(); let server = start_mock_server().await; diff --git a/codex-rs/utils/tokenizer/Cargo.toml b/codex-rs/utils/tokenizer/Cargo.toml deleted file mode 100644 index 7669d6b987..0000000000 --- a/codex-rs/utils/tokenizer/Cargo.toml +++ /dev/null @@ -1,17 +0,0 @@ -[package] -edition.workspace = true -name = "codex-utils-tokenizer" -version.workspace = true - -[lints] -workspace = true - -[dependencies] -anyhow = { workspace = true } -codex-utils-cache = { workspace = true } -thiserror = { workspace = true } -tiktoken-rs = { workspace = true } -tokio = { workspace = true } - -[dev-dependencies] -pretty_assertions = { workspace = true } diff --git a/codex-rs/utils/tokenizer/src/lib.rs b/codex-rs/utils/tokenizer/src/lib.rs deleted file mode 100644 index 1e151393a9..0000000000 --- a/codex-rs/utils/tokenizer/src/lib.rs +++ /dev/null @@ -1,186 +0,0 @@ -use std::fmt; -use std::num::NonZeroUsize; -use std::sync::OnceLock; - -use anyhow::Error as AnyhowError; -use codex_utils_cache::BlockingLruCache; -use thiserror::Error; -use tiktoken_rs::CoreBPE; - -/// Supported local encodings. -#[derive(Debug, Copy, Clone, Eq, PartialEq)] -pub enum EncodingKind { - O200kBase, - Cl100kBase, -} - -impl fmt::Display for EncodingKind { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::O200kBase => f.write_str("o200k_base"), - Self::Cl100kBase => f.write_str("cl100k_base"), - } - } -} - -/// Tokenizer error type. -#[derive(Debug, Error)] -pub enum TokenizerError { - #[error("failed to load encoding {kind}")] - LoadEncoding { - kind: EncodingKind, - #[source] - source: AnyhowError, - }, - #[error("failed to decode tokens")] - Decode { - #[source] - source: AnyhowError, - }, -} - -fn model_cache() -> &'static BlockingLruCache { - static MODEL_CACHE: OnceLock> = OnceLock::new(); - MODEL_CACHE - .get_or_init(|| BlockingLruCache::new(NonZeroUsize::new(64).unwrap_or(NonZeroUsize::MIN))) -} - -/// Fire-and-forget function used to pre-warm model tokenizer loading. This is done -/// on a best-effort basis, without any guarantee about the state of the cache -/// before or after. -/// Only working in Tokio runtimes -pub fn warm_model_cache(model: &str) { - if tokio::runtime::Handle::try_current().is_err() { - return; - } - let model = model.to_string(); - tokio::spawn(async move { - let _ = Tokenizer::for_model(&model); - }); -} - -/// Thin wrapper around a `tiktoken_rs::CoreBPE` tokenizer. -#[derive(Clone)] -pub struct Tokenizer { - inner: CoreBPE, -} - -impl Tokenizer { - /// Build a tokenizer for a specific encoding. - pub fn new(kind: EncodingKind) -> Result { - let loader: fn() -> anyhow::Result = match kind { - EncodingKind::O200kBase => tiktoken_rs::o200k_base, - EncodingKind::Cl100kBase => tiktoken_rs::cl100k_base, - }; - - let inner = loader().map_err(|source| TokenizerError::LoadEncoding { kind, source })?; - Ok(Self { inner }) - } - - /// Default to `O200kBase` - pub fn try_default() -> Result { - Self::new(EncodingKind::O200kBase) - } - - /// Build a tokenizer using an `OpenAI` model name (maps to an encoding). - /// Falls back to the `O200kBase` encoding when the model is unknown. - pub fn for_model(model: &str) -> Result { - let inner = model_cache().get_or_try_insert_with(model.to_owned(), || { - match tiktoken_rs::get_bpe_from_model(model) { - Ok(inner) => Ok(inner), - Err(_model_error) => Tokenizer::new(EncodingKind::O200kBase).map(|e| e.inner), - } - })?; - Ok(Self { inner }) - } - - /// Encode text to token IDs. If `with_special_tokens` is true, special - /// tokens are allowed and may appear in the result. - #[must_use] - pub fn encode(&self, text: &str, with_special_tokens: bool) -> Vec { - let raw = if with_special_tokens { - self.inner.encode_with_special_tokens(text) - } else { - self.inner.encode_ordinary(text) - }; - raw.into_iter().map(|t| t as i32).collect() - } - - /// Count tokens in `text` as a signed integer. - #[must_use] - pub fn count(&self, text: &str) -> i64 { - // Signed length to satisfy our style preference. - i64::try_from(self.inner.encode_ordinary(text).len()).unwrap_or(i64::MAX) - } - - /// Decode token IDs back to text. - pub fn decode(&self, tokens: &[i32]) -> Result { - let raw: Vec = tokens.iter().map(|t| *t as u32).collect(); - self.inner - .decode(raw) - .map_err(|source| TokenizerError::Decode { source }) - } -} - -impl fmt::Debug for Tokenizer { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "Tokenizer {{ inner: }}") - } -} -#[cfg(test)] -mod tests { - use super::*; - use pretty_assertions::assert_eq; - - #[test] - fn cl100k_base_roundtrip_simple() -> Result<(), TokenizerError> { - let tok = Tokenizer::new(EncodingKind::Cl100kBase)?; - let s = "hello world"; - let ids = tok.encode(s, false); - // Stable expectation for cl100k_base - assert_eq!(ids, vec![15339, 1917]); - let back = tok.decode(&ids)?; - assert_eq!(back, s); - Ok(()) - } - - #[test] - fn preserves_whitespace_and_special_tokens_flag() -> Result<(), TokenizerError> { - let tok = Tokenizer::new(EncodingKind::Cl100kBase)?; - let s = "This has multiple spaces"; - let ids_no_special = tok.encode(s, false); - let round = tok.decode(&ids_no_special)?; - assert_eq!(round, s); - - // With special tokens allowed, result may be identical for normal text, - // but the API should still function. - let ids_with_special = tok.encode(s, true); - let round2 = tok.decode(&ids_with_special)?; - assert_eq!(round2, s); - Ok(()) - } - - #[test] - fn model_mapping_builds_tokenizer() -> Result<(), TokenizerError> { - // Choose a long-standing model alias that maps to cl100k_base. - let tok = Tokenizer::for_model("gpt-5.1")?; - let ids = tok.encode("ok", false); - let back = tok.decode(&ids)?; - assert_eq!(back, "ok"); - Ok(()) - } - - #[test] - fn unknown_model_defaults_to_o200k_base() -> Result<(), TokenizerError> { - let fallback = Tokenizer::new(EncodingKind::O200kBase)?; - let tok = Tokenizer::for_model("does-not-exist")?; - let text = "fallback please"; - assert_eq!(tok.encode(text, false), fallback.encode(text, false)); - Ok(()) - } - - #[test] - fn warm_model_cache_without_runtime_is_noop() { - warm_model_cache("gpt-5"); - } -} From 33a6cc66ab50a93cca6640f9525441b6fd086da2 Mon Sep 17 00:00:00 2001 From: Kyuheon Kim <65669461+kyuheon-kr@users.noreply.github.com> Date: Fri, 21 Nov 2025 05:32:12 +0900 Subject: [PATCH 23/42] fix(cli): correct mcp add usage order (#6827) ## Summary - add an explicit `override_usage` string to `AddArgs` so clap prints `` before the command/url choice, matching the actual parser and docs ### Before Usage: codex mcp add [OPTIONS] > ### After Usage: codex mcp add [OPTIONS] [--url | -- ...] --------- Signed-off-by: kyuheon-kr --- codex-rs/cli/src/mcp_cmd.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/codex-rs/cli/src/mcp_cmd.rs b/codex-rs/cli/src/mcp_cmd.rs index ec37c3a6bd..93f22e705c 100644 --- a/codex-rs/cli/src/mcp_cmd.rs +++ b/codex-rs/cli/src/mcp_cmd.rs @@ -79,6 +79,7 @@ pub struct GetArgs { } #[derive(Debug, clap::Parser)] +#[command(override_usage = "codex mcp add [OPTIONS] (--url | -- ...)")] pub struct AddArgs { /// Name for the MCP server configuration. pub name: String, From e150798baf1dc4d88b6627ed9a2c2f684df1d43d Mon Sep 17 00:00:00 2001 From: Lionel Cheng <60159831+lionelchg@users.noreply.github.com> Date: Thu, 20 Nov 2025 12:33:12 -0800 Subject: [PATCH 24/42] Bumped number of fuzzy search results from 8 to 20 (#7013) I just noticed that in the VSCode / Codex extension when you type @ the number of results is around 70: - small video of searching for `mod.rs` inside `codex` repository: https://github.com/user-attachments/assets/46e53d31-adff-465e-b32b-051c4c1c298c - while in the CLI the number of results is currently of 8 which is quite small: Screenshot 2025-11-20 at 09 42 04 I bumped it to 20. I had several cases where I wanted a file and did not find it because the number of results was too small Signed-off-by: lionel-oai Co-authored-by: lionel-oai --- codex-rs/tui/src/file_search.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codex-rs/tui/src/file_search.rs b/codex-rs/tui/src/file_search.rs index 61327cfd24..af46512640 100644 --- a/codex-rs/tui/src/file_search.rs +++ b/codex-rs/tui/src/file_search.rs @@ -31,7 +31,7 @@ use std::time::Duration; use crate::app_event::AppEvent; use crate::app_event_sender::AppEventSender; -const MAX_FILE_SEARCH_RESULTS: NonZeroUsize = NonZeroUsize::new(8).unwrap(); +const MAX_FILE_SEARCH_RESULTS: NonZeroUsize = NonZeroUsize::new(20).unwrap(); const NUM_FILE_SEARCH_THREADS: NonZeroUsize = NonZeroUsize::new(2).unwrap(); /// How long to wait after a keystroke before firing the first search when none From 2c793083f4cb40f31a6b3258aeb1739a9870b8ea Mon Sep 17 00:00:00 2001 From: Jeremy Rose <172423086+nornagon-openai@users.noreply.github.com> Date: Thu, 20 Nov 2025 13:06:22 -0800 Subject: [PATCH 25/42] tui: centralize markdown styling and make inline code cyan (#7023) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Screenshot 2025-11-20 at 12 54 06 PM --- codex-rs/tui/src/markdown_render.rs | 77 ++++++++++++++++++----- codex-rs/tui/src/markdown_render_tests.rs | 2 +- 2 files changed, 63 insertions(+), 16 deletions(-) diff --git a/codex-rs/tui/src/markdown_render.rs b/codex-rs/tui/src/markdown_render.rs index 099d0860c2..19cf944928 100644 --- a/codex-rs/tui/src/markdown_render.rs +++ b/codex-rs/tui/src/markdown_render.rs @@ -10,11 +10,50 @@ use pulldown_cmark::Parser; use pulldown_cmark::Tag; use pulldown_cmark::TagEnd; use ratatui::style::Style; -use ratatui::style::Stylize; use ratatui::text::Line; use ratatui::text::Span; use ratatui::text::Text; +struct MarkdownStyles { + h1: Style, + h2: Style, + h3: Style, + h4: Style, + h5: Style, + h6: Style, + code: Style, + emphasis: Style, + strong: Style, + strikethrough: Style, + ordered_list_marker: Style, + unordered_list_marker: Style, + link: Style, + blockquote: Style, +} + +impl Default for MarkdownStyles { + fn default() -> Self { + use ratatui::style::Stylize; + + Self { + h1: Style::new().bold().underlined(), + h2: Style::new().bold(), + h3: Style::new().bold().italic(), + h4: Style::new().italic(), + h5: Style::new().italic(), + h6: Style::new().italic(), + code: Style::new().cyan(), + emphasis: Style::new().italic(), + strong: Style::new().bold(), + strikethrough: Style::new().crossed_out(), + ordered_list_marker: Style::new().light_blue(), + unordered_list_marker: Style::new(), + link: Style::new().cyan().underlined(), + blockquote: Style::new().green(), + } + } +} + #[derive(Clone, Debug)] struct IndentContext { prefix: Vec>, @@ -51,6 +90,7 @@ where { iter: I, text: Text<'static>, + styles: MarkdownStyles, inline_styles: Vec