From 6b10e22eb3f999fe2b5c6b8054feefbca791e2bd Mon Sep 17 00:00:00 2001 From: pakrym-oai Date: Tue, 29 Jul 2025 16:49:02 -0700 Subject: [PATCH 001/100] Trim bash lc and run with login shell (#1725) include .zshenv, .zprofile by running with the `-l` flag and don't start a shell inside a shell when we see the typical `bash -lc` invocation. --- codex-rs/core/src/shell.rs | 42 +++++++++++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/codex-rs/core/src/shell.rs b/codex-rs/core/src/shell.rs index 463651234c..98addffce2 100644 --- a/codex-rs/core/src/shell.rs +++ b/codex-rs/core/src/shell.rs @@ -20,8 +20,13 @@ impl Shell { return None; } - let mut result = vec![zsh.shell_path.clone(), "-c".to_string()]; - if let Ok(joined) = shlex::try_join(command.iter().map(|s| s.as_str())) { + let mut result = vec![zsh.shell_path.clone()]; + result.push("-lc".to_string()); + + let joined = strip_bash_lc(&command) + .or_else(|| shlex::try_join(command.iter().map(|s| s.as_str())).ok()); + + if let Some(joined) = joined { result.push(format!("source {} && ({joined})", zsh.zshrc_path)); } else { return None; @@ -33,6 +38,19 @@ impl Shell { } } +fn strip_bash_lc(command: &Vec) -> Option { + match command.as_slice() { + // exactly three items + [first, second, third] + // first two must be "bash", "-lc" + if first == "bash" && second == "-lc" => + { + Some(third.clone()) + } + _ => None, + } +} + #[cfg(target_os = "macos")] pub async fn default_user_shell() -> Shell { use tokio::process::Command; @@ -119,15 +137,29 @@ mod tests { let cases = vec![ ( vec!["myecho"], - vec![shell_path, "-c", "source ZSHRC_PATH && (myecho)"], + vec![shell_path, "-lc", "source ZSHRC_PATH && (myecho)"], Some("It works!\n"), ), + ( + vec!["myecho"], + vec![shell_path, "-lc", "source ZSHRC_PATH && (myecho)"], + Some("It works!\n"), + ), + ( + vec!["bash", "-c", "echo 'single' \"double\""], + vec![ + shell_path, + "-lc", + "source ZSHRC_PATH && (bash -c \"echo 'single' \\\"double\\\"\")", + ], + Some("single double\n"), + ), ( vec!["bash", "-lc", "echo 'single' \"double\""], vec![ shell_path, - "-c", - "source ZSHRC_PATH && (bash -lc \"echo 'single' \\\"double\\\"\")", + "-lc", + "source ZSHRC_PATH && (echo 'single' \"double\")", ], Some("single double\n"), ), From 3823b32b7a3cde9fe96b7e4f40ac9e3f5fb73e35 Mon Sep 17 00:00:00 2001 From: aibrahim-oai Date: Tue, 29 Jul 2025 20:14:41 -0700 Subject: [PATCH 002/100] Mcp protocol (#1715) - Add typed MCP protocol surface in `codex-rs/mcp-server/src/mcp_protocol.rs` for `requests`, `responses`, and `notifications` - Requests: `NewConversation`, `Connect`, `SendUserMessage`, `GetConversations` - Message content parts: `Text`, `Image` (`ImageUrl`/`FileId`, optional `ImageDetail`), File (`Url`/`Id`/`inline Data`) - Responses: `ToolCallResponseEnvelope` with optional `isError` and `structuredContent` variants (`NewConversation`, `Connect`, `SendUserMessageAccepted`, `GetConversations`) - Notifications: `InitialState`, `ConnectionRevoked`, `CodexEvent`, `Cancelled` - Uniform `_meta` on `notifications` via `NotificationMeta` (`conversationId`, `requestId`) - Unit tests validate JSON wire shapes for key `requests`/`responses`/`notifications` --- codex-rs/Cargo.lock | 1 + codex-rs/core/src/config_types.rs | 2 +- codex-rs/mcp-server/Cargo.toml | 1 + codex-rs/mcp-server/src/lib.rs | 1 + codex-rs/mcp-server/src/mcp_protocol.rs | 1020 +++++++++++++++++++++++ 5 files changed, 1024 insertions(+), 1 deletion(-) create mode 100644 codex-rs/mcp-server/src/mcp_protocol.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 653c3e4ef2..9abce0c3db 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -822,6 +822,7 @@ dependencies = [ "serde", "serde_json", "shlex", + "strum_macros 0.27.2", "tempfile", "tokio", "tokio-test", diff --git a/codex-rs/core/src/config_types.rs b/codex-rs/core/src/config_types.rs index 735a571edc..9bf0d483e1 100644 --- a/codex-rs/core/src/config_types.rs +++ b/codex-rs/core/src/config_types.rs @@ -78,7 +78,7 @@ pub enum HistoryPersistence { #[derive(Deserialize, Debug, Clone, PartialEq, Default)] pub struct Tui {} -#[derive(Deserialize, Debug, Clone, Copy, PartialEq, Default)] +#[derive(Deserialize, Debug, Clone, Copy, PartialEq, Default, Serialize)] #[serde(rename_all = "kebab-case")] pub enum SandboxMode { #[serde(rename = "read-only")] diff --git a/codex-rs/mcp-server/Cargo.toml b/codex-rs/mcp-server/Cargo.toml index 488ee6a67c..19cf4db538 100644 --- a/codex-rs/mcp-server/Cargo.toml +++ b/codex-rs/mcp-server/Cargo.toml @@ -34,6 +34,7 @@ tokio = { version = "1", features = [ "signal", ] } uuid = { version = "1", features = ["serde", "v4"] } +strum_macros = "0.27.2" [dev-dependencies] assert_cmd = "2" diff --git a/codex-rs/mcp-server/src/lib.rs b/codex-rs/mcp-server/src/lib.rs index aaf67571b4..0912fed118 100644 --- a/codex-rs/mcp-server/src/lib.rs +++ b/codex-rs/mcp-server/src/lib.rs @@ -19,6 +19,7 @@ mod codex_tool_config; mod codex_tool_runner; mod exec_approval; mod json_to_toml; +mod mcp_protocol; mod message_processor; mod outgoing_message; mod patch_approval; diff --git a/codex-rs/mcp-server/src/mcp_protocol.rs b/codex-rs/mcp-server/src/mcp_protocol.rs new file mode 100644 index 0000000000..05eb0a258a --- /dev/null +++ b/codex-rs/mcp-server/src/mcp_protocol.rs @@ -0,0 +1,1020 @@ +use codex_core::config_types::SandboxMode; +use codex_core::protocol::AskForApproval; +use codex_core::protocol::EventMsg; +use serde::Deserialize; +use serde::Serialize; +use strum_macros::Display; +use uuid::Uuid; + +use mcp_types::RequestId; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(transparent)] +pub struct ConversationId(pub Uuid); + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(transparent)] +pub struct MessageId(pub Uuid); + +// Requests +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ToolCallRequest { + #[serde(rename = "jsonrpc")] + pub jsonrpc: &'static str, + pub id: u64, + pub method: &'static str, + pub params: ToolCallRequestParams, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "name", content = "arguments", rename_all = "camelCase")] +pub enum ToolCallRequestParams { + ConversationCreate(ConversationCreateArgs), + ConversationStream(ConversationStreamArgs), + ConversationSendMessage(ConversationSendMessageArgs), + ConversationsList(ConversationsListArgs), +} + +impl ToolCallRequestParams { + /// Wrap this request in a JSON-RPC request. + #[allow(dead_code)] + pub fn into_request(self, id: u64) -> ToolCallRequest { + ToolCallRequest { + jsonrpc: "2.0", + id, + method: "tools/call", + params: self, + } + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ConversationCreateArgs { + pub prompt: String, + pub model: String, + pub cwd: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub approval_policy: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sandbox: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub config: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub profile: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub base_instructions: Option, +} + +/// Optional overrides for an existing conversation's execution context when sending a message. +/// Fields left as `None` inherit the current conversation/session settings. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ConversationOverrides { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub model: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cwd: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub approval_policy: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sandbox: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub config: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub profile: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub base_instructions: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ConversationStreamArgs { + pub conversation_id: ConversationId, +} + +/// If omitted, the message continues from the latest turn. +/// Set to resume/edit from an earlier parent message in the thread. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ConversationSendMessageArgs { + pub conversation_id: ConversationId, + pub content: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub parent_message_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[serde(flatten)] + pub conversation_overrides: Option, +} + +/// Input items for a message. +/// Following OpenAI's Responses API: https://platform.openai.com/docs/api-reference/responses +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "camelCase")] +pub enum MessageInputItem { + Text { + text: String, + }, + Image { + #[serde(flatten)] + source: ImageSource, + #[serde(default, skip_serializing_if = "Option::is_none")] + detail: Option, + }, + File { + #[serde(flatten)] + source: FileSource, + }, +} + +/// Source of an image. +/// Following OpenAI's API: https://platform.openai.com/docs/guides/images-vision#giving-a-model-images-as-input +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum ImageSource { + ImageUrl { image_url: String }, + FileId { file_id: String }, +} + +/// Source of a file. +/// Following OpenAI's Responses API: https://platform.openai.com/docs/guides/pdf-files?api-mode=responses#uploading-files +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum FileSource { + Url { + file_url: String, + }, + Id { + file_id: String, + }, + Base64 { + #[serde(default, skip_serializing_if = "Option::is_none")] + filename: Option, + // Base64-encoded file contents. + file_data: String, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ImageDetail { + Low, + High, + Auto, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ConversationsListArgs { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub limit: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cursor: Option, +} + +// Responses +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ToolCallResponse { + pub request_id: RequestId, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub is_error: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub result: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum ToolCallResponseResult { + ConversationCreate(ConversationCreateResult), + ConversationStream(ConversationStreamResult), + ConversationSendMessage(ConversationSendMessageResult), + ConversationsList(ConversationsListResult), +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ConversationCreateResult { + pub conversation_id: ConversationId, + pub model: String, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ConversationStreamResult {} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ConversationSendMessageResult { + pub success: bool, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ConversationsListResult { + pub conversations: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub next_cursor: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ConversationSummary { + pub conversation_id: ConversationId, + pub title: String, +} + +// Notifications +#[derive(Debug, Clone, Deserialize, Display)] +pub enum ServerNotification { + InitialState(InitialStateNotificationParams), + StreamDisconnected(StreamDisconnectedNotificationParams), + CodexEvent(CodexEventNotificationParams), +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct NotificationMeta { + #[serde(skip_serializing_if = "Option::is_none")] + pub conversation_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub request_id: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct InitialStateNotificationParams { + #[serde(rename = "_meta", skip_serializing_if = "Option::is_none")] + pub meta: Option, + pub initial_state: InitialStatePayload, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct InitialStatePayload { + #[serde(default)] + pub events: Vec, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct StreamDisconnectedNotificationParams { + #[serde(rename = "_meta", skip_serializing_if = "Option::is_none")] + pub meta: Option, + pub reason: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CodexEventNotificationParams { + #[serde(rename = "_meta", skip_serializing_if = "Option::is_none")] + pub meta: Option, + pub msg: EventMsg, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CancelNotificationParams { + pub request_id: RequestId, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reason: Option, +} + +impl Serialize for ServerNotification { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + use serde::ser::SerializeMap; + + let mut map = serializer.serialize_map(Some(2))?; + match self { + ServerNotification::CodexEvent(p) => { + map.serialize_entry("method", &format!("notifications/{}", p.msg))?; + map.serialize_entry("params", p)?; + } + ServerNotification::InitialState(p) => { + map.serialize_entry("method", "notifications/initial_state")?; + map.serialize_entry("params", p)?; + } + ServerNotification::StreamDisconnected(p) => { + map.serialize_entry("method", "notifications/stream_disconnected")?; + map.serialize_entry("params", p)?; + } + } + map.end() + } +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(tag = "method", content = "params", rename_all = "camelCase")] +pub enum ClientNotification { + #[serde(rename = "notifications/cancelled")] + Cancelled(CancelNotificationParams), +} + +#[cfg(test)] +#[allow(clippy::expect_used)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + use serde::Serialize; + use serde_json::Value; + use serde_json::json; + use uuid::uuid; + + fn to_val(v: &T) -> Value { + serde_json::to_value(v).expect("serialize to Value") + } + + // ----- Requests ----- + + #[test] + fn serialize_tool_call_request_params_conversation_create_minimal() { + let req = ToolCallRequestParams::ConversationCreate(ConversationCreateArgs { + prompt: "".into(), + model: "o3".into(), + cwd: "/repo".into(), + approval_policy: None, + sandbox: None, + config: None, + profile: None, + base_instructions: None, + }); + + let observed = to_val(&req.into_request(2)); + let expected = json!({ + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": { + "name": "conversationCreate", + "arguments": { + "prompt": "", + "model": "o3", + "cwd": "/repo" + } + } + }); + assert_eq!(observed, expected); + } + + #[test] + fn serialize_tool_call_request_params_conversation_send_message_with_overrides_and_parent_message_id() + { + let req = ToolCallRequestParams::ConversationSendMessage(ConversationSendMessageArgs { + conversation_id: ConversationId(uuid!("d0f6ecbe-84a2-41c1-b23d-b20473b25eab")), + content: vec![ + MessageInputItem::Text { text: "Hi".into() }, + MessageInputItem::Image { + source: ImageSource::ImageUrl { + image_url: "https://example.com/cat.jpg".into(), + }, + detail: Some(ImageDetail::High), + }, + MessageInputItem::File { + source: FileSource::Base64 { + filename: Some("notes.txt".into()), + file_data: "Zm9vYmFy".into(), + }, + }, + ], + parent_message_id: Some(MessageId(uuid!("67e55044-10b1-426f-9247-bb680e5fe0c8"))), + conversation_overrides: Some(ConversationOverrides { + model: Some("o4-mini".into()), + cwd: Some("/workdir".into()), + approval_policy: None, + sandbox: Some(SandboxMode::DangerFullAccess), + config: Some(json!({"temp": 0.2})), + profile: Some("eng".into()), + base_instructions: Some("Be terse".into()), + }), + }); + + let observed = to_val(&req.into_request(2)); + let expected = json!({ + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": { + "name": "conversationSendMessage", + "arguments": { + "conversation_id": "d0f6ecbe-84a2-41c1-b23d-b20473b25eab", + "content": [ + { "type": "text", "text": "Hi" }, + { "type": "image", "image_url": "https://example.com/cat.jpg", "detail": "high" }, + { "type": "file", "filename": "notes.txt", "file_data": "Zm9vYmFy" } + ], + "parent_message_id": "67e55044-10b1-426f-9247-bb680e5fe0c8", + "model": "o4-mini", + "cwd": "/workdir", + "sandbox": "danger-full-access", + "config": { "temp": 0.2 }, + "profile": "eng", + "base_instructions": "Be terse" + } + } + }); + assert_eq!(observed, expected); + } + + #[test] + fn serialize_tool_call_request_params_conversations_list_with_opts() { + let req = ToolCallRequestParams::ConversationsList(ConversationsListArgs { + limit: Some(50), + cursor: Some("abc".into()), + }); + + let observed = to_val(&req.into_request(2)); + let expected = json!({ + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": { + "name": "conversationsList", + "arguments": { + "limit": 50, + "cursor": "abc" + } + } + }); + assert_eq!(observed, expected); + } + + #[test] + fn serialize_tool_call_request_params_conversation_stream() { + let req = ToolCallRequestParams::ConversationStream(ConversationStreamArgs { + conversation_id: ConversationId(uuid!("67e55044-10b1-426f-9247-bb680e5fe0c8")), + }); + + let observed = to_val(&req.into_request(2)); + let expected = json!({ + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": { + "name": "conversationStream", + "arguments": { + "conversation_id": "67e55044-10b1-426f-9247-bb680e5fe0c8" + } + } + }); + assert_eq!(observed, expected); + } + + // ----- Message inputs / sources ----- + + #[test] + fn serialize_message_input_image_file_id_auto_detail() { + let item = MessageInputItem::Image { + source: ImageSource::FileId { + file_id: "file_123".into(), + }, + detail: Some(ImageDetail::Auto), + }; + let observed = to_val(&item); + let expected = json!({ + "type": "image", + "file_id": "file_123", + "detail": "auto" + }); + assert_eq!(observed, expected); + } + + #[test] + fn serialize_message_input_file_url_and_id_variants() { + let url = MessageInputItem::File { + source: FileSource::Url { + file_url: "https://example.com/a.pdf".into(), + }, + }; + let id = MessageInputItem::File { + source: FileSource::Id { + file_id: "file_456".into(), + }, + }; + assert_eq!( + to_val(&url), + json!({"type":"file","file_url":"https://example.com/a.pdf"}) + ); + assert_eq!(to_val(&id), json!({"type":"file","file_id":"file_456"})); + } + + #[test] + fn serialize_message_input_image_url_without_detail() { + let item = MessageInputItem::Image { + source: ImageSource::ImageUrl { + image_url: "https://example.com/x.png".into(), + }, + detail: None, + }; + let observed = to_val(&item); + let expected = json!({ + "type": "image", + "image_url": "https://example.com/x.png" + }); + assert_eq!(observed, expected); + } + + // ----- Responses ----- + + #[test] + fn response_success_conversation_create_full_schema() { + let env = ToolCallResponse { + request_id: RequestId::Integer(1), + is_error: None, + result: Some(ToolCallResponseResult::ConversationCreate( + ConversationCreateResult { + conversation_id: ConversationId(uuid!("d0f6ecbe-84a2-41c1-b23d-b20473b25eab")), + model: "o3".into(), + }, + )), + }; + let observed = to_val(&env); + let expected = json!({ + "requestId": 1, + "result": { + "conversation_id": "d0f6ecbe-84a2-41c1-b23d-b20473b25eab", + "model": "o3" + } + }); + assert_eq!( + observed, expected, + "response (ConversationCreate) must match" + ); + } + + #[test] + fn response_success_conversation_stream_empty_result_object() { + let env = ToolCallResponse { + request_id: RequestId::Integer(2), + is_error: None, + result: Some(ToolCallResponseResult::ConversationStream( + ConversationStreamResult {}, + )), + }; + let observed = to_val(&env); + let expected = json!({ + "requestId": 2, + "result": {} + }); + assert_eq!( + observed, expected, + "response (ConversationStream) must have empty object result" + ); + } + + #[test] + fn response_success_send_message_accepted_full_schema() { + let env = ToolCallResponse { + request_id: RequestId::Integer(3), + is_error: None, + result: Some(ToolCallResponseResult::ConversationSendMessage( + ConversationSendMessageResult { success: true }, + )), + }; + let observed = to_val(&env); + let expected = json!({ + "requestId": 3, + "result": { "success": true } + }); + assert_eq!( + observed, expected, + "response (ConversationSendMessageAccepted) must match" + ); + } + + #[test] + fn response_success_conversations_list_with_next_cursor_full_schema() { + let env = ToolCallResponse { + request_id: RequestId::Integer(4), + is_error: None, + result: Some(ToolCallResponseResult::ConversationsList( + ConversationsListResult { + conversations: vec![ConversationSummary { + conversation_id: ConversationId(uuid!( + "67e55044-10b1-426f-9247-bb680e5fe0c8" + )), + title: "Refactor config loader".into(), + }], + next_cursor: Some("next123".into()), + }, + )), + }; + let observed = to_val(&env); + let expected = json!({ + "requestId": 4, + "result": { + "conversations": [ + { + "conversation_id": "67e55044-10b1-426f-9247-bb680e5fe0c8", + "title": "Refactor config loader" + } + ], + "next_cursor": "next123" + } + }); + assert_eq!( + observed, expected, + "response (ConversationsList with cursor) must match" + ); + } + + #[test] + fn response_error_only_is_error_and_request_id_string() { + let env = ToolCallResponse { + request_id: RequestId::Integer(4), + is_error: Some(true), + result: None, + }; + let observed = to_val(&env); + let expected = json!({ + "requestId": 4, + "isError": true + }); + assert_eq!( + observed, expected, + "error response must omit `result` and include `isError`" + ); + } + + // ----- Notifications ----- + + #[test] + fn serialize_notification_initial_state_minimal() { + let params = InitialStateNotificationParams { + meta: Some(NotificationMeta { + conversation_id: Some(ConversationId(uuid!( + "67e55044-10b1-426f-9247-bb680e5fe0c8" + ))), + request_id: Some(RequestId::Integer(44)), + }), + initial_state: InitialStatePayload { + events: vec![ + CodexEventNotificationParams { + meta: None, + msg: EventMsg::TaskStarted, + }, + CodexEventNotificationParams { + meta: None, + msg: EventMsg::AgentMessageDelta( + codex_core::protocol::AgentMessageDeltaEvent { + delta: "Loading...".into(), + }, + ), + }, + ], + }, + }; + + let observed = to_val(&ServerNotification::InitialState(params.clone())); + let expected = json!({ + "method": "notifications/initial_state", + "params": { + "_meta": { + "conversationId": "67e55044-10b1-426f-9247-bb680e5fe0c8", + "requestId": 44 + }, + "initial_state": { + "events": [ + { "msg": { "type": "task_started" } }, + { "msg": { "type": "agent_message_delta", "delta": "Loading..." } } + ] + } + } + }); + assert_eq!(observed, expected); + } + + #[test] + fn serialize_notification_initial_state_omits_empty_events_full_json() { + let params = InitialStateNotificationParams { + meta: None, + initial_state: InitialStatePayload { events: vec![] }, + }; + + let observed = to_val(&ServerNotification::InitialState(params)); + let expected = json!({ + "method": "notifications/initial_state", + "params": { + "initial_state": { "events": [] } + } + }); + assert_eq!(observed, expected); + } + + #[test] + fn serialize_notification_stream_disconnected() { + let params = StreamDisconnectedNotificationParams { + meta: Some(NotificationMeta { + conversation_id: Some(ConversationId(uuid!( + "67e55044-10b1-426f-9247-bb680e5fe0c8" + ))), + request_id: None, + }), + reason: "New stream() took over".into(), + }; + + let observed = to_val(&ServerNotification::StreamDisconnected(params)); + let expected = json!({ + "method": "notifications/stream_disconnected", + "params": { + "_meta": { "conversationId": "67e55044-10b1-426f-9247-bb680e5fe0c8" }, + "reason": "New stream() took over" + } + }); + assert_eq!(observed, expected); + } + + #[test] + fn serialize_notification_codex_event_uses_eventmsg_type_in_method() { + let params = CodexEventNotificationParams { + meta: Some(NotificationMeta { + conversation_id: Some(ConversationId(uuid!( + "67e55044-10b1-426f-9247-bb680e5fe0c8" + ))), + request_id: Some(RequestId::Integer(44)), + }), + msg: EventMsg::AgentMessage(codex_core::protocol::AgentMessageEvent { + message: "hi".into(), + }), + }; + + let observed = to_val(&ServerNotification::CodexEvent(params)); + let expected = json!({ + "method": "notifications/agent_message", + "params": { + "_meta": { + "conversationId": "67e55044-10b1-426f-9247-bb680e5fe0c8", + "requestId": 44 + }, + "msg": { "type": "agent_message", "message": "hi" } + } + }); + assert_eq!(observed, expected); + } + + #[test] + fn serialize_notification_codex_event_task_started_full_json() { + let params = CodexEventNotificationParams { + meta: Some(NotificationMeta { + conversation_id: Some(ConversationId(uuid!( + "67e55044-10b1-426f-9247-bb680e5fe0c8" + ))), + request_id: Some(RequestId::Integer(7)), + }), + msg: EventMsg::TaskStarted, + }; + + let observed = to_val(&ServerNotification::CodexEvent(params)); + let expected = json!({ + "method": "notifications/task_started", + "params": { + "_meta": { + "conversationId": "67e55044-10b1-426f-9247-bb680e5fe0c8", + "requestId": 7 + }, + "msg": { "type": "task_started" } + } + }); + assert_eq!(observed, expected); + } + + #[test] + fn serialize_notification_codex_event_agent_message_delta_full_json() { + let params = CodexEventNotificationParams { + meta: None, + msg: EventMsg::AgentMessageDelta(codex_core::protocol::AgentMessageDeltaEvent { + delta: "stream...".into(), + }), + }; + + let observed = to_val(&ServerNotification::CodexEvent(params)); + let expected = json!({ + "method": "notifications/agent_message_delta", + "params": { + "msg": { "type": "agent_message_delta", "delta": "stream..." } + } + }); + assert_eq!(observed, expected); + } + + #[test] + fn serialize_notification_codex_event_agent_message_full_json() { + let params = CodexEventNotificationParams { + meta: Some(NotificationMeta { + conversation_id: Some(ConversationId(uuid!( + "67e55044-10b1-426f-9247-bb680e5fe0c8" + ))), + request_id: Some(RequestId::Integer(44)), + }), + msg: EventMsg::AgentMessage(codex_core::protocol::AgentMessageEvent { + message: "hi".into(), + }), + }; + + let observed = to_val(&ServerNotification::CodexEvent(params)); + let expected = json!({ + "method": "notifications/agent_message", + "params": { + "_meta": { + "conversationId": "67e55044-10b1-426f-9247-bb680e5fe0c8", + "requestId": 44 + }, + "msg": { "type": "agent_message", "message": "hi" } + } + }); + assert_eq!(observed, expected); + } + + #[test] + fn serialize_notification_codex_event_agent_reasoning_full_json() { + let params = CodexEventNotificationParams { + meta: None, + msg: EventMsg::AgentReasoning(codex_core::protocol::AgentReasoningEvent { + text: "thinking…".into(), + }), + }; + + let observed = to_val(&ServerNotification::CodexEvent(params)); + let expected = json!({ + "method": "notifications/agent_reasoning", + "params": { + "msg": { "type": "agent_reasoning", "text": "thinking…" } + } + }); + assert_eq!(observed, expected); + } + + #[test] + fn serialize_notification_codex_event_token_count_full_json() { + let usage = codex_core::protocol::TokenUsage { + input_tokens: 10, + cached_input_tokens: Some(2), + output_tokens: 5, + reasoning_output_tokens: Some(1), + total_tokens: 16, + }; + let params = CodexEventNotificationParams { + meta: None, + msg: EventMsg::TokenCount(usage), + }; + + let observed = to_val(&ServerNotification::CodexEvent(params)); + let expected = json!({ + "method": "notifications/token_count", + "params": { + "msg": { + "type": "token_count", + "input_tokens": 10, + "cached_input_tokens": 2, + "output_tokens": 5, + "reasoning_output_tokens": 1, + "total_tokens": 16 + } + } + }); + assert_eq!(observed, expected); + } + + #[test] + fn serialize_notification_codex_event_session_configured_full_json() { + let params = CodexEventNotificationParams { + meta: Some(NotificationMeta { + conversation_id: Some(ConversationId(uuid!( + "67e55044-10b1-426f-9247-bb680e5fe0c8" + ))), + request_id: None, + }), + msg: EventMsg::SessionConfigured(codex_core::protocol::SessionConfiguredEvent { + session_id: uuid!("67e55044-10b1-426f-9247-bb680e5fe0c8"), + model: "codex-mini-latest".into(), + history_log_id: 42, + history_entry_count: 3, + }), + }; + + let observed = to_val(&ServerNotification::CodexEvent(params)); + let expected = json!({ + "method": "notifications/session_configured", + "params": { + "_meta": { "conversationId": "67e55044-10b1-426f-9247-bb680e5fe0c8" }, + "msg": { + "type": "session_configured", + "session_id": "67e55044-10b1-426f-9247-bb680e5fe0c8", + "model": "codex-mini-latest", + "history_log_id": 42, + "history_entry_count": 3 + } + } + }); + assert_eq!(observed, expected); + } + + #[test] + fn serialize_notification_codex_event_exec_command_begin_full_json() { + let params = CodexEventNotificationParams { + meta: None, + msg: EventMsg::ExecCommandBegin(codex_core::protocol::ExecCommandBeginEvent { + call_id: "c1".into(), + command: vec!["bash".into(), "-lc".into(), "echo hi".into()], + cwd: std::path::PathBuf::from("/work"), + }), + }; + + let observed = to_val(&ServerNotification::CodexEvent(params)); + let expected = json!({ + "method": "notifications/exec_command_begin", + "params": { + "msg": { + "type": "exec_command_begin", + "call_id": "c1", + "command": ["bash", "-lc", "echo hi"], + "cwd": "/work" + } + } + }); + assert_eq!(observed, expected); + } + + #[test] + fn serialize_notification_codex_event_mcp_tool_call_begin_full_json() { + let params = CodexEventNotificationParams { + meta: None, + msg: EventMsg::McpToolCallBegin(codex_core::protocol::McpToolCallBeginEvent { + call_id: "m1".into(), + server: "calc".into(), + tool: "add".into(), + arguments: Some(json!({"a":1,"b":2})), + }), + }; + + let observed = to_val(&ServerNotification::CodexEvent(params)); + let expected = json!({ + "method": "notifications/mcp_tool_call_begin", + "params": { + "msg": { + "type": "mcp_tool_call_begin", + "call_id": "m1", + "server": "calc", + "tool": "add", + "arguments": { "a": 1, "b": 2 } + } + } + }); + assert_eq!(observed, expected); + } + + #[test] + fn serialize_notification_codex_event_patch_apply_end_full_json() { + let params = CodexEventNotificationParams { + meta: None, + msg: EventMsg::PatchApplyEnd(codex_core::protocol::PatchApplyEndEvent { + call_id: "p1".into(), + stdout: "ok".into(), + stderr: "".into(), + success: true, + }), + }; + + let observed = to_val(&ServerNotification::CodexEvent(params)); + let expected = json!({ + "method": "notifications/patch_apply_end", + "params": { + "msg": { + "type": "patch_apply_end", + "call_id": "p1", + "stdout": "ok", + "stderr": "", + "success": true + } + } + }); + assert_eq!(observed, expected); + } + + // ----- Cancelled notifications ----- + + #[test] + fn serialize_notification_cancelled_with_reason_full_json() { + let params = CancelNotificationParams { + request_id: RequestId::String("r-123".into()), + reason: Some("user_cancelled".into()), + }; + + let observed = to_val(&ClientNotification::Cancelled(params)); + let expected = json!({ + "method": "notifications/cancelled", + "params": { + "requestId": "r-123", + "reason": "user_cancelled" + } + }); + assert_eq!(observed, expected); + } + + #[test] + fn serialize_notification_cancelled_without_reason_full_json() { + let params = CancelNotificationParams { + request_id: RequestId::Integer(77), + reason: None, + }; + + let observed = to_val(&ClientNotification::Cancelled(params)); + + // Check exact structure: reason must be omitted. + assert_eq!(observed["method"], "notifications/cancelled"); + assert_eq!(observed["params"]["requestId"], 77); + assert!( + observed["params"].get("reason").is_none(), + "reason must be omitted when None" + ); + } +} From 347c81ad0049103c84e0aa2c0d7e2988db18218a Mon Sep 17 00:00:00 2001 From: Jeremy Rose <172423086+nornagon-openai@users.noreply.github.com> Date: Wed, 30 Jul 2025 10:05:40 -0700 Subject: [PATCH 003/100] remove conversation history widget (#1727) this widget is no longer used. --- codex-rs/core/src/mcp_tool_call.rs | 15 +- codex-rs/core/src/protocol.rs | 16 +- .../src/event_processor_with_human_output.rs | 94 ++-- codex-rs/tui/src/app.rs | 26 -- codex-rs/tui/src/app_event.rs | 4 - codex-rs/tui/src/cell_widget.rs | 20 - codex-rs/tui/src/chatwidget.rs | 135 +++--- .../tui/src/conversation_history_widget.rs | 429 ------------------ codex-rs/tui/src/history_cell.rs | 319 +++---------- codex-rs/tui/src/insert_history.rs | 2 +- codex-rs/tui/src/lib.rs | 3 - codex-rs/tui/src/scroll_event_helper.rs | 77 ---- codex-rs/tui/src/text_block.rs | 18 - 13 files changed, 164 insertions(+), 994 deletions(-) delete mode 100644 codex-rs/tui/src/cell_widget.rs delete mode 100644 codex-rs/tui/src/conversation_history_widget.rs delete mode 100644 codex-rs/tui/src/scroll_event_helper.rs diff --git a/codex-rs/core/src/mcp_tool_call.rs b/codex-rs/core/src/mcp_tool_call.rs index 61a51a0e7a..e92d7e8481 100644 --- a/codex-rs/core/src/mcp_tool_call.rs +++ b/codex-rs/core/src/mcp_tool_call.rs @@ -1,4 +1,5 @@ use std::time::Duration; +use std::time::Instant; use tracing::error; @@ -7,6 +8,7 @@ use crate::models::FunctionCallOutputPayload; use crate::models::ResponseInputItem; use crate::protocol::Event; use crate::protocol::EventMsg; +use crate::protocol::McpInvocation; use crate::protocol::McpToolCallBeginEvent; use crate::protocol::McpToolCallEndEvent; @@ -41,21 +43,28 @@ pub(crate) async fn handle_mcp_tool_call( } }; - let tool_call_begin_event = EventMsg::McpToolCallBegin(McpToolCallBeginEvent { - call_id: call_id.clone(), + let invocation = McpInvocation { server: server.clone(), tool: tool_name.clone(), arguments: arguments_value.clone(), + }; + + let tool_call_begin_event = EventMsg::McpToolCallBegin(McpToolCallBeginEvent { + call_id: call_id.clone(), + invocation: invocation.clone(), }); notify_mcp_tool_call_event(sess, sub_id, tool_call_begin_event).await; + let start = Instant::now(); // Perform the tool call. let result = sess - .call_tool(&server, &tool_name, arguments_value, timeout) + .call_tool(&server, &tool_name, arguments_value.clone(), timeout) .await .map_err(|e| format!("tool call error: {e}")); let tool_call_end_event = EventMsg::McpToolCallEnd(McpToolCallEndEvent { call_id: call_id.clone(), + invocation, + duration: start.elapsed(), result: result.clone(), }); diff --git a/codex-rs/core/src/protocol.rs b/codex-rs/core/src/protocol.rs index 041a8c58ce..bc922eb0e2 100644 --- a/codex-rs/core/src/protocol.rs +++ b/codex-rs/core/src/protocol.rs @@ -7,7 +7,8 @@ use std::collections::HashMap; use std::fmt; use std::path::Path; use std::path::PathBuf; -use std::str::FromStr; // Added for FinalOutput Display implementation +use std::str::FromStr; +use std::time::Duration; use mcp_types::CallToolResult; use serde::Deserialize; @@ -414,9 +415,7 @@ pub struct AgentReasoningDeltaEvent { } #[derive(Debug, Clone, Deserialize, Serialize)] -pub struct McpToolCallBeginEvent { - /// Identifier so this can be paired with the McpToolCallEnd event. - pub call_id: String, +pub struct McpInvocation { /// Name of the MCP server as defined in the config. pub server: String, /// Name of the tool as given by the MCP server. @@ -425,10 +424,19 @@ pub struct McpToolCallBeginEvent { pub arguments: Option, } +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct McpToolCallBeginEvent { + /// Identifier so this can be paired with the McpToolCallEnd event. + pub call_id: String, + pub invocation: McpInvocation, +} + #[derive(Debug, Clone, Deserialize, Serialize)] pub struct McpToolCallEndEvent { /// Identifier for the corresponding McpToolCallBegin that finished. pub call_id: String, + pub invocation: McpInvocation, + pub duration: Duration, /// Result of the tool call. Note this could be an error. pub result: Result, } diff --git a/codex-rs/exec/src/event_processor_with_human_output.rs b/codex-rs/exec/src/event_processor_with_human_output.rs index 6c3f73f0ca..54604d538b 100644 --- a/codex-rs/exec/src/event_processor_with_human_output.rs +++ b/codex-rs/exec/src/event_processor_with_human_output.rs @@ -1,3 +1,4 @@ +use codex_common::elapsed::format_duration; use codex_common::elapsed::format_elapsed; use codex_core::config::Config; use codex_core::plan_tool::UpdatePlanArgs; @@ -11,6 +12,7 @@ use codex_core::protocol::EventMsg; use codex_core::protocol::ExecCommandBeginEvent; use codex_core::protocol::ExecCommandEndEvent; use codex_core::protocol::FileChange; +use codex_core::protocol::McpInvocation; use codex_core::protocol::McpToolCallBeginEvent; use codex_core::protocol::McpToolCallEndEvent; use codex_core::protocol::PatchApplyBeginEvent; @@ -38,11 +40,6 @@ pub(crate) struct EventProcessorWithHumanOutput { call_id_to_command: HashMap, call_id_to_patch: HashMap, - /// Tracks in-flight MCP tool calls so we can calculate duration and print - /// a concise summary when the corresponding `McpToolCallEnd` event is - /// received. - call_id_to_tool_call: HashMap, - // To ensure that --color=never is respected, ANSI escapes _must_ be added // using .style() with one of these fields. If you need a new style, add a // new field here. @@ -70,7 +67,6 @@ impl EventProcessorWithHumanOutput { ) -> Self { let call_id_to_command = HashMap::new(); let call_id_to_patch = HashMap::new(); - let call_id_to_tool_call = HashMap::new(); if with_ansi { Self { @@ -83,7 +79,6 @@ impl EventProcessorWithHumanOutput { red: Style::new().red(), green: Style::new().green(), cyan: Style::new().cyan(), - call_id_to_tool_call, show_agent_reasoning: !config.hide_agent_reasoning, answer_started: false, reasoning_started: false, @@ -100,7 +95,6 @@ impl EventProcessorWithHumanOutput { red: Style::new(), green: Style::new(), cyan: Style::new(), - call_id_to_tool_call, show_agent_reasoning: !config.hide_agent_reasoning, answer_started: false, reasoning_started: false, @@ -115,14 +109,6 @@ struct ExecCommandBegin { start_time: Instant, } -/// Metadata captured when an `McpToolCallBegin` event is received. -struct McpToolCallBegin { - /// Formatted invocation string, e.g. `server.tool({"city":"sf"})`. - invocation: String, - /// Timestamp when the call started so we can compute duration later. - start_time: Instant, -} - struct PatchApplyBegin { start_time: Instant, auto_approved: bool, @@ -292,63 +278,33 @@ impl EventProcessor for EventProcessorWithHumanOutput { println!("{}", truncated_output.style(self.dimmed)); } EventMsg::McpToolCallBegin(McpToolCallBeginEvent { - call_id, - server, - tool, - arguments, + call_id: _, + invocation, }) => { - // Build fully-qualified tool name: server.tool - let fq_tool_name = format!("{server}.{tool}"); - - // Format arguments as compact JSON so they fit on one line. - let args_str = arguments - .as_ref() - .map(|v: &serde_json::Value| { - serde_json::to_string(v).unwrap_or_else(|_| v.to_string()) - }) - .unwrap_or_default(); - - let invocation = if args_str.is_empty() { - format!("{fq_tool_name}()") - } else { - format!("{fq_tool_name}({args_str})") - }; - - self.call_id_to_tool_call.insert( - call_id.clone(), - McpToolCallBegin { - invocation: invocation.clone(), - start_time: Instant::now(), - }, - ); - ts_println!( self, "{} {}", "tool".style(self.magenta), - invocation.style(self.bold), + format_mcp_invocation(&invocation).style(self.bold), ); } EventMsg::McpToolCallEnd(tool_call_end_event) => { let is_success = tool_call_end_event.is_success(); - let McpToolCallEndEvent { call_id, result } = tool_call_end_event; - // Retrieve start time and invocation for duration calculation and labeling. - let info = self.call_id_to_tool_call.remove(&call_id); - - let (duration, invocation) = if let Some(McpToolCallBegin { + let McpToolCallEndEvent { + call_id: _, + result, invocation, - start_time, - .. - }) = info - { - (format!(" in {}", format_elapsed(start_time)), invocation) - } else { - (String::new(), format!("tool('{call_id}')")) - }; + duration, + } = tool_call_end_event; + + let duration = format!(" in {}", format_duration(duration)); let status_str = if is_success { "success" } else { "failed" }; let title_style = if is_success { self.green } else { self.red }; - let title = format!("{invocation} {status_str}{duration}:"); + let title = format!( + "{} {status_str}{duration}:", + format_mcp_invocation(&invocation) + ); ts_println!(self, "{}", title.style(title_style)); @@ -544,3 +500,21 @@ fn format_file_change(change: &FileChange) -> &'static str { } => "M", } } + +fn format_mcp_invocation(invocation: &McpInvocation) -> String { + // Build fully-qualified tool name: server.tool + let fq_tool_name = format!("{}.{}", invocation.server, invocation.tool); + + // Format arguments as compact JSON so they fit on one line. + let args_str = invocation + .arguments + .as_ref() + .map(|v: &serde_json::Value| serde_json::to_string(v).unwrap_or_else(|_| v.to_string())) + .unwrap_or_default(); + + if args_str.is_empty() { + format!("{fq_tool_name}()") + } else { + format!("{fq_tool_name}({args_str})") + } +} diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index b671075ba8..6823a83a50 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -5,7 +5,6 @@ use crate::file_search::FileSearchManager; use crate::get_git_diff::get_git_diff; use crate::git_warning_screen::GitWarningOutcome; use crate::git_warning_screen::GitWarningScreen; -use crate::scroll_event_helper::ScrollEventHelper; use crate::slash_command::SlashCommand; use crate::tui; use codex_core::config::Config; @@ -13,8 +12,6 @@ use codex_core::protocol::Event; use color_eyre::eyre::Result; use crossterm::event::KeyCode; use crossterm::event::KeyEvent; -use crossterm::event::MouseEvent; -use crossterm::event::MouseEventKind; use std::path::PathBuf; use std::sync::Arc; use std::sync::atomic::AtomicBool; @@ -77,7 +74,6 @@ impl App<'_> { let (app_event_tx, app_event_rx) = channel(); let app_event_tx = AppEventSender::new(app_event_tx); let pending_redraw = Arc::new(AtomicBool::new(false)); - let scroll_event_helper = ScrollEventHelper::new(app_event_tx.clone()); // Spawn a dedicated thread for reading the crossterm event loop and // re-publishing the events as AppEvents, as appropriate. @@ -100,18 +96,6 @@ impl App<'_> { crossterm::event::Event::Resize(_, _) => { app_event_tx.send(AppEvent::RequestRedraw); } - crossterm::event::Event::Mouse(MouseEvent { - kind: MouseEventKind::ScrollUp, - .. - }) => { - scroll_event_helper.scroll_up(); - } - crossterm::event::Event::Mouse(MouseEvent { - kind: MouseEventKind::ScrollDown, - .. - }) => { - scroll_event_helper.scroll_down(); - } crossterm::event::Event::Paste(pasted) => { // Many terminals convert newlines to \r when // pasting, e.g. [iTerm2][]. But [tui-textarea @@ -259,9 +243,6 @@ impl App<'_> { } }; } - AppEvent::Scroll(scroll_delta) => { - self.dispatch_scroll_event(scroll_delta); - } AppEvent::Paste(text) => { self.dispatch_paste_event(text); } @@ -392,13 +373,6 @@ impl App<'_> { } } - fn dispatch_scroll_event(&mut self, scroll_delta: i32) { - match &mut self.app_state { - AppState::Chat { widget } => widget.handle_scroll_delta(scroll_delta), - AppState::GitWarning { .. } => {} - } - } - fn dispatch_codex_event(&mut self, event: Event) { match &mut self.app_state { AppState::Chat { widget } => widget.handle_codex_event(event), diff --git a/codex-rs/tui/src/app_event.rs b/codex-rs/tui/src/app_event.rs index a1f304fe42..77a600d304 100644 --- a/codex-rs/tui/src/app_event.rs +++ b/codex-rs/tui/src/app_event.rs @@ -20,10 +20,6 @@ pub(crate) enum AppEvent { /// Text pasted from the terminal clipboard. Paste(String), - /// Scroll event with a value representing the "scroll delta" as the net - /// scroll up/down events within a short time window. - Scroll(i32), - /// Request to exit the application gracefully. ExitRequest, diff --git a/codex-rs/tui/src/cell_widget.rs b/codex-rs/tui/src/cell_widget.rs deleted file mode 100644 index 8acdc0553a..0000000000 --- a/codex-rs/tui/src/cell_widget.rs +++ /dev/null @@ -1,20 +0,0 @@ -use ratatui::prelude::*; - -/// Trait implemented by every type that can live inside the conversation -/// history list. It provides two primitives that the parent scroll-view -/// needs: how *tall* the widget is at a given width and how to render an -/// arbitrary contiguous *window* of that widget. -/// -/// The `first_visible_line` argument to [`render_window`] allows partial -/// rendering when the top of the widget is scrolled off-screen. The caller -/// guarantees that `first_visible_line + area.height as usize` never exceeds -/// the total height previously returned by [`height`]. -pub(crate) trait CellWidget { - /// Total height measured in wrapped terminal lines when drawn with the - /// given *content* width (no scrollbar column included). - fn height(&self, width: u16) -> usize; - - /// Render a *window* that starts `first_visible_line` lines below the top - /// of the widget. The window’s size is given by `area`. - fn render_window(&self, first_visible_line: usize, area: Rect, buf: &mut Buffer); -} diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index a896ae37bc..fde6978634 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -1,5 +1,6 @@ use std::path::PathBuf; use std::sync::Arc; +use std::time::Duration; use codex_core::codex_wrapper::CodexConversation; use codex_core::codex_wrapper::init_codex; @@ -36,8 +37,9 @@ use crate::bottom_pane::BottomPane; use crate::bottom_pane::BottomPaneParams; use crate::bottom_pane::CancellationEvent; use crate::bottom_pane::InputResult; -use crate::conversation_history_widget::ConversationHistoryWidget; use crate::exec_command::strip_bash_lc_and_escape; +use crate::history_cell::CommandOutput; +use crate::history_cell::HistoryCell; use crate::history_cell::PatchEventType; use crate::user_approval_widget::ApprovalRequest; use codex_file_search::FileMatch; @@ -45,7 +47,6 @@ use codex_file_search::FileMatch; pub(crate) struct ChatWidget<'a> { app_event_tx: AppEventSender, codex_op_tx: UnboundedSender, - conversation_history: ConversationHistoryWidget, bottom_pane: BottomPane<'a>, config: Config, initial_user_message: Option, @@ -127,7 +128,6 @@ impl ChatWidget<'_> { Self { app_event_tx: app_event_tx.clone(), codex_op_tx, - conversation_history: ConversationHistoryWidget::new(), bottom_pane: BottomPane::new(BottomPaneParams { app_event_tx, has_input_focus: true, @@ -158,11 +158,9 @@ impl ChatWidget<'_> { self.bottom_pane.handle_paste(text); } - /// Emits the last entry's plain lines from conversation_history, if any. - fn emit_last_history_entry(&mut self) { - if let Some(lines) = self.conversation_history.last_entry_plain_lines() { - self.app_event_tx.send(AppEvent::InsertHistory(lines)); - } + fn add_to_history(&mut self, cell: HistoryCell) { + self.app_event_tx + .send(AppEvent::InsertHistory(cell.plain_lines())); } fn submit_user_message(&mut self, user_message: UserMessage) { @@ -198,28 +196,18 @@ impl ChatWidget<'_> { // Only show text portion in conversation history for now. if !text.is_empty() { - self.conversation_history.add_user_message(text.clone()); - self.emit_last_history_entry(); + self.add_to_history(HistoryCell::new_user_prompt(text.clone())); } - self.conversation_history.scroll_to_bottom(); } pub(crate) fn handle_codex_event(&mut self, event: Event) { let Event { id, msg } = event; match msg { EventMsg::SessionConfigured(event) => { - // Record session information at the top of the conversation. - self.conversation_history - .add_session_info(&self.config, event.clone()); - // Immediately surface the session banner / settings summary in - // scrollback so the user can review configuration (model, - // sandbox, approvals, etc.) before interacting. - self.emit_last_history_entry(); - - // Forward history metadata to the bottom pane so the chat - // composer can navigate through past messages. self.bottom_pane .set_history_metadata(event.history_log_id, event.history_entry_count); + // Record session information at the top of the conversation. + self.add_to_history(HistoryCell::new_session_info(&self.config, event, true)); if let Some(user_message) = self.initial_user_message.take() { // If the user provided an initial message, add it to the @@ -241,9 +229,7 @@ impl ChatWidget<'_> { message }; if !full.is_empty() { - self.conversation_history - .add_agent_message(&self.config, full); - self.emit_last_history_entry(); + self.add_to_history(HistoryCell::new_agent_message(&self.config, full)); } self.request_redraw(); } @@ -270,9 +256,7 @@ impl ChatWidget<'_> { text }; if !full.is_empty() { - self.conversation_history - .add_agent_reasoning(&self.config, full); - self.emit_last_history_entry(); + self.add_to_history(HistoryCell::new_agent_reasoning(&self.config, full)); } self.request_redraw(); } @@ -293,8 +277,7 @@ impl ChatWidget<'_> { .set_token_usage(self.token_usage.clone(), self.config.model_context_window); } EventMsg::Error(ErrorEvent { message }) => { - self.conversation_history.add_error(message.clone()); - self.emit_last_history_entry(); + self.add_to_history(HistoryCell::new_error_event(message.clone())); self.bottom_pane.set_task_running(false); } EventMsg::ExecApprovalRequest(ExecApprovalRequestEvent { @@ -313,9 +296,7 @@ impl ChatWidget<'_> { .map(|r| format!("\n{r}")) .unwrap_or_default() ); - self.conversation_history.add_background_event(text); - self.emit_last_history_entry(); - self.conversation_history.scroll_to_bottom(); + self.add_to_history(HistoryCell::new_background_event(text)); let request = ApprovalRequest::Exec { id, @@ -343,11 +324,10 @@ impl ChatWidget<'_> { // prompt before they have seen *what* is being requested. // ------------------------------------------------------------------ - self.conversation_history - .add_patch_event(PatchEventType::ApprovalRequest, changes); - self.emit_last_history_entry(); - - self.conversation_history.scroll_to_bottom(); + self.add_to_history(HistoryCell::new_patch_event( + PatchEventType::ApprovalRequest, + changes, + )); // Now surface the approval request in the BottomPane as before. let request = ApprovalRequest::ApplyPatch { @@ -359,13 +339,11 @@ impl ChatWidget<'_> { self.request_redraw(); } EventMsg::ExecCommandBegin(ExecCommandBeginEvent { - call_id, + call_id: _, command, cwd: _, }) => { - self.conversation_history - .add_active_exec_command(call_id, command); - self.emit_last_history_entry(); + self.add_to_history(HistoryCell::new_active_exec_command(command)); self.request_redraw(); } EventMsg::PatchApplyBegin(PatchApplyBeginEvent { @@ -375,12 +353,10 @@ impl ChatWidget<'_> { }) => { // Even when a patch is auto‑approved we still display the // summary so the user can follow along. - self.conversation_history - .add_patch_event(PatchEventType::ApplyBegin { auto_approved }, changes); - self.emit_last_history_entry(); - if !auto_approved { - self.conversation_history.scroll_to_bottom(); - } + self.add_to_history(HistoryCell::new_patch_event( + PatchEventType::ApplyBegin { auto_approved }, + changes, + )); self.request_redraw(); } EventMsg::ExecCommandEnd(ExecCommandEndEvent { @@ -389,27 +365,39 @@ impl ChatWidget<'_> { stdout, stderr, }) => { - self.conversation_history - .record_completed_exec_command(call_id, stdout, stderr, exit_code); - self.request_redraw(); + self.add_to_history(HistoryCell::new_completed_exec_command( + call_id, + CommandOutput { + exit_code, + stdout, + stderr, + duration: Duration::from_secs(0), + }, + )); } EventMsg::McpToolCallBegin(McpToolCallBeginEvent { - call_id, - server, - tool, - arguments, + call_id: _, + invocation, }) => { - self.conversation_history - .add_active_mcp_tool_call(call_id, server, tool, arguments); - self.emit_last_history_entry(); + self.add_to_history(HistoryCell::new_active_mcp_tool_call(invocation)); self.request_redraw(); } - EventMsg::McpToolCallEnd(mcp_tool_call_end_event) => { - let success = mcp_tool_call_end_event.is_success(); - let McpToolCallEndEvent { call_id, result } = mcp_tool_call_end_event; - self.conversation_history - .record_completed_mcp_tool_call(call_id, success, result); - self.request_redraw(); + EventMsg::McpToolCallEnd(McpToolCallEndEvent { + call_id: _, + duration, + invocation, + result, + }) => { + self.add_to_history(HistoryCell::new_completed_mcp_tool_call( + 80, + invocation, + duration, + result + .as_ref() + .map(|r| r.is_error.unwrap_or(false)) + .unwrap_or(false), + result, + )); } EventMsg::GetHistoryEntryResponse(event) => { let codex_core::protocol::GetHistoryEntryResponseEvent { @@ -426,9 +414,7 @@ impl ChatWidget<'_> { self.app_event_tx.send(AppEvent::ExitRequest); } event => { - self.conversation_history - .add_background_event(format!("{event:?}")); - self.emit_last_history_entry(); + self.add_to_history(HistoryCell::new_background_event(format!("{event:?}"))); self.request_redraw(); } } @@ -445,22 +431,7 @@ impl ChatWidget<'_> { } pub(crate) fn add_diff_output(&mut self, diff_output: String) { - self.conversation_history - .add_diff_output(diff_output.clone()); - self.emit_last_history_entry(); - self.request_redraw(); - } - - pub(crate) fn handle_scroll_delta(&mut self, scroll_delta: i32) { - // If the user is trying to scroll exactly one line, we let them, but - // otherwise we assume they are trying to scroll in larger increments. - let magnified_scroll_delta = if scroll_delta == 1 { - 1 - } else { - // Play with this: perhaps it should be non-linear? - scroll_delta * 2 - }; - self.conversation_history.scroll(magnified_scroll_delta); + self.add_to_history(HistoryCell::new_diff_output(diff_output.clone())); self.request_redraw(); } diff --git a/codex-rs/tui/src/conversation_history_widget.rs b/codex-rs/tui/src/conversation_history_widget.rs deleted file mode 100644 index dede0caf5f..0000000000 --- a/codex-rs/tui/src/conversation_history_widget.rs +++ /dev/null @@ -1,429 +0,0 @@ -use crate::cell_widget::CellWidget; -use crate::history_cell::CommandOutput; -use crate::history_cell::HistoryCell; -use crate::history_cell::PatchEventType; -use codex_core::config::Config; -use codex_core::protocol::FileChange; -use codex_core::protocol::SessionConfiguredEvent; -use ratatui::prelude::*; -use ratatui::style::Style; -use ratatui::widgets::*; -use serde_json::Value as JsonValue; -use std::cell::Cell as StdCell; -use std::cell::Cell; -use std::collections::HashMap; -use std::path::PathBuf; - -/// A single history entry plus its cached wrapped-line count. -struct Entry { - cell: HistoryCell, - line_count: Cell, -} - -pub struct ConversationHistoryWidget { - entries: Vec, - /// The width (in terminal cells/columns) that [`Entry::line_count`] was - /// computed for. When the available width changes we recompute counts. - cached_width: StdCell, - scroll_position: usize, - /// Number of lines the last time render_ref() was called - num_rendered_lines: StdCell, - /// The height of the viewport last time render_ref() was called - last_viewport_height: StdCell, - has_input_focus: bool, -} - -impl ConversationHistoryWidget { - pub fn new() -> Self { - Self { - entries: Vec::new(), - cached_width: StdCell::new(0), - scroll_position: usize::MAX, - num_rendered_lines: StdCell::new(0), - last_viewport_height: StdCell::new(0), - has_input_focus: false, - } - } - - /// Negative delta scrolls up; positive delta scrolls down. - pub(crate) fn scroll(&mut self, delta: i32) { - match delta.cmp(&0) { - std::cmp::Ordering::Less => self.scroll_up(-delta as u32), - std::cmp::Ordering::Greater => self.scroll_down(delta as u32), - std::cmp::Ordering::Equal => {} - } - } - - fn scroll_up(&mut self, num_lines: u32) { - // If a user is scrolling up from the "stick to bottom" mode, we need to - // map this to a specific scroll position so we can calculate the delta. - // This requires us to care about how tall the screen is. - if self.scroll_position == usize::MAX { - self.scroll_position = self - .num_rendered_lines - .get() - .saturating_sub(self.last_viewport_height.get()); - } - - self.scroll_position = self.scroll_position.saturating_sub(num_lines as usize); - } - - fn scroll_down(&mut self, num_lines: u32) { - // If we're already pinned to the bottom there's nothing to do. - if self.scroll_position == usize::MAX { - return; - } - - let viewport_height = self.last_viewport_height.get().max(1); - let num_rendered_lines = self.num_rendered_lines.get(); - - // Compute the maximum explicit scroll offset that still shows a full - // viewport. This mirrors the calculation in `scroll_page_down()` and - // in the render path. - let max_scroll = num_rendered_lines.saturating_sub(viewport_height); - - let new_pos = self.scroll_position.saturating_add(num_lines as usize); - - if new_pos >= max_scroll { - // Reached (or passed) the bottom – switch to stick‑to‑bottom mode - // so that additional output keeps the view pinned automatically. - self.scroll_position = usize::MAX; - } else { - self.scroll_position = new_pos; - } - } - - pub fn scroll_to_bottom(&mut self) { - self.scroll_position = usize::MAX; - } - - /// Note `model` could differ from `config.model` if the agent decided to - /// use a different model than the one requested by the user. - pub fn add_session_info(&mut self, config: &Config, event: SessionConfiguredEvent) { - // In practice, SessionConfiguredEvent should always be the first entry - // in the history, but it is possible that an error could be sent - // before the session info. - let has_welcome_message = self - .entries - .iter() - .any(|entry| matches!(entry.cell, HistoryCell::WelcomeMessage { .. })); - self.add_to_history(HistoryCell::new_session_info( - config, - event, - !has_welcome_message, - )); - } - - pub fn add_user_message(&mut self, message: String) { - self.add_to_history(HistoryCell::new_user_prompt(message)); - } - - pub fn add_agent_message(&mut self, config: &Config, message: String) { - self.add_to_history(HistoryCell::new_agent_message(config, message)); - } - - pub fn add_agent_reasoning(&mut self, config: &Config, text: String) { - self.add_to_history(HistoryCell::new_agent_reasoning(config, text)); - } - - pub fn add_background_event(&mut self, message: String) { - self.add_to_history(HistoryCell::new_background_event(message)); - } - - pub fn add_diff_output(&mut self, diff_output: String) { - self.add_to_history(HistoryCell::new_diff_output(diff_output)); - } - - pub fn add_error(&mut self, message: String) { - self.add_to_history(HistoryCell::new_error_event(message)); - } - - /// Add a pending patch entry (before user approval). - pub fn add_patch_event( - &mut self, - event_type: PatchEventType, - changes: HashMap, - ) { - self.add_to_history(HistoryCell::new_patch_event(event_type, changes)); - } - - pub fn add_active_exec_command(&mut self, call_id: String, command: Vec) { - self.add_to_history(HistoryCell::new_active_exec_command(call_id, command)); - } - - pub fn add_active_mcp_tool_call( - &mut self, - call_id: String, - server: String, - tool: String, - arguments: Option, - ) { - self.add_to_history(HistoryCell::new_active_mcp_tool_call( - call_id, server, tool, arguments, - )); - } - - fn add_to_history(&mut self, cell: HistoryCell) { - let width = self.cached_width.get(); - let count = if width > 0 { cell.height(width) } else { 0 }; - - self.entries.push(Entry { - cell, - line_count: Cell::new(count), - }); - } - - /// Return the lines for the most recently appended entry (if any) so the - /// parent widget can surface them via the new scrollback insertion path. - pub(crate) fn last_entry_plain_lines(&self) -> Option>> { - self.entries.last().map(|e| e.cell.plain_lines()) - } - - pub fn record_completed_exec_command( - &mut self, - call_id: String, - stdout: String, - stderr: String, - exit_code: i32, - ) { - let width = self.cached_width.get(); - for entry in self.entries.iter_mut() { - let cell = &mut entry.cell; - if let HistoryCell::ActiveExecCommand { - call_id: history_id, - command, - start, - .. - } = cell - { - if &call_id == history_id { - *cell = HistoryCell::new_completed_exec_command( - command.clone(), - CommandOutput { - exit_code, - stdout, - stderr, - duration: start.elapsed(), - }, - ); - - // Update cached line count. - if width > 0 { - entry.line_count.set(cell.height(width)); - } - break; - } - } - } - } - - pub fn record_completed_mcp_tool_call( - &mut self, - call_id: String, - success: bool, - result: Result, - ) { - let width = self.cached_width.get(); - for entry in self.entries.iter_mut() { - if let HistoryCell::ActiveMcpToolCall { - call_id: history_id, - invocation, - start, - .. - } = &entry.cell - { - if &call_id == history_id { - let completed = HistoryCell::new_completed_mcp_tool_call( - width, - invocation.clone(), - *start, - success, - result, - ); - entry.cell = completed; - - if width > 0 { - entry.line_count.set(entry.cell.height(width)); - } - - break; - } - } - } - } -} - -impl WidgetRef for ConversationHistoryWidget { - fn render_ref(&self, area: Rect, buf: &mut Buffer) { - let (title, border_style) = if self.has_input_focus { - ( - "Messages (↑/↓ or j/k = line, b/space = page)", - Style::default().fg(Color::LightYellow), - ) - } else { - ("Messages (tab to focus)", Style::default().dim()) - }; - - let block = Block::default() - .title(title) - .borders(Borders::ALL) - .border_type(BorderType::Rounded) - .border_style(border_style); - - // Compute the inner area that will be available for the list after - // the surrounding `Block` is drawn. - let inner = block.inner(area); - let viewport_height = inner.height as usize; - - // Cache (and if necessary recalculate) the wrapped line counts for every - // [`HistoryCell`] so that our scrolling math accounts for text - // wrapping. We always reserve one column on the right-hand side for the - // scrollbar so that the content never renders "under" the scrollbar. - let effective_width = inner.width.saturating_sub(1); - - if effective_width == 0 { - return; // Nothing to draw – avoid division by zero. - } - - // Recompute cache if the effective width changed. - let num_lines: usize = if self.cached_width.get() != effective_width { - self.cached_width.set(effective_width); - - let mut num_lines: usize = 0; - for entry in &self.entries { - let count = entry.cell.height(effective_width); - num_lines += count; - entry.line_count.set(count); - } - num_lines - } else { - self.entries.iter().map(|e| e.line_count.get()).sum() - }; - - // Determine the scroll position. Note the existing value of - // `self.scroll_position` could exceed the maximum scroll offset if the - // user made the window wider since the last render. - let max_scroll = num_lines.saturating_sub(viewport_height); - let scroll_pos = if self.scroll_position == usize::MAX { - max_scroll - } else { - self.scroll_position.min(max_scroll) - }; - - // ------------------------------------------------------------------ - // Render order: - // 1. Clear full widget area (avoid artifacts from prior frame). - // 2. Draw the surrounding Block (border and title). - // 3. Render *each* visible HistoryCell into its own sub-Rect while - // respecting partial visibility at the top and bottom. - // 4. Draw the scrollbar track / thumb in the reserved column. - // ------------------------------------------------------------------ - - // Clear entire widget area first. - Clear.render(area, buf); - - // Draw border + title. - block.render(area, buf); - - // ------------------------------------------------------------------ - // Calculate which cells are visible for the current scroll position - // and paint them one by one. - // ------------------------------------------------------------------ - - let mut y_cursor = inner.y; // first line inside viewport - let mut remaining_height = inner.height as usize; - let mut lines_to_skip = scroll_pos; // number of wrapped lines to skip (above viewport) - - for entry in &self.entries { - let cell_height = entry.line_count.get(); - - // Completely above viewport? Skip whole cell. - if lines_to_skip >= cell_height { - lines_to_skip -= cell_height; - continue; - } - - // Determine how much of this cell is visible. - let visible_height = (cell_height - lines_to_skip).min(remaining_height); - - if visible_height == 0 { - break; // no space left - } - - let cell_rect = Rect { - x: inner.x, - y: y_cursor, - width: effective_width, - height: visible_height as u16, - }; - - entry.cell.render_window(lines_to_skip, cell_rect, buf); - - // Advance cursor inside viewport. - y_cursor += visible_height as u16; - remaining_height -= visible_height; - - // After the first (possibly partially skipped) cell, we no longer - // need to skip lines at the top. - lines_to_skip = 0; - - if remaining_height == 0 { - break; // viewport filled - } - } - - // Always render a scrollbar *track* so the reserved column is filled. - let overflow = num_lines.saturating_sub(viewport_height); - - let mut scroll_state = ScrollbarState::default() - // The Scrollbar widget expects the *content* height minus the - // viewport height. When there is no overflow we still provide 0 - // so that the widget renders only the track without a thumb. - .content_length(overflow) - .position(scroll_pos); - - { - // Choose a thumb color that stands out only when this pane has focus so that the - // user's attention is naturally drawn to the active viewport. When unfocused we show - // a low-contrast thumb so the scrollbar fades into the background without becoming - // invisible. - let thumb_style = if self.has_input_focus { - Style::reset().fg(Color::LightYellow) - } else { - Style::reset().fg(Color::Gray) - }; - - // By default the Scrollbar widget inherits any style that was - // present in the underlying buffer cells. That means if a colored - // line happens to be underneath the scrollbar, the track (and - // potentially the thumb) adopt that color. Explicitly setting the - // track/thumb styles ensures we always draw the scrollbar with a - // consistent palette regardless of what content is behind it. - StatefulWidget::render( - Scrollbar::new(ScrollbarOrientation::VerticalRight) - .begin_symbol(Some("↑")) - .end_symbol(Some("↓")) - .begin_style(Style::reset().fg(Color::DarkGray)) - .end_style(Style::reset().fg(Color::DarkGray)) - .thumb_symbol("█") - .thumb_style(thumb_style) - .track_symbol(Some("│")) - .track_style(Style::reset().fg(Color::DarkGray)), - inner, - buf, - &mut scroll_state, - ); - } - - // Update auxiliary stats that the scroll handlers rely on. - self.num_rendered_lines.set(num_lines); - self.last_viewport_height.set(viewport_height); - } -} - -/// Common [`Wrap`] configuration used for both measurement and rendering so -/// they stay in sync. -#[inline] -pub(crate) const fn wrap_cfg() -> ratatui::widgets::Wrap { - ratatui::widgets::Wrap { trim: false } -} diff --git a/codex-rs/tui/src/history_cell.rs b/codex-rs/tui/src/history_cell.rs index ab657163ad..04279a01f8 100644 --- a/codex-rs/tui/src/history_cell.rs +++ b/codex-rs/tui/src/history_cell.rs @@ -1,4 +1,3 @@ -use crate::cell_widget::CellWidget; use crate::exec_command::escape_command; use crate::markdown::append_markdown; use crate::text_block::TextBlock; @@ -11,11 +10,10 @@ use codex_core::WireApi; use codex_core::config::Config; use codex_core::model_supports_reasoning_summaries; use codex_core::protocol::FileChange; +use codex_core::protocol::McpInvocation; use codex_core::protocol::SessionConfiguredEvent; use image::DynamicImage; -use image::GenericImageView; use image::ImageReader; -use lazy_static::lazy_static; use mcp_types::EmbeddedResourceResource; use mcp_types::ResourceLink; use ratatui::prelude::*; @@ -24,14 +22,10 @@ use ratatui::style::Modifier; use ratatui::style::Style; use ratatui::text::Line as RtLine; use ratatui::text::Span as RtSpan; -use ratatui_image::Image as TuiImage; -use ratatui_image::Resize as ImgResize; -use ratatui_image::picker::ProtocolType; use std::collections::HashMap; use std::io::Cursor; use std::path::PathBuf; use std::time::Duration; -use std::time::Instant; use tracing::error; pub(crate) struct CommandOutput { @@ -46,6 +40,21 @@ pub(crate) enum PatchEventType { ApplyBegin { auto_approved: bool }, } +fn span_to_static(span: &Span) -> Span<'static> { + Span { + style: span.style, + content: std::borrow::Cow::Owned(span.content.clone().into_owned()), + } +} + +fn line_to_static(line: &Line) -> Line<'static> { + Line { + style: line.style, + alignment: line.alignment, + spans: line.spans.iter().map(span_to_static).collect(), + } +} + /// Represents an event to display in the conversation history. Returns its /// `Vec>` representation to make it easier to display in a /// scrollable list. @@ -63,25 +72,13 @@ pub(crate) enum HistoryCell { AgentReasoning { view: TextBlock }, /// An exec tool call that has not finished yet. - ActiveExecCommand { - call_id: String, - /// The shell command, escaped and formatted. - command: String, - start: Instant, - view: TextBlock, - }, + ActiveExecCommand { view: TextBlock }, /// Completed exec tool call. CompletedExecCommand { view: TextBlock }, /// An MCP tool call that has not finished yet. - ActiveMcpToolCall { - call_id: String, - /// Formatted line that shows the command name and arguments - invocation: Line<'static>, - start: Instant, - view: TextBlock, - }, + ActiveMcpToolCall { view: TextBlock }, /// Completed MCP tool call where we show the result serialized as JSON. CompletedMcpToolCall { view: TextBlock }, @@ -94,13 +91,7 @@ pub(crate) enum HistoryCell { // resized version avoids doing the potentially expensive rescale twice // because the scroll-view first calls `height()` for layouting and then // `render_window()` for painting. - CompletedMcpToolCallWithImageOutput { - image: DynamicImage, - /// Cached data derived from the current terminal width. The cache is - /// invalidated whenever the width changes (e.g. when the user - /// resizes the window). - render_cache: std::cell::RefCell>, - }, + CompletedMcpToolCallWithImageOutput { _image: DynamicImage }, /// Background event. BackgroundEvent { view: TextBlock }, @@ -140,7 +131,9 @@ impl HistoryCell { | HistoryCell::CompletedMcpToolCall { view } | HistoryCell::PendingPatch { view } | HistoryCell::ActiveExecCommand { view, .. } - | HistoryCell::ActiveMcpToolCall { view, .. } => view.lines.clone(), + | HistoryCell::ActiveMcpToolCall { view, .. } => { + view.lines.iter().map(line_to_static).collect() + } HistoryCell::CompletedMcpToolCallWithImageOutput { .. } => vec![ Line::from("tool result (image output omitted)"), Line::from(""), @@ -252,9 +245,8 @@ impl HistoryCell { } } - pub(crate) fn new_active_exec_command(call_id: String, command: Vec) -> Self { + pub(crate) fn new_active_exec_command(command: Vec) -> Self { let command_escaped = escape_command(&command); - let start = Instant::now(); let lines: Vec> = vec![ Line::from(vec!["command".magenta(), " running...".dim()]), @@ -263,9 +255,6 @@ impl HistoryCell { ]; HistoryCell::ActiveExecCommand { - call_id, - command: command_escaped, - start, view: TextBlock::new(lines), } } @@ -310,41 +299,15 @@ impl HistoryCell { } } - pub(crate) fn new_active_mcp_tool_call( - call_id: String, - server: String, - tool: String, - arguments: Option, - ) -> Self { - // Format the arguments as compact JSON so they roughly fit on one - // line. If there are no arguments we keep it empty so the invocation - // mirrors a function-style call. - let args_str = arguments - .as_ref() - .map(|v| { - // Use compact form to keep things short but readable. - serde_json::to_string(v).unwrap_or_else(|_| v.to_string()) - }) - .unwrap_or_default(); - - let invocation_spans = vec![ - Span::styled(server, Style::default().fg(Color::Blue)), - Span::raw("."), - Span::styled(tool, Style::default().fg(Color::Blue)), - Span::raw("("), - Span::styled(args_str, Style::default().fg(Color::Gray)), - Span::raw(")"), - ]; - let invocation = Line::from(invocation_spans); - - let start = Instant::now(); + pub(crate) fn new_active_mcp_tool_call(invocation: McpInvocation) -> Self { let title_line = Line::from(vec!["tool".magenta(), " running...".dim()]); - let lines: Vec> = vec![title_line, invocation.clone(), Line::from("")]; + let lines: Vec = vec![ + title_line, + format_mcp_invocation(invocation.clone()), + Line::from(""), + ]; HistoryCell::ActiveMcpToolCall { - call_id, - invocation, - start, view: TextBlock::new(lines), } } @@ -382,10 +345,7 @@ impl HistoryCell { } }; - Some(HistoryCell::CompletedMcpToolCallWithImageOutput { - image, - render_cache: std::cell::RefCell::new(None), - }) + Some(HistoryCell::CompletedMcpToolCallWithImageOutput { _image: image }) } else { None } @@ -396,8 +356,8 @@ impl HistoryCell { pub(crate) fn new_completed_mcp_tool_call( num_cols: u16, - invocation: Line<'static>, - start: Instant, + invocation: McpInvocation, + duration: Duration, success: bool, result: Result, ) -> Self { @@ -405,7 +365,7 @@ impl HistoryCell { return cell; } - let duration = format_duration(start.elapsed()); + let duration = format_duration(duration); let status_str = if success { "success" } else { "failed" }; let title_line = Line::from(vec![ "tool".magenta(), @@ -420,7 +380,7 @@ impl HistoryCell { let mut lines: Vec> = Vec::new(); lines.push(title_line); - lines.push(invocation); + lines.push(format_mcp_invocation(invocation)); match result { Ok(mcp_types::CallToolResult { content, .. }) => { @@ -581,85 +541,6 @@ impl HistoryCell { } } -// --------------------------------------------------------------------------- -// `CellWidget` implementation – most variants delegate to their internal -// `TextBlock`. Variants that need custom painting can add their own logic in -// the match arms. -// --------------------------------------------------------------------------- - -impl CellWidget for HistoryCell { - fn height(&self, width: u16) -> usize { - match self { - HistoryCell::WelcomeMessage { view } - | HistoryCell::UserPrompt { view } - | HistoryCell::AgentMessage { view } - | HistoryCell::AgentReasoning { view } - | HistoryCell::BackgroundEvent { view } - | HistoryCell::GitDiffOutput { view } - | HistoryCell::ErrorEvent { view } - | HistoryCell::SessionInfo { view } - | HistoryCell::CompletedExecCommand { view } - | HistoryCell::CompletedMcpToolCall { view } - | HistoryCell::PendingPatch { view } - | HistoryCell::ActiveExecCommand { view, .. } - | HistoryCell::ActiveMcpToolCall { view, .. } => view.height(width), - HistoryCell::CompletedMcpToolCallWithImageOutput { - image, - render_cache, - } => ensure_image_cache(image, width, render_cache), - } - } - - fn render_window(&self, first_visible_line: usize, area: Rect, buf: &mut Buffer) { - match self { - HistoryCell::WelcomeMessage { view } - | HistoryCell::UserPrompt { view } - | HistoryCell::AgentMessage { view } - | HistoryCell::AgentReasoning { view } - | HistoryCell::BackgroundEvent { view } - | HistoryCell::GitDiffOutput { view } - | HistoryCell::ErrorEvent { view } - | HistoryCell::SessionInfo { view } - | HistoryCell::CompletedExecCommand { view } - | HistoryCell::CompletedMcpToolCall { view } - | HistoryCell::PendingPatch { view } - | HistoryCell::ActiveExecCommand { view, .. } - | HistoryCell::ActiveMcpToolCall { view, .. } => { - view.render_window(first_visible_line, area, buf) - } - HistoryCell::CompletedMcpToolCallWithImageOutput { - image, - render_cache, - } => { - // Ensure we have a cached, resized copy that matches the current width. - // `height()` should have prepared the cache, but if something invalidated it - // (e.g. the first `render_window()` call happens *before* `height()` after a - // resize) we rebuild it here. - - let width_cells = area.width; - - // Ensure the cache is up-to-date and extract the scaled image. - let _ = ensure_image_cache(image, width_cells, render_cache); - - let Some(resized) = render_cache - .borrow() - .as_ref() - .map(|c| c.scaled_image.clone()) - else { - return; - }; - - let picker = &*TERMINAL_PICKER; - - if let Ok(protocol) = picker.new_protocol(resized, area, ImgResize::Fit(None)) { - let img_widget = TuiImage::new(&protocol); - img_widget.render(area, buf); - } - } - } - } -} - fn create_diff_summary(changes: HashMap) -> Vec { // Build a concise, human‑readable summary list similar to the // `git status` short format so the user can reason about the @@ -692,119 +573,23 @@ fn create_diff_summary(changes: HashMap) -> Vec { summaries } -// ------------------------------------- -// Helper types for image rendering -// ------------------------------------- +fn format_mcp_invocation<'a>(invocation: McpInvocation) -> Line<'a> { + let args_str = invocation + .arguments + .as_ref() + .map(|v| { + // Use compact form to keep things short but readable. + serde_json::to_string(v).unwrap_or_else(|_| v.to_string()) + }) + .unwrap_or_default(); -/// Cached information for rendering an image inside a conversation cell. -/// -/// The cache ties the resized image to a *specific* content width (in -/// terminal cells). Whenever the terminal is resized and the width changes -/// we need to re-compute the scaled variant so that it still fits the -/// available space. Keeping the resized copy around saves a costly rescale -/// between the back-to-back `height()` and `render_window()` calls that the -/// scroll-view performs while laying out the UI. -pub(crate) struct ImageRenderCache { - /// Width in *terminal cells* the cached image was generated for. - width_cells: u16, - /// Height in *terminal rows* that the conversation cell must occupy so - /// the whole image becomes visible. - height_rows: usize, - /// The resized image that fits the given width / height constraints. - scaled_image: DynamicImage, -} - -lazy_static! { - static ref TERMINAL_PICKER: ratatui_image::picker::Picker = { - use ratatui_image::picker::Picker; - use ratatui_image::picker::cap_parser::QueryStdioOptions; - - // Ask the terminal for capabilities and explicit font size. Request the - // Kitty *text-sizing protocol* as a fallback mechanism for terminals - // (like iTerm2) that do not reply to the standard CSI 16/18 queries. - match Picker::from_query_stdio_with_options(QueryStdioOptions { - text_sizing_protocol: true, - }) { - Ok(picker) => picker, - Err(err) => { - // Fall back to the conservative default that assumes ~8×16 px cells. - // Still better than breaking the build in a headless test run. - tracing::warn!("terminal capability query failed: {err:?}; using default font size"); - Picker::from_fontsize((8, 16)) - } - } - }; -} - -/// Resize `image` to fit into `width_cells`×10-rows keeping the original aspect -/// ratio. The function updates `render_cache` and returns the number of rows -/// (<= 10) the picture will occupy. -fn ensure_image_cache( - image: &DynamicImage, - width_cells: u16, - render_cache: &std::cell::RefCell>, -) -> usize { - if let Some(cache) = render_cache.borrow().as_ref() { - if cache.width_cells == width_cells { - return cache.height_rows; - } - } - - let picker = &*TERMINAL_PICKER; - let (char_w_px, char_h_px) = picker.font_size(); - - // Heuristic to compensate for Hi-DPI terminals (iTerm2 on Retina Mac) that - // report logical pixels (≈ 8×16) while the iTerm2 graphics protocol - // expects *device* pixels. Empirically the device-pixel-ratio is almost - // always 2 on macOS Retina panels. - let hidpi_scale = if picker.protocol_type() == ProtocolType::Iterm2 { - 2.0f64 - } else { - 1.0 - }; - - // The fallback Halfblocks protocol encodes two pixel rows per cell, so each - // terminal *row* represents only half the (possibly scaled) font height. - let effective_char_h_px: f64 = if picker.protocol_type() == ProtocolType::Halfblocks { - (char_h_px as f64) * hidpi_scale / 2.0 - } else { - (char_h_px as f64) * hidpi_scale - }; - - let char_w_px_f64 = (char_w_px as f64) * hidpi_scale; - - const MAX_ROWS: f64 = 10.0; - let max_height_px: f64 = effective_char_h_px * MAX_ROWS; - - let (orig_w_px, orig_h_px) = { - let (w, h) = image.dimensions(); - (w as f64, h as f64) - }; - - if orig_w_px == 0.0 || orig_h_px == 0.0 || width_cells == 0 { - *render_cache.borrow_mut() = None; - return 0; - } - - let max_w_px = char_w_px_f64 * width_cells as f64; - let scale_w = max_w_px / orig_w_px; - let scale_h = max_height_px / orig_h_px; - let scale = scale_w.min(scale_h).min(1.0); - - use image::imageops::FilterType; - let scaled_w_px = (orig_w_px * scale).round().max(1.0) as u32; - let scaled_h_px = (orig_h_px * scale).round().max(1.0) as u32; - - let scaled_image = image.resize(scaled_w_px, scaled_h_px, FilterType::Lanczos3); - - let height_rows = ((scaled_h_px as f64 / effective_char_h_px).ceil()) as usize; - - let new_cache = ImageRenderCache { - width_cells, - height_rows, - scaled_image, - }; - *render_cache.borrow_mut() = Some(new_cache); - - height_rows + let invocation_spans = vec![ + Span::styled(invocation.server.clone(), Style::default().fg(Color::Blue)), + Span::raw("."), + Span::styled(invocation.tool.clone(), Style::default().fg(Color::Blue)), + Span::raw("("), + Span::styled(args_str, Style::default().fg(Color::Gray)), + Span::raw(")"), + ]; + Line::from(invocation_spans) } diff --git a/codex-rs/tui/src/insert_history.rs b/codex-rs/tui/src/insert_history.rs index 1e8b1f5392..32d0b4b297 100644 --- a/codex-rs/tui/src/insert_history.rs +++ b/codex-rs/tui/src/insert_history.rs @@ -21,7 +21,7 @@ use ratatui::text::Line; use ratatui::text::Span; /// Insert `lines` above the viewport. -pub(crate) fn insert_history_lines(terminal: &mut tui::Tui, lines: Vec>) { +pub(crate) fn insert_history_lines(terminal: &mut tui::Tui, lines: Vec) { let screen_size = terminal.backend().size().unwrap_or(Size::new(0, 0)); let mut area = terminal.get_frame().area(); diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index 6c6c662154..7bc041a554 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -24,11 +24,9 @@ mod app; mod app_event; mod app_event_sender; mod bottom_pane; -mod cell_widget; mod chatwidget; mod citation_regex; mod cli; -mod conversation_history_widget; mod exec_command; mod file_search; mod get_git_diff; @@ -37,7 +35,6 @@ mod history_cell; mod insert_history; mod log_layer; mod markdown; -mod scroll_event_helper; mod slash_command; mod status_indicator_widget; mod text_block; diff --git a/codex-rs/tui/src/scroll_event_helper.rs b/codex-rs/tui/src/scroll_event_helper.rs deleted file mode 100644 index ad3ae37e0d..0000000000 --- a/codex-rs/tui/src/scroll_event_helper.rs +++ /dev/null @@ -1,77 +0,0 @@ -use std::sync::Arc; -use std::sync::atomic::AtomicBool; -use std::sync::atomic::AtomicI32; -use std::sync::atomic::Ordering; - -use tokio::runtime::Handle; -use tokio::time::Duration; -use tokio::time::sleep; - -use crate::app_event::AppEvent; -use crate::app_event_sender::AppEventSender; - -pub(crate) struct ScrollEventHelper { - app_event_tx: AppEventSender, - scroll_delta: Arc, - timer_scheduled: Arc, - runtime: Handle, -} - -/// How long to wait after the first scroll event before sending the -/// accumulated scroll delta to the main thread. -const DEBOUNCE_WINDOW: Duration = Duration::from_millis(100); - -/// Utility to debounce scroll events so we can determine the **magnitude** of -/// each scroll burst by accumulating individual wheel events over a short -/// window. The debounce timer now runs on Tokio so we avoid spinning up a new -/// operating-system thread for every burst. -impl ScrollEventHelper { - pub(crate) fn new(app_event_tx: AppEventSender) -> Self { - Self { - app_event_tx, - scroll_delta: Arc::new(AtomicI32::new(0)), - timer_scheduled: Arc::new(AtomicBool::new(false)), - runtime: Handle::current(), - } - } - - pub(crate) fn scroll_up(&self) { - self.scroll_delta.fetch_sub(1, Ordering::Relaxed); - self.schedule_notification(); - } - - pub(crate) fn scroll_down(&self) { - self.scroll_delta.fetch_add(1, Ordering::Relaxed); - self.schedule_notification(); - } - - /// Starts a one-shot timer **only once** per burst of wheel events. - fn schedule_notification(&self) { - // If the timer is already scheduled, do nothing. - if self - .timer_scheduled - .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) - .is_err() - { - return; - } - - // Otherwise, schedule a new timer. - let tx = self.app_event_tx.clone(); - let delta = Arc::clone(&self.scroll_delta); - let timer_flag = Arc::clone(&self.timer_scheduled); - - // Use self.runtime instead of tokio::spawn() because the calling thread - // in app.rs is not part of the Tokio runtime: it is a plain OS thread. - self.runtime.spawn(async move { - sleep(DEBOUNCE_WINDOW).await; - - let accumulated = delta.swap(0, Ordering::SeqCst); - if accumulated != 0 { - tx.send(AppEvent::Scroll(accumulated)); - } - - timer_flag.store(false, Ordering::SeqCst); - }); - } -} diff --git a/codex-rs/tui/src/text_block.rs b/codex-rs/tui/src/text_block.rs index 2c68d90f11..33f326b83d 100644 --- a/codex-rs/tui/src/text_block.rs +++ b/codex-rs/tui/src/text_block.rs @@ -1,4 +1,3 @@ -use crate::cell_widget::CellWidget; use ratatui::prelude::*; /// A simple widget that just displays a list of `Line`s via a `Paragraph`. @@ -13,20 +12,3 @@ impl TextBlock { Self { lines } } } - -impl CellWidget for TextBlock { - fn height(&self, width: u16) -> usize { - // Use the same wrapping configuration as ConversationHistoryWidget so - // measurement stays in sync with rendering. - ratatui::widgets::Paragraph::new(self.lines.clone()) - .wrap(crate::conversation_history_widget::wrap_cfg()) - .line_count(width) - } - - fn render_window(&self, first_visible_line: usize, area: Rect, buf: &mut Buffer) { - ratatui::widgets::Paragraph::new(self.lines.clone()) - .wrap(crate::conversation_history_widget::wrap_cfg()) - .scroll((first_visible_line as u16, 0)) - .render(area, buf); - } -} From 93341797c4172e948fdba93c8878c3d286c36e51 Mon Sep 17 00:00:00 2001 From: aibrahim-oai Date: Wed, 30 Jul 2025 11:32:38 -0700 Subject: [PATCH 004/100] fix ci (#1739) I think this commit broke the CI because it changed the `McpToolCallBeginEvent` type: https://github.com/openai/codex/commit/347c81ad0049103c84e0aa2c0d7e2988db18218a --- codex-rs/mcp-server/src/mcp_protocol.rs | 42 ++++++++++++++----------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/codex-rs/mcp-server/src/mcp_protocol.rs b/codex-rs/mcp-server/src/mcp_protocol.rs index 05eb0a258a..933c145b48 100644 --- a/codex-rs/mcp-server/src/mcp_protocol.rs +++ b/codex-rs/mcp-server/src/mcp_protocol.rs @@ -219,7 +219,7 @@ pub struct ConversationSummary { pub enum ServerNotification { InitialState(InitialStateNotificationParams), StreamDisconnected(StreamDisconnectedNotificationParams), - CodexEvent(CodexEventNotificationParams), + CodexEvent(Box), } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -304,6 +304,8 @@ pub enum ClientNotification { #[allow(clippy::unwrap_used)] mod tests { use super::*; + use codex_core::protocol::McpInvocation; + use codex_core::protocol::McpToolCallBeginEvent; use pretty_assertions::assert_eq; use serde::Serialize; use serde_json::Value; @@ -727,7 +729,7 @@ mod tests { }), }; - let observed = to_val(&ServerNotification::CodexEvent(params)); + let observed = to_val(&ServerNotification::CodexEvent(Box::new(params))); let expected = json!({ "method": "notifications/agent_message", "params": { @@ -753,7 +755,7 @@ mod tests { msg: EventMsg::TaskStarted, }; - let observed = to_val(&ServerNotification::CodexEvent(params)); + let observed = to_val(&ServerNotification::CodexEvent(Box::new(params))); let expected = json!({ "method": "notifications/task_started", "params": { @@ -776,7 +778,7 @@ mod tests { }), }; - let observed = to_val(&ServerNotification::CodexEvent(params)); + let observed = to_val(&ServerNotification::CodexEvent(Box::new(params))); let expected = json!({ "method": "notifications/agent_message_delta", "params": { @@ -800,7 +802,7 @@ mod tests { }), }; - let observed = to_val(&ServerNotification::CodexEvent(params)); + let observed = to_val(&ServerNotification::CodexEvent(Box::new(params))); let expected = json!({ "method": "notifications/agent_message", "params": { @@ -823,7 +825,7 @@ mod tests { }), }; - let observed = to_val(&ServerNotification::CodexEvent(params)); + let observed = to_val(&ServerNotification::CodexEvent(Box::new(params))); let expected = json!({ "method": "notifications/agent_reasoning", "params": { @@ -847,7 +849,7 @@ mod tests { msg: EventMsg::TokenCount(usage), }; - let observed = to_val(&ServerNotification::CodexEvent(params)); + let observed = to_val(&ServerNotification::CodexEvent(Box::new(params))); let expected = json!({ "method": "notifications/token_count", "params": { @@ -881,7 +883,7 @@ mod tests { }), }; - let observed = to_val(&ServerNotification::CodexEvent(params)); + let observed = to_val(&ServerNotification::CodexEvent(Box::new(params))); let expected = json!({ "method": "notifications/session_configured", "params": { @@ -909,7 +911,7 @@ mod tests { }), }; - let observed = to_val(&ServerNotification::CodexEvent(params)); + let observed = to_val(&ServerNotification::CodexEvent(Box::new(params))); let expected = json!({ "method": "notifications/exec_command_begin", "params": { @@ -928,24 +930,28 @@ mod tests { fn serialize_notification_codex_event_mcp_tool_call_begin_full_json() { let params = CodexEventNotificationParams { meta: None, - msg: EventMsg::McpToolCallBegin(codex_core::protocol::McpToolCallBeginEvent { + msg: EventMsg::McpToolCallBegin(McpToolCallBeginEvent { call_id: "m1".into(), - server: "calc".into(), - tool: "add".into(), - arguments: Some(json!({"a":1,"b":2})), + invocation: McpInvocation { + server: "calc".into(), + tool: "add".into(), + arguments: Some(json!({"a":1,"b":2})), + }, }), }; - let observed = to_val(&ServerNotification::CodexEvent(params)); + let observed = to_val(&ServerNotification::CodexEvent(Box::new(params))); let expected = json!({ "method": "notifications/mcp_tool_call_begin", "params": { "msg": { "type": "mcp_tool_call_begin", "call_id": "m1", - "server": "calc", - "tool": "add", - "arguments": { "a": 1, "b": 2 } + "invocation": { + "server": "calc", + "tool": "add", + "arguments": { "a": 1, "b": 2 } + } } } }); @@ -964,7 +970,7 @@ mod tests { }), }; - let observed = to_val(&ServerNotification::CodexEvent(params)); + let observed = to_val(&ServerNotification::CodexEvent(Box::new(params))); let expected = json!({ "method": "notifications/patch_apply_end", "params": { From ea01a5ffe213b2e75783fd5c1a685b3a74688437 Mon Sep 17 00:00:00 2001 From: pakrym-oai Date: Wed, 30 Jul 2025 12:40:15 -0700 Subject: [PATCH 005/100] Add support for a separate chatgpt auth endpoint (#1712) Adds a `CodexAuth` type that encapsulates information about available auth modes and logic for refreshing the token. Changes `Responses` API to send requests to different endpoints based on the auth type. Updates login_with_chatgpt to support API-less mode and skip the key exchange. --- codex-rs/Cargo.lock | 2 + codex-rs/chatgpt/src/chatgpt_client.rs | 6 +- codex-rs/chatgpt/src/chatgpt_token.rs | 7 +- codex-rs/cli/src/proto.rs | 4 +- codex-rs/config.md | 3 + codex-rs/core/Cargo.toml | 2 + codex-rs/core/src/client.rs | 47 ++++- codex-rs/core/src/codex.rs | 11 +- codex-rs/core/src/codex_wrapper.rs | 4 +- codex-rs/core/src/config.rs | 4 +- codex-rs/core/src/lib.rs | 2 +- codex-rs/core/src/model_provider_info.rs | 129 +++++++----- codex-rs/core/src/openai_api_key.rs | 24 --- codex-rs/core/tests/client.rs | 180 ++++++++++++---- codex-rs/core/tests/live_agent.rs | 2 +- codex-rs/core/tests/stream_no_completed.rs | 12 +- codex-rs/login/src/lib.rs | 232 ++++++++++++++++----- codex-rs/login/src/login_with_chatgpt.py | 150 +++++++------ codex-rs/tui/src/lib.rs | 39 ++-- 19 files changed, 575 insertions(+), 285 deletions(-) delete mode 100644 codex-rs/core/src/openai_api_key.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 9abce0c3db..120050c227 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -673,7 +673,9 @@ dependencies = [ "async-channel", "base64 0.22.1", "bytes", + "chrono", "codex-apply-patch", + "codex-login", "codex-mcp-client", "core_test_support", "dirs", diff --git a/codex-rs/chatgpt/src/chatgpt_client.rs b/codex-rs/chatgpt/src/chatgpt_client.rs index 4c4cb4c4c3..907783bb81 100644 --- a/codex-rs/chatgpt/src/chatgpt_client.rs +++ b/codex-rs/chatgpt/src/chatgpt_client.rs @@ -21,10 +21,14 @@ pub(crate) async fn chatgpt_get_request( let token = get_chatgpt_token_data().ok_or_else(|| anyhow::anyhow!("ChatGPT token not available"))?; + let account_id = token.account_id.ok_or_else(|| { + anyhow::anyhow!("ChatGPT account ID not available, please re-run `codex login`") + }); + let response = client .get(&url) .bearer_auth(&token.access_token) - .header("chatgpt-account-id", &token.account_id) + .header("chatgpt-account-id", account_id?) .header("Content-Type", "application/json") .header("User-Agent", "codex-cli") .send() diff --git a/codex-rs/chatgpt/src/chatgpt_token.rs b/codex-rs/chatgpt/src/chatgpt_token.rs index adf9a6ba96..55ebc22a08 100644 --- a/codex-rs/chatgpt/src/chatgpt_token.rs +++ b/codex-rs/chatgpt/src/chatgpt_token.rs @@ -18,7 +18,10 @@ pub fn set_chatgpt_token_data(value: TokenData) { /// Initialize the ChatGPT token from auth.json file pub async fn init_chatgpt_token_from_auth(codex_home: &Path) -> std::io::Result<()> { - let auth_json = codex_login::try_read_auth_json(codex_home).await?; - set_chatgpt_token_data(auth_json.tokens.clone()); + let auth = codex_login::load_auth(codex_home)?; + if let Some(auth) = auth { + let token_data = auth.get_token_data().await?; + set_chatgpt_token_data(token_data); + } Ok(()) } diff --git a/codex-rs/cli/src/proto.rs b/codex-rs/cli/src/proto.rs index 64b292d50b..291e1680f1 100644 --- a/codex-rs/cli/src/proto.rs +++ b/codex-rs/cli/src/proto.rs @@ -9,6 +9,7 @@ use codex_core::config::Config; use codex_core::config::ConfigOverrides; use codex_core::protocol::Submission; use codex_core::util::notify_on_sigint; +use codex_login::load_auth; use tokio::io::AsyncBufReadExt; use tokio::io::BufReader; use tracing::error; @@ -35,8 +36,9 @@ pub async fn run_main(opts: ProtoCli) -> anyhow::Result<()> { .map_err(anyhow::Error::msg)?; let config = Config::load_with_cli_overrides(overrides_vec, ConfigOverrides::default())?; + let auth = load_auth(&config.codex_home)?; let ctrl_c = notify_on_sigint(); - let CodexSpawnOk { codex, .. } = Codex::spawn(config, ctrl_c.clone()).await?; + let CodexSpawnOk { codex, .. } = Codex::spawn(config, auth, ctrl_c.clone()).await?; let codex = Arc::new(codex); // Task that reads JSON lines from stdin and forwards to Submission Queue diff --git a/codex-rs/config.md b/codex-rs/config.md index c45d81180d..1a407a239b 100644 --- a/codex-rs/config.md +++ b/codex-rs/config.md @@ -110,12 +110,15 @@ stream_idle_timeout_ms = 300000 # 5m idle timeout ``` #### request_max_retries + How many times Codex will retry a failed HTTP request to the model provider. Defaults to `4`. #### stream_max_retries + Number of times Codex will attempt to reconnect when a streaming response is interrupted. Defaults to `10`. #### stream_idle_timeout_ms + How long Codex will wait for activity on a streaming response before treating the connection as lost. Defaults to `300_000` (5 minutes). ## model_provider diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index 2e0489c9b5..5ebb5ef63d 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -17,6 +17,8 @@ base64 = "0.22" bytes = "1.10.1" codex-apply-patch = { path = "../apply-patch" } codex-mcp-client = { path = "../mcp-client" } +chrono = { version = "0.4", features = ["serde"] } +codex-login = { path = "../login" } dirs = "6" env-flags = "0.1.1" eventsource-stream = "0.2.3" diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index aa31b67ecb..72104da254 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -3,6 +3,8 @@ use std::path::Path; use std::time::Duration; use bytes::Bytes; +use codex_login::AuthMode; +use codex_login::CodexAuth; use eventsource_stream::Eventsource; use futures::prelude::*; use reqwest::StatusCode; @@ -28,6 +30,7 @@ use crate::config::Config; use crate::config_types::ReasoningEffort as ReasoningEffortConfig; use crate::config_types::ReasoningSummary as ReasoningSummaryConfig; use crate::error::CodexErr; +use crate::error::EnvVarError; use crate::error::Result; use crate::flags::CODEX_RS_SSE_FIXTURE; use crate::model_provider_info::ModelProviderInfo; @@ -41,6 +44,7 @@ use std::sync::Arc; #[derive(Clone)] pub struct ModelClient { config: Arc, + auth: Option, client: reqwest::Client, provider: ModelProviderInfo, session_id: Uuid, @@ -51,6 +55,7 @@ pub struct ModelClient { impl ModelClient { pub fn new( config: Arc, + auth: Option, provider: ModelProviderInfo, effort: ReasoningEffortConfig, summary: ReasoningSummaryConfig, @@ -58,6 +63,7 @@ impl ModelClient { ) -> Self { Self { config, + auth, client: reqwest::Client::new(), provider, session_id, @@ -115,6 +121,25 @@ impl ModelClient { return stream_from_fixture(path, self.provider.clone()).await; } + let auth = self.auth.as_ref().ok_or_else(|| { + CodexErr::EnvVar(EnvVarError { + var: "OPENAI_API_KEY".to_string(), + instructions: Some("Create an API key (https://platform.openai.com) and export it as an environment variable.".to_string()), + }) + })?; + + let store = prompt.store && auth.mode != AuthMode::ChatGPT; + + let base_url = match self.provider.base_url.clone() { + Some(url) => url, + None => match auth.mode { + AuthMode::ChatGPT => "https://chatgpt.com/backend-api/codex".to_string(), + AuthMode::ApiKey => "https://api.openai.com/v1".to_string(), + }, + }; + + let token = auth.get_token().await?; + let full_instructions = prompt.get_full_instructions(&self.config.model); let tools_json = create_tools_json_for_responses_api( prompt, @@ -125,7 +150,7 @@ impl ModelClient { // Request encrypted COT if we are not storing responses, // otherwise reasoning items will be referenced by ID - let include = if !prompt.store && reasoning.is_some() { + let include: Vec = if !store && reasoning.is_some() { vec!["reasoning.encrypted_content".to_string()] } else { vec![] @@ -139,8 +164,7 @@ impl ModelClient { tool_choice: "auto", parallel_tool_calls: false, reasoning, - store: prompt.store, - // TODO: make this configurable + store, stream: true, include, }; @@ -153,17 +177,21 @@ impl ModelClient { let mut attempt = 0; let max_retries = self.provider.request_max_retries(); + loop { attempt += 1; let req_builder = self - .provider - .create_request_builder(&self.client)? + .client + .post(format!("{base_url}/responses")) .header("OpenAI-Beta", "responses=experimental") .header("session_id", self.session_id.to_string()) + .bearer_auth(&token) .header(reqwest::header::ACCEPT, "text/event-stream") .json(&payload); + let req_builder = self.provider.apply_http_headers(req_builder); + let res = req_builder.send().await; if let Ok(resp) = &res { trace!( @@ -572,7 +600,7 @@ mod tests { let provider = ModelProviderInfo { name: "test".to_string(), - base_url: "https://test.com".to_string(), + base_url: Some("https://test.com".to_string()), env_key: Some("TEST_API_KEY".to_string()), env_key_instructions: None, wire_api: WireApi::Responses, @@ -582,6 +610,7 @@ mod tests { request_max_retries: Some(0), stream_max_retries: Some(0), stream_idle_timeout_ms: Some(1000), + requires_auth: false, }; let events = collect_events( @@ -631,7 +660,7 @@ mod tests { let sse1 = format!("event: response.output_item.done\ndata: {item1}\n\n"); let provider = ModelProviderInfo { name: "test".to_string(), - base_url: "https://test.com".to_string(), + base_url: Some("https://test.com".to_string()), env_key: Some("TEST_API_KEY".to_string()), env_key_instructions: None, wire_api: WireApi::Responses, @@ -641,6 +670,7 @@ mod tests { request_max_retries: Some(0), stream_max_retries: Some(0), stream_idle_timeout_ms: Some(1000), + requires_auth: false, }; let events = collect_events(&[sse1.as_bytes()], provider).await; @@ -733,7 +763,7 @@ mod tests { let provider = ModelProviderInfo { name: "test".to_string(), - base_url: "https://test.com".to_string(), + base_url: Some("https://test.com".to_string()), env_key: Some("TEST_API_KEY".to_string()), env_key_instructions: None, wire_api: WireApi::Responses, @@ -743,6 +773,7 @@ mod tests { request_max_retries: Some(0), stream_max_retries: Some(0), stream_idle_timeout_ms: Some(1000), + requires_auth: false, }; let out = run_sse(evs, provider).await; diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 6efc878fb5..92ca7bf88b 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -15,6 +15,7 @@ use async_channel::Sender; use codex_apply_patch::ApplyPatchAction; use codex_apply_patch::MaybeApplyPatchVerified; use codex_apply_patch::maybe_parse_apply_patch_verified; +use codex_login::CodexAuth; use futures::prelude::*; use mcp_types::CallToolResult; use serde::Serialize; @@ -103,7 +104,11 @@ pub struct CodexSpawnOk { impl Codex { /// Spawn a new [`Codex`] and initialize the session. - pub async fn spawn(config: Config, ctrl_c: Arc) -> CodexResult { + pub async fn spawn( + config: Config, + auth: Option, + ctrl_c: Arc, + ) -> CodexResult { // experimental resume path (undocumented) let resume_path = config.experimental_resume.clone(); info!("resume_path: {resume_path:?}"); @@ -132,7 +137,7 @@ impl Codex { // Generate a unique ID for the lifetime of this Codex session. let session_id = Uuid::new_v4(); tokio::spawn(submission_loop( - session_id, config, rx_sub, tx_event, ctrl_c, + session_id, config, auth, rx_sub, tx_event, ctrl_c, )); let codex = Codex { next_id: AtomicU64::new(0), @@ -525,6 +530,7 @@ impl AgentTask { async fn submission_loop( mut session_id: Uuid, config: Arc, + auth: Option, rx_sub: Receiver, tx_event: Sender, ctrl_c: Arc, @@ -636,6 +642,7 @@ async fn submission_loop( let client = ModelClient::new( config.clone(), + auth.clone(), provider.clone(), model_reasoning_effort, model_reasoning_summary, diff --git a/codex-rs/core/src/codex_wrapper.rs b/codex-rs/core/src/codex_wrapper.rs index b80579297a..1e26a9ebed 100644 --- a/codex-rs/core/src/codex_wrapper.rs +++ b/codex-rs/core/src/codex_wrapper.rs @@ -6,6 +6,7 @@ use crate::config::Config; use crate::protocol::Event; use crate::protocol::EventMsg; use crate::util::notify_on_sigint; +use codex_login::load_auth; use tokio::sync::Notify; use uuid::Uuid; @@ -25,11 +26,12 @@ pub struct CodexConversation { /// that callers can surface the information to the UI. pub async fn init_codex(config: Config) -> anyhow::Result { let ctrl_c = notify_on_sigint(); + let auth = load_auth(&config.codex_home)?; let CodexSpawnOk { codex, init_id, session_id, - } = Codex::spawn(config, ctrl_c.clone()).await?; + } = Codex::spawn(config, auth, ctrl_c.clone()).await?; // The first event must be `SessionInitialized`. Validate and forward it to // the caller so that they can display it in the conversation history. diff --git a/codex-rs/core/src/config.rs b/codex-rs/core/src/config.rs index 53ca8d5ba9..a65ec09674 100644 --- a/codex-rs/core/src/config.rs +++ b/codex-rs/core/src/config.rs @@ -526,6 +526,7 @@ impl Config { .chatgpt_base_url .or(cfg.chatgpt_base_url) .unwrap_or("https://chatgpt.com/backend-api/".to_string()), + experimental_resume, include_plan_tool: include_plan_tool.unwrap_or(false), }; @@ -794,7 +795,7 @@ disable_response_storage = true let openai_chat_completions_provider = ModelProviderInfo { name: "OpenAI using Chat Completions".to_string(), - base_url: "https://api.openai.com/v1".to_string(), + base_url: Some("https://api.openai.com/v1".to_string()), env_key: Some("OPENAI_API_KEY".to_string()), wire_api: crate::WireApi::Chat, env_key_instructions: None, @@ -804,6 +805,7 @@ disable_response_storage = true request_max_retries: Some(4), stream_max_retries: Some(10), stream_idle_timeout_ms: Some(300_000), + requires_auth: false, }; let model_provider_map = { let mut model_provider_map = built_in_model_providers(); diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index b2dbded5f1..ffe64d7c90 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -30,8 +30,8 @@ mod message_history; mod model_provider_info; pub use model_provider_info::ModelProviderInfo; pub use model_provider_info::WireApi; +pub use model_provider_info::built_in_model_providers; mod models; -pub mod openai_api_key; mod openai_model_info; mod openai_tools; pub mod plan_tool; diff --git a/codex-rs/core/src/model_provider_info.rs b/codex-rs/core/src/model_provider_info.rs index 72ef58c60a..4640f53ad7 100644 --- a/codex-rs/core/src/model_provider_info.rs +++ b/codex-rs/core/src/model_provider_info.rs @@ -12,7 +12,6 @@ use std::env::VarError; use std::time::Duration; use crate::error::EnvVarError; -use crate::openai_api_key::get_openai_api_key; /// Value for the `OpenAI-Originator` header that is sent with requests to /// OpenAI. @@ -30,7 +29,7 @@ const DEFAULT_REQUEST_MAX_RETRIES: u64 = 4; #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "lowercase")] pub enum WireApi { - /// The experimental "Responses" API exposed by OpenAI at `/v1/responses`. + /// The Responses API exposed by OpenAI at `/v1/responses`. Responses, /// Regular Chat Completions compatible with `/v1/chat/completions`. @@ -44,7 +43,7 @@ pub struct ModelProviderInfo { /// Friendly display name. pub name: String, /// Base URL for the provider's OpenAI-compatible API. - pub base_url: String, + pub base_url: Option, /// Environment variable that stores the user's API key for this provider. pub env_key: Option, @@ -78,6 +77,10 @@ pub struct ModelProviderInfo { /// Idle timeout (in milliseconds) to wait for activity on a streaming response before treating /// the connection as lost. pub stream_idle_timeout_ms: Option, + + /// Whether this provider requires some form of standard authentication (API key, ChatGPT token). + #[serde(default)] + pub requires_auth: bool, } impl ModelProviderInfo { @@ -93,11 +96,11 @@ impl ModelProviderInfo { &'a self, client: &'a reqwest::Client, ) -> crate::error::Result { - let api_key = self.api_key()?; - let url = self.get_full_url(); let mut builder = client.post(url); + + let api_key = self.api_key()?; if let Some(key) = api_key { builder = builder.bearer_auth(key); } @@ -117,9 +120,15 @@ impl ModelProviderInfo { .join("&"); format!("?{full_params}") }); - let base_url = &self.base_url; + let base_url = self + .base_url + .clone() + .unwrap_or("https://api.openai.com/v1".to_string()); + match self.wire_api { - WireApi::Responses => format!("{base_url}/responses{query_string}"), + WireApi::Responses => { + format!("{base_url}/responses{query_string}") + } WireApi::Chat => format!("{base_url}/chat/completions{query_string}"), } } @@ -127,7 +136,10 @@ impl ModelProviderInfo { /// Apply provider-specific HTTP headers (both static and environment-based) /// onto an existing `reqwest::RequestBuilder` and return the updated /// builder. - fn apply_http_headers(&self, mut builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + pub fn apply_http_headers( + &self, + mut builder: reqwest::RequestBuilder, + ) -> reqwest::RequestBuilder { if let Some(extra) = &self.http_headers { for (k, v) in extra { builder = builder.header(k, v); @@ -152,11 +164,7 @@ impl ModelProviderInfo { fn api_key(&self) -> crate::error::Result> { match &self.env_key { Some(env_key) => { - let env_value = if env_key == crate::openai_api_key::OPENAI_API_KEY_ENV_VAR { - get_openai_api_key().map_or_else(|| Err(VarError::NotPresent), Ok) - } else { - std::env::var(env_key) - }; + let env_value = std::env::var(env_key); env_value .and_then(|v| { if v.trim().is_empty() { @@ -204,47 +212,51 @@ pub fn built_in_model_providers() -> HashMap { // providers are bundled with Codex CLI, so we only include the OpenAI // provider by default. Users are encouraged to add to `model_providers` // in config.toml to add their own providers. - [ - ( - "openai", - P { - name: "OpenAI".into(), - // Allow users to override the default OpenAI endpoint by - // exporting `OPENAI_BASE_URL`. This is useful when pointing - // Codex at a proxy, mock server, or Azure-style deployment - // without requiring a full TOML override for the built-in - // OpenAI provider. - base_url: std::env::var("OPENAI_BASE_URL") - .ok() - .filter(|v| !v.trim().is_empty()) - .unwrap_or_else(|| "https://api.openai.com/v1".to_string()), - env_key: Some("OPENAI_API_KEY".into()), - env_key_instructions: Some("Create an API key (https://platform.openai.com) and export it as an environment variable.".into()), - wire_api: WireApi::Responses, - query_params: None, - http_headers: Some( - [ - ("originator".to_string(), OPENAI_ORIGINATOR_HEADER.to_string()), - ("version".to_string(), env!("CARGO_PKG_VERSION").to_string()), - ] - .into_iter() - .collect(), - ), - env_http_headers: Some( - [ - ("OpenAI-Organization".to_string(), "OPENAI_ORGANIZATION".to_string()), - ("OpenAI-Project".to_string(), "OPENAI_PROJECT".to_string()), - ] - .into_iter() - .collect(), - ), - // Use global defaults for retry/timeout unless overridden in config.toml. - request_max_retries: None, - stream_max_retries: None, - stream_idle_timeout_ms: None, - }, - ), - ] + [( + "openai", + P { + name: "OpenAI".into(), + // Allow users to override the default OpenAI endpoint by + // exporting `OPENAI_BASE_URL`. This is useful when pointing + // Codex at a proxy, mock server, or Azure-style deployment + // without requiring a full TOML override for the built-in + // OpenAI provider. + base_url: std::env::var("OPENAI_BASE_URL") + .ok() + .filter(|v| !v.trim().is_empty()), + env_key: None, + env_key_instructions: None, + wire_api: WireApi::Responses, + query_params: None, + http_headers: Some( + [ + ( + "originator".to_string(), + OPENAI_ORIGINATOR_HEADER.to_string(), + ), + ("version".to_string(), env!("CARGO_PKG_VERSION").to_string()), + ] + .into_iter() + .collect(), + ), + env_http_headers: Some( + [ + ( + "OpenAI-Organization".to_string(), + "OPENAI_ORGANIZATION".to_string(), + ), + ("OpenAI-Project".to_string(), "OPENAI_PROJECT".to_string()), + ] + .into_iter() + .collect(), + ), + // Use global defaults for retry/timeout unless overridden in config.toml. + request_max_retries: None, + stream_max_retries: None, + stream_idle_timeout_ms: None, + requires_auth: true, + }, + )] .into_iter() .map(|(k, v)| (k.to_string(), v)) .collect() @@ -264,7 +276,7 @@ base_url = "http://localhost:11434/v1" "#; let expected_provider = ModelProviderInfo { name: "Ollama".into(), - base_url: "http://localhost:11434/v1".into(), + base_url: Some("http://localhost:11434/v1".into()), env_key: None, env_key_instructions: None, wire_api: WireApi::Chat, @@ -274,6 +286,7 @@ base_url = "http://localhost:11434/v1" request_max_retries: None, stream_max_retries: None, stream_idle_timeout_ms: None, + requires_auth: false, }; let provider: ModelProviderInfo = toml::from_str(azure_provider_toml).unwrap(); @@ -290,7 +303,7 @@ query_params = { api-version = "2025-04-01-preview" } "#; let expected_provider = ModelProviderInfo { name: "Azure".into(), - base_url: "https://xxxxx.openai.azure.com/openai".into(), + base_url: Some("https://xxxxx.openai.azure.com/openai".into()), env_key: Some("AZURE_OPENAI_API_KEY".into()), env_key_instructions: None, wire_api: WireApi::Chat, @@ -302,6 +315,7 @@ query_params = { api-version = "2025-04-01-preview" } request_max_retries: None, stream_max_retries: None, stream_idle_timeout_ms: None, + requires_auth: false, }; let provider: ModelProviderInfo = toml::from_str(azure_provider_toml).unwrap(); @@ -319,7 +333,7 @@ env_http_headers = { "X-Example-Env-Header" = "EXAMPLE_ENV_VAR" } "#; let expected_provider = ModelProviderInfo { name: "Example".into(), - base_url: "https://example.com".into(), + base_url: Some("https://example.com".into()), env_key: Some("API_KEY".into()), env_key_instructions: None, wire_api: WireApi::Chat, @@ -333,6 +347,7 @@ env_http_headers = { "X-Example-Env-Header" = "EXAMPLE_ENV_VAR" } request_max_retries: None, stream_max_retries: None, stream_idle_timeout_ms: None, + requires_auth: false, }; let provider: ModelProviderInfo = toml::from_str(azure_provider_toml).unwrap(); diff --git a/codex-rs/core/src/openai_api_key.rs b/codex-rs/core/src/openai_api_key.rs deleted file mode 100644 index 728914c0f2..0000000000 --- a/codex-rs/core/src/openai_api_key.rs +++ /dev/null @@ -1,24 +0,0 @@ -use std::env; -use std::sync::LazyLock; -use std::sync::RwLock; - -pub const OPENAI_API_KEY_ENV_VAR: &str = "OPENAI_API_KEY"; - -static OPENAI_API_KEY: LazyLock>> = LazyLock::new(|| { - let val = env::var(OPENAI_API_KEY_ENV_VAR) - .ok() - .and_then(|s| if s.is_empty() { None } else { Some(s) }); - RwLock::new(val) -}); - -pub fn get_openai_api_key() -> Option { - #![allow(clippy::unwrap_used)] - OPENAI_API_KEY.read().unwrap().clone() -} - -pub fn set_openai_api_key(value: String) { - #![allow(clippy::unwrap_used)] - if !value.is_empty() { - *OPENAI_API_KEY.write().unwrap() = Some(value); - } -} diff --git a/codex-rs/core/tests/client.rs b/codex-rs/core/tests/client.rs index 9de2d56036..fbe63fb326 100644 --- a/codex-rs/core/tests/client.rs +++ b/codex-rs/core/tests/client.rs @@ -1,11 +1,19 @@ +use std::path::PathBuf; + +use chrono::Utc; use codex_core::Codex; use codex_core::CodexSpawnOk; use codex_core::ModelProviderInfo; +use codex_core::built_in_model_providers; use codex_core::exec::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR; use codex_core::protocol::EventMsg; use codex_core::protocol::InputItem; use codex_core::protocol::Op; use codex_core::protocol::SessionConfiguredEvent; +use codex_login::AuthDotJson; +use codex_login::AuthMode; +use codex_login::CodexAuth; +use codex_login::TokenData; use core_test_support::load_default_config_for_test; use core_test_support::load_sse_fixture_with_id; use core_test_support::wait_for_event; @@ -48,32 +56,23 @@ async fn includes_session_id_and_model_headers_in_request() { .await; let model_provider = ModelProviderInfo { - name: "openai".into(), - base_url: format!("{}/v1", server.uri()), - // Environment variable that should exist in the test environment. - // ModelClient will return an error if the environment variable for the - // provider is not set. - env_key: Some("PATH".into()), - env_key_instructions: None, - wire_api: codex_core::WireApi::Responses, - query_params: None, - http_headers: Some( - [("originator".to_string(), "codex_cli_rs".to_string())] - .into_iter() - .collect(), - ), - env_http_headers: None, - request_max_retries: Some(0), - stream_max_retries: Some(0), - stream_idle_timeout_ms: None, + base_url: Some(format!("{}/v1", server.uri())), + ..built_in_model_providers()["openai"].clone() }; // Init session let codex_home = TempDir::new().unwrap(); let mut config = load_default_config_for_test(&codex_home); config.model_provider = model_provider; + let ctrl_c = std::sync::Arc::new(tokio::sync::Notify::new()); - let CodexSpawnOk { codex, .. } = Codex::spawn(config, ctrl_c.clone()).await.unwrap(); + let CodexSpawnOk { codex, .. } = Codex::spawn( + config, + Some(CodexAuth::from_api_key("Test API Key".to_string())), + ctrl_c.clone(), + ) + .await + .unwrap(); codex .submit(Op::UserInput { @@ -95,15 +94,20 @@ async fn includes_session_id_and_model_headers_in_request() { // get request from the server let request = &server.received_requests().await.unwrap()[0]; - let request_body = request.headers.get("session_id").unwrap(); - let originator = request.headers.get("originator").unwrap(); + let request_session_id = request.headers.get("session_id").unwrap(); + let request_originator = request.headers.get("originator").unwrap(); + let request_authorization = request.headers.get("authorization").unwrap(); assert!(current_session_id.is_some()); assert_eq!( - request_body.to_str().unwrap(), + request_session_id.to_str().unwrap(), current_session_id.as_ref().unwrap() ); - assert_eq!(originator.to_str().unwrap(), "codex_cli_rs"); + assert_eq!(request_originator.to_str().unwrap(), "codex_cli_rs"); + assert_eq!( + request_authorization.to_str().unwrap(), + "Bearer Test API Key" + ); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -126,22 +130,9 @@ async fn includes_base_instructions_override_in_request() { .await; let model_provider = ModelProviderInfo { - name: "openai".into(), - base_url: format!("{}/v1", server.uri()), - // Environment variable that should exist in the test environment. - // ModelClient will return an error if the environment variable for the - // provider is not set. - env_key: Some("PATH".into()), - env_key_instructions: None, - wire_api: codex_core::WireApi::Responses, - query_params: None, - http_headers: None, - env_http_headers: None, - request_max_retries: Some(0), - stream_max_retries: Some(0), - stream_idle_timeout_ms: None, + base_url: Some(format!("{}/v1", server.uri())), + ..built_in_model_providers()["openai"].clone() }; - let codex_home = TempDir::new().unwrap(); let mut config = load_default_config_for_test(&codex_home); @@ -149,7 +140,13 @@ async fn includes_base_instructions_override_in_request() { config.model_provider = model_provider; let ctrl_c = std::sync::Arc::new(tokio::sync::Notify::new()); - let CodexSpawnOk { codex, .. } = Codex::spawn(config, ctrl_c.clone()).await.unwrap(); + let CodexSpawnOk { codex, .. } = Codex::spawn( + config, + Some(CodexAuth::from_api_key("Test API Key".to_string())), + ctrl_c.clone(), + ) + .await + .unwrap(); codex .submit(Op::UserInput { @@ -172,3 +169,108 @@ async fn includes_base_instructions_override_in_request() { .contains("test instructions") ); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn chatgpt_auth_sends_correct_request() { + #![allow(clippy::unwrap_used)] + + if std::env::var(CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR).is_ok() { + println!( + "Skipping test because it cannot execute when network is disabled in a Codex sandbox." + ); + return; + } + + // Mock server + let server = MockServer::start().await; + + // First request – must NOT include `previous_response_id`. + let first = ResponseTemplate::new(200) + .insert_header("content-type", "text/event-stream") + .set_body_raw(sse_completed("resp1"), "text/event-stream"); + + Mock::given(method("POST")) + .and(path("/api/codex/responses")) + .respond_with(first) + .expect(1) + .mount(&server) + .await; + + let model_provider = ModelProviderInfo { + base_url: Some(format!("{}/api/codex", server.uri())), + ..built_in_model_providers()["openai"].clone() + }; + + // Init session + let codex_home = TempDir::new().unwrap(); + let mut config = load_default_config_for_test(&codex_home); + config.model_provider = model_provider; + let ctrl_c = std::sync::Arc::new(tokio::sync::Notify::new()); + let CodexSpawnOk { codex, .. } = Codex::spawn( + config, + Some(auth_from_token("Access Token".to_string())), + ctrl_c.clone(), + ) + .await + .unwrap(); + + codex + .submit(Op::UserInput { + items: vec![InputItem::Text { + text: "hello".into(), + }], + }) + .await + .unwrap(); + + let EventMsg::SessionConfigured(SessionConfiguredEvent { session_id, .. }) = + wait_for_event(&codex, |ev| matches!(ev, EventMsg::SessionConfigured(_))).await + else { + unreachable!() + }; + + let current_session_id = Some(session_id.to_string()); + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TaskComplete(_))).await; + + // get request from the server + let request = &server.received_requests().await.unwrap()[0]; + let request_session_id = request.headers.get("session_id").unwrap(); + let request_originator = request.headers.get("originator").unwrap(); + let request_authorization = request.headers.get("authorization").unwrap(); + let request_body = request.body_json::().unwrap(); + + assert!(current_session_id.is_some()); + assert_eq!( + request_session_id.to_str().unwrap(), + current_session_id.as_ref().unwrap() + ); + assert_eq!(request_originator.to_str().unwrap(), "codex_cli_rs"); + assert_eq!( + request_authorization.to_str().unwrap(), + "Bearer Access Token" + ); + assert!(!request_body["store"].as_bool().unwrap()); + assert!(request_body["stream"].as_bool().unwrap()); + assert_eq!( + request_body["include"][0].as_str().unwrap(), + "reasoning.encrypted_content" + ); +} + +fn auth_from_token(id_token: String) -> CodexAuth { + CodexAuth::new( + None, + AuthMode::ChatGPT, + PathBuf::new(), + Some(AuthDotJson { + tokens: TokenData { + id_token, + access_token: "Access Token".to_string(), + refresh_token: "test".to_string(), + account_id: None, + }, + last_refresh: Utc::now(), + openai_api_key: None, + }), + ) +} diff --git a/codex-rs/core/tests/live_agent.rs b/codex-rs/core/tests/live_agent.rs index 9895343045..95408e20e5 100644 --- a/codex-rs/core/tests/live_agent.rs +++ b/codex-rs/core/tests/live_agent.rs @@ -50,7 +50,7 @@ async fn spawn_codex() -> Result { config.model_provider.request_max_retries = Some(2); config.model_provider.stream_max_retries = Some(2); let CodexSpawnOk { codex: agent, .. } = - Codex::spawn(config, std::sync::Arc::new(Notify::new())).await?; + Codex::spawn(config, None, std::sync::Arc::new(Notify::new())).await?; Ok(agent) } diff --git a/codex-rs/core/tests/stream_no_completed.rs b/codex-rs/core/tests/stream_no_completed.rs index 8e5d83a03e..d2fc035569 100644 --- a/codex-rs/core/tests/stream_no_completed.rs +++ b/codex-rs/core/tests/stream_no_completed.rs @@ -10,6 +10,7 @@ use codex_core::exec::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR; use codex_core::protocol::EventMsg; use codex_core::protocol::InputItem; use codex_core::protocol::Op; +use codex_login::CodexAuth; use core_test_support::load_default_config_for_test; use core_test_support::load_sse_fixture; use core_test_support::load_sse_fixture_with_id; @@ -75,7 +76,7 @@ async fn retries_on_early_close() { let model_provider = ModelProviderInfo { name: "openai".into(), - base_url: format!("{}/v1", server.uri()), + base_url: Some(format!("{}/v1", server.uri())), // Environment variable that should exist in the test environment. // ModelClient will return an error if the environment variable for the // provider is not set. @@ -89,13 +90,20 @@ async fn retries_on_early_close() { request_max_retries: Some(0), stream_max_retries: Some(1), stream_idle_timeout_ms: Some(2000), + requires_auth: false, }; let ctrl_c = std::sync::Arc::new(tokio::sync::Notify::new()); let codex_home = TempDir::new().unwrap(); let mut config = load_default_config_for_test(&codex_home); config.model_provider = model_provider; - let CodexSpawnOk { codex, .. } = Codex::spawn(config, ctrl_c).await.unwrap(); + let CodexSpawnOk { codex, .. } = Codex::spawn( + config, + Some(CodexAuth::from_api_key("Test API Key".to_string())), + ctrl_c, + ) + .await + .unwrap(); codex .submit(Op::UserInput { diff --git a/codex-rs/login/src/lib.rs b/codex-rs/login/src/lib.rs index ab92ecf616..47dbbca9fb 100644 --- a/codex-rs/login/src/lib.rs +++ b/codex-rs/login/src/lib.rs @@ -1,20 +1,152 @@ use chrono::DateTime; + use chrono::Utc; use serde::Deserialize; use serde::Serialize; +use std::env; use std::fs::OpenOptions; use std::io::Read; use std::io::Write; #[cfg(unix)] use std::os::unix::fs::OpenOptionsExt; use std::path::Path; +use std::path::PathBuf; use std::process::Stdio; +use std::sync::Arc; +use std::sync::Mutex; use std::time::Duration; use tokio::process::Command; const SOURCE_FOR_PYTHON_SERVER: &str = include_str!("./login_with_chatgpt.py"); const CLIENT_ID: &str = "app_EMoamEEZ73f0CkXaXp7hrann"; +const OPENAI_API_KEY_ENV_VAR: &str = "OPENAI_API_KEY"; + +#[derive(Clone, Debug, PartialEq)] +pub enum AuthMode { + ApiKey, + ChatGPT, +} + +#[derive(Debug, Clone)] +pub struct CodexAuth { + pub api_key: Option, + pub mode: AuthMode, + auth_dot_json: Arc>>, + auth_file: PathBuf, +} + +impl PartialEq for CodexAuth { + fn eq(&self, other: &Self) -> bool { + self.mode == other.mode + } +} + +impl CodexAuth { + pub fn new( + api_key: Option, + mode: AuthMode, + auth_file: PathBuf, + auth_dot_json: Option, + ) -> Self { + let auth_dot_json = Arc::new(Mutex::new(auth_dot_json)); + Self { + api_key, + mode, + auth_file, + auth_dot_json, + } + } + + pub fn from_api_key(api_key: String) -> Self { + Self { + api_key: Some(api_key), + mode: AuthMode::ApiKey, + auth_file: PathBuf::new(), + auth_dot_json: Arc::new(Mutex::new(None)), + } + } + + pub async fn get_token_data(&self) -> Result { + #[expect(clippy::unwrap_used)] + let auth_dot_json = self.auth_dot_json.lock().unwrap().clone(); + + match auth_dot_json { + Some(auth_dot_json) => { + if auth_dot_json.last_refresh < Utc::now() - chrono::Duration::days(28) { + let refresh_response = tokio::time::timeout( + Duration::from_secs(60), + try_refresh_token(auth_dot_json.tokens.refresh_token.clone()), + ) + .await + .map_err(|_| { + std::io::Error::other("timed out while refreshing OpenAI API key") + })? + .map_err(std::io::Error::other)?; + + let updated_auth_dot_json = update_tokens( + &self.auth_file, + refresh_response.id_token, + refresh_response.access_token, + refresh_response.refresh_token, + ) + .await?; + + #[expect(clippy::unwrap_used)] + let mut auth_dot_json = self.auth_dot_json.lock().unwrap(); + *auth_dot_json = Some(updated_auth_dot_json); + } + Ok(auth_dot_json.tokens.clone()) + } + None => Err(std::io::Error::other("Token data is not available.")), + } + } + + pub async fn get_token(&self) -> Result { + match self.mode { + AuthMode::ApiKey => Ok(self.api_key.clone().unwrap_or_default()), + AuthMode::ChatGPT => { + let id_token = self.get_token_data().await?.access_token; + + Ok(id_token) + } + } + } +} + +// Loads the available auth information from the auth.json or OPENAI_API_KEY environment variable. +pub fn load_auth(codex_home: &Path) -> std::io::Result> { + let auth_file = codex_home.join("auth.json"); + + let auth_dot_json = try_read_auth_json(&auth_file).ok(); + + let auth_json_api_key = auth_dot_json + .as_ref() + .and_then(|a| a.openai_api_key.clone()) + .filter(|s| !s.is_empty()); + + let openai_api_key = env::var(OPENAI_API_KEY_ENV_VAR) + .ok() + .filter(|s| !s.is_empty()) + .or(auth_json_api_key); + + if openai_api_key.is_none() && auth_dot_json.is_none() { + return Ok(None); + } + + let mode = if openai_api_key.is_some() { + AuthMode::ApiKey + } else { + AuthMode::ChatGPT + }; + + Ok(Some(CodexAuth { + api_key: openai_api_key, + mode, + auth_file, + auth_dot_json: Arc::new(Mutex::new(auth_dot_json)), + })) +} /// Run `python3 -c {{SOURCE_FOR_PYTHON_SERVER}}` with the CODEX_HOME /// environment variable set to the provided `codex_home` path. If the @@ -25,14 +157,12 @@ const CLIENT_ID: &str = "app_EMoamEEZ73f0CkXaXp7hrann"; /// If `capture_output` is true, the subprocess's output will be captured and /// recorded in memory. Otherwise, the subprocess's output will be sent to the /// current process's stdout/stderr. -pub async fn login_with_chatgpt( - codex_home: &Path, - capture_output: bool, -) -> std::io::Result { +pub async fn login_with_chatgpt(codex_home: &Path, capture_output: bool) -> std::io::Result<()> { let child = Command::new("python3") .arg("-c") .arg(SOURCE_FOR_PYTHON_SERVER) .env("CODEX_HOME", codex_home) + .env("CODEX_CLIENT_ID", CLIENT_ID) .stdin(Stdio::null()) .stdout(if capture_output { Stdio::piped() @@ -48,7 +178,7 @@ pub async fn login_with_chatgpt( let output = child.wait_with_output().await?; if output.status.success() { - try_read_openai_api_key(codex_home).await + Ok(()) } else { let stderr = String::from_utf8_lossy(&output.stderr); Err(std::io::Error::other(format!( @@ -57,65 +187,54 @@ pub async fn login_with_chatgpt( } } -/// Attempt to read the `OPENAI_API_KEY` from the `auth.json` file in the given -/// `CODEX_HOME` directory, refreshing it, if necessary. -pub async fn try_read_openai_api_key(codex_home: &Path) -> std::io::Result { - let auth_dot_json = try_read_auth_json(codex_home).await?; - Ok(auth_dot_json.openai_api_key) -} - /// Attempt to read and refresh the `auth.json` file in the given `CODEX_HOME` directory. /// Returns the full AuthDotJson structure after refreshing if necessary. -pub async fn try_read_auth_json(codex_home: &Path) -> std::io::Result { - let auth_path = codex_home.join("auth.json"); - let mut file = std::fs::File::open(&auth_path)?; +pub fn try_read_auth_json(auth_file: &Path) -> std::io::Result { + let mut file = std::fs::File::open(auth_file)?; let mut contents = String::new(); file.read_to_string(&mut contents)?; let auth_dot_json: AuthDotJson = serde_json::from_str(&contents)?; - if is_expired(&auth_dot_json) { - let refresh_response = - tokio::time::timeout(Duration::from_secs(60), try_refresh_token(&auth_dot_json)) - .await - .map_err(|_| std::io::Error::other("timed out while refreshing OpenAI API key"))? - .map_err(std::io::Error::other)?; - let mut auth_dot_json = auth_dot_json; - auth_dot_json.tokens.id_token = refresh_response.id_token; - if let Some(refresh_token) = refresh_response.refresh_token { - auth_dot_json.tokens.refresh_token = refresh_token; - } - auth_dot_json.last_refresh = Utc::now(); + Ok(auth_dot_json) +} - let mut options = OpenOptions::new(); - options.truncate(true).write(true).create(true); - #[cfg(unix)] - { - options.mode(0o600); - } - - let json_data = serde_json::to_string(&auth_dot_json)?; - { - let mut file = options.open(&auth_path)?; - file.write_all(json_data.as_bytes())?; - file.flush()?; - } - - Ok(auth_dot_json) - } else { - Ok(auth_dot_json) +async fn update_tokens( + auth_file: &Path, + id_token: String, + access_token: Option, + refresh_token: Option, +) -> std::io::Result { + let mut options = OpenOptions::new(); + options.truncate(true).write(true).create(true); + #[cfg(unix)] + { + options.mode(0o600); } + let mut auth_dot_json = try_read_auth_json(auth_file)?; + + auth_dot_json.tokens.id_token = id_token.to_string(); + if let Some(access_token) = access_token { + auth_dot_json.tokens.access_token = access_token.to_string(); + } + if let Some(refresh_token) = refresh_token { + auth_dot_json.tokens.refresh_token = refresh_token.to_string(); + } + auth_dot_json.last_refresh = Utc::now(); + + let json_data = serde_json::to_string_pretty(&auth_dot_json)?; + { + let mut file = options.open(auth_file)?; + file.write_all(json_data.as_bytes())?; + file.flush()?; + } + Ok(auth_dot_json) } -fn is_expired(auth_dot_json: &AuthDotJson) -> bool { - let last_refresh = auth_dot_json.last_refresh; - last_refresh < Utc::now() - chrono::Duration::days(28) -} - -async fn try_refresh_token(auth_dot_json: &AuthDotJson) -> std::io::Result { +async fn try_refresh_token(refresh_token: String) -> std::io::Result { let refresh_request = RefreshRequest { client_id: CLIENT_ID, grant_type: "refresh_token", - refresh_token: auth_dot_json.tokens.refresh_token.clone(), + refresh_token, scope: "openid profile email", }; @@ -150,24 +269,25 @@ struct RefreshRequest { scope: &'static str, } -#[derive(Deserialize)] +#[derive(Deserialize, Clone)] struct RefreshResponse { id_token: String, + access_token: Option, refresh_token: Option, } /// Expected structure for $CODEX_HOME/auth.json. -#[derive(Deserialize, Serialize)] +#[derive(Deserialize, Serialize, Clone, Debug, PartialEq)] pub struct AuthDotJson { #[serde(rename = "OPENAI_API_KEY")] - pub openai_api_key: String, + pub openai_api_key: Option, pub tokens: TokenData, pub last_refresh: DateTime, } -#[derive(Deserialize, Serialize, Clone)] +#[derive(Deserialize, Serialize, Clone, Debug, PartialEq)] pub struct TokenData { /// This is a JWT. pub id_token: String, @@ -177,5 +297,5 @@ pub struct TokenData { pub refresh_token: String, - pub account_id: String, + pub account_id: Option, } diff --git a/codex-rs/login/src/login_with_chatgpt.py b/codex-rs/login/src/login_with_chatgpt.py index ccb051c0af..2dbf5be58a 100644 --- a/codex-rs/login/src/login_with_chatgpt.py +++ b/codex-rs/login/src/login_with_chatgpt.py @@ -41,7 +41,6 @@ from typing import Any, Dict # for type hints REQUIRED_PORT = 1455 URL_BASE = f"http://localhost:{REQUIRED_PORT}" DEFAULT_ISSUER = "https://auth.openai.com" -DEFAULT_CLIENT_ID = "app_EMoamEEZ73f0CkXaXp7hrann" EXIT_CODE_WHEN_ADDRESS_ALREADY_IN_USE = 13 @@ -58,7 +57,7 @@ class TokenData: class AuthBundle: """Aggregates authentication data produced after successful OAuth flow.""" - api_key: str + api_key: str | None token_data: TokenData last_refresh: str @@ -78,12 +77,18 @@ def main() -> None: eprint("ERROR: CODEX_HOME environment variable is not set") sys.exit(1) + client_id = os.getenv("CODEX_CLIENT_ID") + if not client_id: + eprint("ERROR: CODEX_CLIENT_ID environment variable is not set") + sys.exit(1) + # Spawn server. try: httpd = _ApiKeyHTTPServer( ("127.0.0.1", REQUIRED_PORT), _ApiKeyHTTPHandler, codex_home=codex_home, + client_id=client_id, verbose=args.verbose, ) except OSError as e: @@ -157,7 +162,7 @@ class _ApiKeyHTTPHandler(http.server.BaseHTTPRequestHandler): return try: - auth_bundle, success_url = self._exchange_code_for_api_key(code) + auth_bundle, success_url = self._exchange_code(code) except Exception as exc: # noqa: BLE001 – propagate to client self.send_error(500, f"Token exchange failed: {exc}") return @@ -211,68 +216,22 @@ class _ApiKeyHTTPHandler(http.server.BaseHTTPRequestHandler): if getattr(self.server, "verbose", False): # type: ignore[attr-defined] super().log_message(fmt, *args) - def _exchange_code_for_api_key(self, code: str) -> tuple[AuthBundle, str]: - """Perform token + token-exchange to obtain an OpenAI API key. + def _obtain_api_key( + self, + token_claims: Dict[str, Any], + access_claims: Dict[str, Any], + token_data: TokenData, + ) -> tuple[str | None, str | None]: + """Obtain an API key from the auth service. - Returns (AuthBundle, success_url). + Returns (api_key, success_url) if successful, None otherwise. """ - token_endpoint = f"{self.server.issuer}/oauth/token" - - # 1. Authorization-code -> (id_token, access_token, refresh_token) - data = urllib.parse.urlencode( - { - "grant_type": "authorization_code", - "code": code, - "redirect_uri": self.server.redirect_uri, - "client_id": self.server.client_id, - "code_verifier": self.server.pkce.code_verifier, - } - ).encode() - - token_data: TokenData - - with urllib.request.urlopen( - urllib.request.Request( - token_endpoint, - data=data, - method="POST", - headers={"Content-Type": "application/x-www-form-urlencoded"}, - ) - ) as resp: - payload = json.loads(resp.read().decode()) - - # Extract chatgpt_account_id from id_token - id_token_parts = payload["id_token"].split(".") - if len(id_token_parts) != 3: - raise ValueError("Invalid ID token") - id_token_claims = _decode_jwt_segment(id_token_parts[1]) - auth_claims = id_token_claims.get("https://api.openai.com/auth", {}) - chatgpt_account_id = auth_claims.get("chatgpt_account_id", "") - - token_data = TokenData( - id_token=payload["id_token"], - access_token=payload["access_token"], - refresh_token=payload["refresh_token"], - account_id=chatgpt_account_id, - ) - - access_token_parts = token_data.access_token.split(".") - if len(access_token_parts) != 3: - raise ValueError("Invalid access token") - - access_token_claims = _decode_jwt_segment(access_token_parts[1]) - - token_claims = id_token_claims.get("https://api.openai.com/auth", {}) - access_claims = access_token_claims.get("https://api.openai.com/auth", {}) - org_id = token_claims.get("organization_id") - if not org_id: - raise ValueError("Missing organization in id_token claims") - project_id = token_claims.get("project_id") - if not project_id: - raise ValueError("Missing project in id_token claims") + + if not org_id or not project_id: + return (None, None) random_id = secrets.token_hex(6) @@ -292,7 +251,7 @@ class _ApiKeyHTTPHandler(http.server.BaseHTTPRequestHandler): exchanged_access_token: str with urllib.request.urlopen( urllib.request.Request( - token_endpoint, + self.server.token_endpoint, data=exchange_data, method="POST", headers={"Content-Type": "application/x-www-form-urlencoded"}, @@ -340,6 +299,65 @@ class _ApiKeyHTTPHandler(http.server.BaseHTTPRequestHandler): except Exception as exc: # pragma: no cover – best-effort only eprint(f"Unable to redeem ChatGPT subscriber API credits: {exc}") + return (exchanged_access_token, success_url) + + def _exchange_code(self, code: str) -> tuple[AuthBundle, str]: + """Perform token + token-exchange to obtain an OpenAI API key. + + Returns (AuthBundle, success_url). + """ + + # 1. Authorization-code -> (id_token, access_token, refresh_token) + data = urllib.parse.urlencode( + { + "grant_type": "authorization_code", + "code": code, + "redirect_uri": self.server.redirect_uri, + "client_id": self.server.client_id, + "code_verifier": self.server.pkce.code_verifier, + } + ).encode() + + token_data: TokenData + + with urllib.request.urlopen( + urllib.request.Request( + self.server.token_endpoint, + data=data, + method="POST", + headers={"Content-Type": "application/x-www-form-urlencoded"}, + ) + ) as resp: + payload = json.loads(resp.read().decode()) + + # Extract chatgpt_account_id from id_token + id_token_parts = payload["id_token"].split(".") + if len(id_token_parts) != 3: + raise ValueError("Invalid ID token") + id_token_claims = _decode_jwt_segment(id_token_parts[1]) + auth_claims = id_token_claims.get("https://api.openai.com/auth", {}) + chatgpt_account_id = auth_claims.get("chatgpt_account_id", "") + + token_data = TokenData( + id_token=payload["id_token"], + access_token=payload["access_token"], + refresh_token=payload["refresh_token"], + account_id=chatgpt_account_id, + ) + + access_token_parts = token_data.access_token.split(".") + if len(access_token_parts) != 3: + raise ValueError("Invalid access token") + + access_token_claims = _decode_jwt_segment(access_token_parts[1]) + + token_claims = id_token_claims.get("https://api.openai.com/auth", {}) + access_claims = access_token_claims.get("https://api.openai.com/auth", {}) + + exchanged_access_token, success_url = self._obtain_api_key( + token_claims, access_claims, token_data + ) + # Persist refresh_token/id_token for future use (redeem credits etc.) last_refresh_str = ( datetime.datetime.now(datetime.timezone.utc) @@ -353,7 +371,7 @@ class _ApiKeyHTTPHandler(http.server.BaseHTTPRequestHandler): last_refresh=last_refresh_str, ) - return (auth_bundle, success_url) + return (auth_bundle, success_url or f"{URL_BASE}/success") def request_shutdown(self) -> None: # shutdown() must be invoked from another thread to avoid @@ -413,6 +431,7 @@ class _ApiKeyHTTPServer(http.server.HTTPServer): request_handler_class: type[http.server.BaseHTTPRequestHandler], *, codex_home: str, + client_id: str, verbose: bool = False, ) -> None: super().__init__(server_address, request_handler_class, bind_and_activate=True) @@ -422,7 +441,8 @@ class _ApiKeyHTTPServer(http.server.HTTPServer): self.verbose: bool = verbose self.issuer: str = DEFAULT_ISSUER - self.client_id: str = DEFAULT_CLIENT_ID + self.token_endpoint: str = f"{self.issuer}/oauth/token" + self.client_id: str = client_id port = server_address[1] self.redirect_uri: str = f"http://localhost:{port}/auth/callback" self.pkce: PkceCodes = _generate_pkce() @@ -581,8 +601,8 @@ def maybe_redeem_credits( granted = redeem_data.get("granted_chatgpt_subscriber_api_credits", 0) if granted and granted > 0: eprint( - f"""Thanks for being a ChatGPT {'Plus' if plan_type=='plus' else 'Pro'} subscriber! -If you haven't already redeemed, you should receive {'$5' if plan_type=='plus' else '$50'} in API credits. + f"""Thanks for being a ChatGPT {"Plus" if plan_type == "plus" else "Pro"} subscriber! +If you haven't already redeemed, you should receive {"$5" if plan_type == "plus" else "$50"} in API credits. Credits: https://platform.openai.com/settings/organization/billing/credit-grants More info: https://help.openai.com/en/articles/11381614""", diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index 7bc041a554..424b5ac2fc 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -6,16 +6,14 @@ use app::App; use codex_core::config::Config; use codex_core::config::ConfigOverrides; use codex_core::config_types::SandboxMode; -use codex_core::openai_api_key::OPENAI_API_KEY_ENV_VAR; -use codex_core::openai_api_key::get_openai_api_key; -use codex_core::openai_api_key::set_openai_api_key; use codex_core::protocol::AskForApproval; use codex_core::util::is_inside_git_repo; -use codex_login::try_read_openai_api_key; +use codex_login::load_auth; use log_layer::TuiLogLayer; use std::fs::OpenOptions; use std::io::Write; use std::path::PathBuf; +use tracing::error; use tracing_appender::non_blocking; use tracing_subscriber::EnvFilter; use tracing_subscriber::prelude::*; @@ -140,7 +138,7 @@ pub async fn run_main( .with(tui_layer) .try_init(); - let show_login_screen = should_show_login_screen(&config).await; + let show_login_screen = should_show_login_screen(&config); if show_login_screen { std::io::stdout() .write_all(b"No API key detected.\nLogin with your ChatGPT account? [Yn] ")?; @@ -153,8 +151,8 @@ pub async fn run_main( } // Spawn a task to run the login command. // Block until the login command is finished. - let new_key = codex_login::login_with_chatgpt(&config.codex_home, false).await?; - set_openai_api_key(new_key); + codex_login::login_with_chatgpt(&config.codex_home, false).await?; + std::io::stdout().write_all(b"Login successful.\n")?; } @@ -217,28 +215,21 @@ fn restore() { } } -async fn should_show_login_screen(config: &Config) -> bool { - if is_in_need_of_openai_api_key(config) { +#[allow(clippy::unwrap_used)] +fn should_show_login_screen(config: &Config) -> bool { + if config.model_provider.requires_auth { // Reading the OpenAI API key is an async operation because it may need // to refresh the token. Block on it. let codex_home = config.codex_home.clone(); - if let Ok(openai_api_key) = try_read_openai_api_key(&codex_home).await { - set_openai_api_key(openai_api_key); - false - } else { - true + match load_auth(&codex_home) { + Ok(Some(_)) => false, + Ok(None) => true, + Err(err) => { + error!("Failed to read auth.json: {err}"); + true + } } } else { false } } - -fn is_in_need_of_openai_api_key(config: &Config) -> bool { - let is_using_openai_key = config - .model_provider - .env_key - .as_ref() - .map(|s| s == OPENAI_API_KEY_ENV_VAR) - .unwrap_or(false); - is_using_openai_key && get_openai_api_key().is_none() -} From 2f5557056d259873062509ca47162cc696257aa9 Mon Sep 17 00:00:00 2001 From: aibrahim-oai Date: Wed, 30 Jul 2025 13:43:08 -0700 Subject: [PATCH 006/100] moving input item from MCP Protocol back to core Protocol (#1740) - Currently we have duplicate input item. Let's have one source of truth in the core. - Used Requestid type --- codex-rs/mcp-server/src/mcp_protocol.rs | 140 +++++++----------------- 1 file changed, 38 insertions(+), 102 deletions(-) diff --git a/codex-rs/mcp-server/src/mcp_protocol.rs b/codex-rs/mcp-server/src/mcp_protocol.rs index 933c145b48..e507376c16 100644 --- a/codex-rs/mcp-server/src/mcp_protocol.rs +++ b/codex-rs/mcp-server/src/mcp_protocol.rs @@ -1,6 +1,7 @@ use codex_core::config_types::SandboxMode; use codex_core::protocol::AskForApproval; use codex_core::protocol::EventMsg; +use codex_core::protocol::InputItem; use serde::Deserialize; use serde::Serialize; use strum_macros::Display; @@ -21,7 +22,7 @@ pub struct MessageId(pub Uuid); pub struct ToolCallRequest { #[serde(rename = "jsonrpc")] pub jsonrpc: &'static str, - pub id: u64, + pub id: RequestId, pub method: &'static str, pub params: ToolCallRequestParams, } @@ -38,7 +39,7 @@ pub enum ToolCallRequestParams { impl ToolCallRequestParams { /// Wrap this request in a JSON-RPC request. #[allow(dead_code)] - pub fn into_request(self, id: u64) -> ToolCallRequest { + pub fn into_request(self, id: RequestId) -> ToolCallRequest { ToolCallRequest { jsonrpc: "2.0", id, @@ -95,70 +96,13 @@ pub struct ConversationStreamArgs { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct ConversationSendMessageArgs { pub conversation_id: ConversationId, - pub content: Vec, + pub content: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] pub parent_message_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] #[serde(flatten)] pub conversation_overrides: Option, } - -/// Input items for a message. -/// Following OpenAI's Responses API: https://platform.openai.com/docs/api-reference/responses -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "camelCase")] -pub enum MessageInputItem { - Text { - text: String, - }, - Image { - #[serde(flatten)] - source: ImageSource, - #[serde(default, skip_serializing_if = "Option::is_none")] - detail: Option, - }, - File { - #[serde(flatten)] - source: FileSource, - }, -} - -/// Source of an image. -/// Following OpenAI's API: https://platform.openai.com/docs/guides/images-vision#giving-a-model-images-as-input -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(untagged)] -pub enum ImageSource { - ImageUrl { image_url: String }, - FileId { file_id: String }, -} - -/// Source of a file. -/// Following OpenAI's Responses API: https://platform.openai.com/docs/guides/pdf-files?api-mode=responses#uploading-files -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(untagged)] -pub enum FileSource { - Url { - file_url: String, - }, - Id { - file_id: String, - }, - Base64 { - #[serde(default, skip_serializing_if = "Option::is_none")] - filename: Option, - // Base64-encoded file contents. - file_data: String, - }, -} - -#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum ImageDetail { - Low, - High, - Auto, -} - #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct ConversationsListArgs { #[serde(default, skip_serializing_if = "Option::is_none")] @@ -303,6 +247,8 @@ pub enum ClientNotification { #[allow(clippy::expect_used)] #[allow(clippy::unwrap_used)] mod tests { + use std::path::PathBuf; + use super::*; use codex_core::protocol::McpInvocation; use codex_core::protocol::McpToolCallBeginEvent; @@ -331,7 +277,7 @@ mod tests { base_instructions: None, }); - let observed = to_val(&req.into_request(2)); + let observed = to_val(&req.into_request(mcp_types::RequestId::Integer(2))); let expected = json!({ "jsonrpc": "2.0", "id": 2, @@ -354,18 +300,12 @@ mod tests { let req = ToolCallRequestParams::ConversationSendMessage(ConversationSendMessageArgs { conversation_id: ConversationId(uuid!("d0f6ecbe-84a2-41c1-b23d-b20473b25eab")), content: vec![ - MessageInputItem::Text { text: "Hi".into() }, - MessageInputItem::Image { - source: ImageSource::ImageUrl { - image_url: "https://example.com/cat.jpg".into(), - }, - detail: Some(ImageDetail::High), + InputItem::Text { text: "Hi".into() }, + InputItem::Image { + image_url: "https://example.com/cat.jpg".into(), }, - MessageInputItem::File { - source: FileSource::Base64 { - filename: Some("notes.txt".into()), - file_data: "Zm9vYmFy".into(), - }, + InputItem::LocalImage { + path: "notes.txt".into(), }, ], parent_message_id: Some(MessageId(uuid!("67e55044-10b1-426f-9247-bb680e5fe0c8"))), @@ -380,7 +320,7 @@ mod tests { }), }); - let observed = to_val(&req.into_request(2)); + let observed = to_val(&req.into_request(mcp_types::RequestId::Integer(2))); let expected = json!({ "jsonrpc": "2.0", "id": 2, @@ -391,8 +331,8 @@ mod tests { "conversation_id": "d0f6ecbe-84a2-41c1-b23d-b20473b25eab", "content": [ { "type": "text", "text": "Hi" }, - { "type": "image", "image_url": "https://example.com/cat.jpg", "detail": "high" }, - { "type": "file", "filename": "notes.txt", "file_data": "Zm9vYmFy" } + { "type": "image", "image_url": "https://example.com/cat.jpg" }, + { "type": "local_image", "path": "notes.txt" } ], "parent_message_id": "67e55044-10b1-426f-9247-bb680e5fe0c8", "model": "o4-mini", @@ -414,7 +354,7 @@ mod tests { cursor: Some("abc".into()), }); - let observed = to_val(&req.into_request(2)); + let observed = to_val(&req.into_request(RequestId::Integer(2))); let expected = json!({ "jsonrpc": "2.0", "id": 2, @@ -436,7 +376,7 @@ mod tests { conversation_id: ConversationId(uuid!("67e55044-10b1-426f-9247-bb680e5fe0c8")), }); - let observed = to_val(&req.into_request(2)); + let observed = to_val(&req.into_request(mcp_types::RequestId::Integer(2))); let expected = json!({ "jsonrpc": "2.0", "id": 2, @@ -454,48 +394,44 @@ mod tests { // ----- Message inputs / sources ----- #[test] - fn serialize_message_input_image_file_id_auto_detail() { - let item = MessageInputItem::Image { - source: ImageSource::FileId { - file_id: "file_123".into(), - }, - detail: Some(ImageDetail::Auto), + fn serialize_message_input_image_url() { + let item = InputItem::Image { + image_url: "https://example.com/x.png".into(), }; let observed = to_val(&item); let expected = json!({ "type": "image", - "file_id": "file_123", - "detail": "auto" + "image_url": "https://example.com/x.png" }); assert_eq!(observed, expected); } #[test] - fn serialize_message_input_file_url_and_id_variants() { - let url = MessageInputItem::File { - source: FileSource::Url { - file_url: "https://example.com/a.pdf".into(), - }, + fn serialize_message_input_local_image_path() { + let url = InputItem::LocalImage { + path: PathBuf::from("https://example.com/a.pdf"), }; - let id = MessageInputItem::File { - source: FileSource::Id { - file_id: "file_456".into(), - }, + let id = InputItem::LocalImage { + path: PathBuf::from("file_456"), }; + let observed_url = to_val(&url); + let expected_url = json!({"type":"local_image","path":"https://example.com/a.pdf"}); assert_eq!( - to_val(&url), - json!({"type":"file","file_url":"https://example.com/a.pdf"}) + observed_url, expected_url, + "LocalImage with URL path should serialize as image_url" + ); + let observed_id = to_val(&id); + let expected_id = json!({"type":"local_image","path":"file_456"}); + assert_eq!( + observed_id, expected_id, + "LocalImage with file id should serialize as image_url" ); - assert_eq!(to_val(&id), json!({"type":"file","file_id":"file_456"})); } #[test] fn serialize_message_input_image_url_without_detail() { - let item = MessageInputItem::Image { - source: ImageSource::ImageUrl { - image_url: "https://example.com/x.png".into(), - }, - detail: None, + let item = InputItem::Image { + image_url: "https://example.com/x.png".into(), }; let observed = to_val(&item); let expected = json!({ From e0e245cc1c2527d4ab788cc4b954d223112553cd Mon Sep 17 00:00:00 2001 From: pakrym-oai Date: Wed, 30 Jul 2025 13:56:24 -0700 Subject: [PATCH 007/100] Send AGENTS.md as a separate user message (#1737) --- codex-rs/core/src/chat_completions.rs | 4 ++ codex-rs/core/src/client.rs | 13 +++++- codex-rs/core/src/client_common.rs | 19 ++++++-- codex-rs/core/tests/client.rs | 64 +++++++++++++++++++++++++++ 4 files changed, 96 insertions(+), 4 deletions(-) diff --git a/codex-rs/core/src/chat_completions.rs b/codex-rs/core/src/chat_completions.rs index 3042ec452a..5ede774b1c 100644 --- a/codex-rs/core/src/chat_completions.rs +++ b/codex-rs/core/src/chat_completions.rs @@ -40,6 +40,10 @@ pub(crate) async fn stream_chat_completions( let full_instructions = prompt.get_full_instructions(model); messages.push(json!({"role": "system", "content": full_instructions})); + if let Some(instr) = &prompt.user_instructions { + messages.push(json!({"role": "user", "content": instr})); + } + for item in &prompt.input { match item { ResponseItem::Message { role, content, .. } => { diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index 72104da254..4e0e62c0f7 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -35,6 +35,7 @@ use crate::error::Result; use crate::flags::CODEX_RS_SSE_FIXTURE; use crate::model_provider_info::ModelProviderInfo; use crate::model_provider_info::WireApi; +use crate::models::ContentItem; use crate::models::ResponseItem; use crate::openai_tools::create_tools_json_for_responses_api; use crate::protocol::TokenUsage; @@ -156,10 +157,20 @@ impl ModelClient { vec![] }; + let mut input_with_instructions = Vec::with_capacity(prompt.input.len() + 1); + if let Some(ui) = &prompt.user_instructions { + input_with_instructions.push(ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::InputText { text: ui.clone() }], + }); + } + input_with_instructions.extend(prompt.input.clone()); + let payload = ResponsesApiRequest { model: &self.config.model, instructions: &full_instructions, - input: &prompt.input, + input: &input_with_instructions, tools: &tools_json, tool_choice: "auto", parallel_tool_calls: false, diff --git a/codex-rs/core/src/client_common.rs b/codex-rs/core/src/client_common.rs index afd2f04556..157f35872a 100644 --- a/codex-rs/core/src/client_common.rs +++ b/codex-rs/core/src/client_common.rs @@ -44,9 +44,6 @@ impl Prompt { .as_deref() .unwrap_or(BASE_INSTRUCTIONS); let mut sections: Vec<&str> = vec![base]; - if let Some(ref user) = self.user_instructions { - sections.push(user); - } if model.starts_with("gpt-4.1") { sections.push(APPLY_PATCH_TOOL_INSTRUCTIONS); } @@ -188,3 +185,19 @@ impl Stream for ResponseStream { self.rx_event.poll_recv(cx) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn get_full_instructions_no_user_content() { + let prompt = Prompt { + user_instructions: Some("custom instruction".to_string()), + ..Default::default() + }; + let expected = format!("{BASE_INSTRUCTIONS}\n{APPLY_PATCH_TOOL_INSTRUCTIONS}"); + let full = prompt.get_full_instructions("gpt-4.1"); + assert_eq!(full, expected); + } +} diff --git a/codex-rs/core/tests/client.rs b/codex-rs/core/tests/client.rs index fbe63fb326..67d95cb8f6 100644 --- a/codex-rs/core/tests/client.rs +++ b/codex-rs/core/tests/client.rs @@ -257,6 +257,70 @@ async fn chatgpt_auth_sends_correct_request() { ); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn includes_user_instructions_message_in_request() { + #![allow(clippy::unwrap_used)] + + let server = MockServer::start().await; + + let first = ResponseTemplate::new(200) + .insert_header("content-type", "text/event-stream") + .set_body_raw(sse_completed("resp1"), "text/event-stream"); + + Mock::given(method("POST")) + .and(path("/v1/responses")) + .respond_with(first) + .expect(1) + .mount(&server) + .await; + + let model_provider = ModelProviderInfo { + base_url: Some(format!("{}/v1", server.uri())), + ..built_in_model_providers()["openai"].clone() + }; + + let codex_home = TempDir::new().unwrap(); + let mut config = load_default_config_for_test(&codex_home); + config.model_provider = model_provider; + config.user_instructions = Some("be nice".to_string()); + + let ctrl_c = std::sync::Arc::new(tokio::sync::Notify::new()); + let CodexSpawnOk { codex, .. } = Codex::spawn( + config, + Some(CodexAuth::from_api_key("Test API Key".to_string())), + ctrl_c.clone(), + ) + .await + .unwrap(); + + codex + .submit(Op::UserInput { + items: vec![InputItem::Text { + text: "hello".into(), + }], + }) + .await + .unwrap(); + + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TaskComplete(_))).await; + + let request = &server.received_requests().await.unwrap()[0]; + let request_body = request.body_json::().unwrap(); + + assert!( + !request_body["instructions"] + .as_str() + .unwrap() + .contains("be nice") + ); + assert_eq!(request_body["input"][0]["role"], "user"); + assert!( + request_body["input"][0]["content"][0]["text"] + .as_str() + .unwrap() + .starts_with("be nice") + ); +} fn auth_from_token(id_token: String) -> CodexAuth { CodexAuth::new( None, From 301ec72107f107ac5c6103660e276a25bc18273e Mon Sep 17 00:00:00 2001 From: pakrym-oai Date: Wed, 30 Jul 2025 14:09:26 -0700 Subject: [PATCH 008/100] Add login status command (#1716) Print the current login mode, sanitized key and return an appropriate status. --- README.md | 6 +++ codex-rs/cli/src/login.rs | 93 ++++++++++++++++++++++++++++++++------- codex-rs/cli/src/main.rs | 21 ++++++++- 3 files changed, 102 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 60e44298a3..c7f6a1d595 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,12 @@ codex login If you complete the process successfully, you should have a `~/.codex/auth.json` file that contains the credentials that Codex will use. +To verify whether you are currently logged in, run: + +``` +codex login status +``` + If you encounter problems with the login flow, please comment on .
diff --git a/codex-rs/cli/src/login.rs b/codex-rs/cli/src/login.rs index af3fb667f6..390c310030 100644 --- a/codex-rs/cli/src/login.rs +++ b/codex-rs/cli/src/login.rs @@ -1,25 +1,12 @@ use codex_common::CliConfigOverrides; use codex_core::config::Config; use codex_core::config::ConfigOverrides; +use codex_login::AuthMode; +use codex_login::load_auth; use codex_login::login_with_chatgpt; pub async fn run_login_with_chatgpt(cli_config_overrides: CliConfigOverrides) -> ! { - let cli_overrides = match cli_config_overrides.parse_overrides() { - Ok(v) => v, - Err(e) => { - eprintln!("Error parsing -c overrides: {e}"); - std::process::exit(1); - } - }; - - let config_overrides = ConfigOverrides::default(); - let config = match Config::load_with_cli_overrides(cli_overrides, config_overrides) { - Ok(config) => config, - Err(e) => { - eprintln!("Error loading configuration: {e}"); - std::process::exit(1); - } - }; + let config = load_config_or_exit(cli_config_overrides); let capture_output = false; match login_with_chatgpt(&config.codex_home, capture_output).await { @@ -33,3 +20,77 @@ pub async fn run_login_with_chatgpt(cli_config_overrides: CliConfigOverrides) -> } } } + +pub async fn run_login_status(cli_config_overrides: CliConfigOverrides) -> ! { + let config = load_config_or_exit(cli_config_overrides); + + match load_auth(&config.codex_home) { + Ok(Some(auth)) => match auth.mode { + AuthMode::ApiKey => { + if let Some(api_key) = auth.api_key.as_deref() { + eprintln!("Logged in using an API key - {}", safe_format_key(api_key)); + } else { + eprintln!("Logged in using an API key"); + } + std::process::exit(0); + } + AuthMode::ChatGPT => { + eprintln!("Logged in using ChatGPT"); + std::process::exit(0); + } + }, + Ok(None) => { + eprintln!("Not logged in"); + std::process::exit(1); + } + Err(e) => { + eprintln!("Error checking login status: {e}"); + std::process::exit(1); + } + } +} + +fn load_config_or_exit(cli_config_overrides: CliConfigOverrides) -> Config { + let cli_overrides = match cli_config_overrides.parse_overrides() { + Ok(v) => v, + Err(e) => { + eprintln!("Error parsing -c overrides: {e}"); + std::process::exit(1); + } + }; + + let config_overrides = ConfigOverrides::default(); + match Config::load_with_cli_overrides(cli_overrides, config_overrides) { + Ok(config) => config, + Err(e) => { + eprintln!("Error loading configuration: {e}"); + std::process::exit(1); + } + } +} + +fn safe_format_key(key: &str) -> String { + if key.len() <= 13 { + return "***".to_string(); + } + let prefix = &key[..8]; + let suffix = &key[key.len() - 5..]; + format!("{prefix}***{suffix}") +} + +#[cfg(test)] +mod tests { + use super::safe_format_key; + + #[test] + fn formats_long_key() { + let key = "sk-proj-1234567890ABCDE"; + assert_eq!(safe_format_key(key), "sk-proj-***ABCDE"); + } + + #[test] + fn short_key_returns_stars() { + let key = "sk-proj-12345"; + assert_eq!(safe_format_key(key), "***"); + } +} diff --git a/codex-rs/cli/src/main.rs b/codex-rs/cli/src/main.rs index 6dd596ff9f..c5fd69f9cd 100644 --- a/codex-rs/cli/src/main.rs +++ b/codex-rs/cli/src/main.rs @@ -7,6 +7,7 @@ use codex_chatgpt::apply_command::ApplyCommand; use codex_chatgpt::apply_command::run_apply_command; use codex_cli::LandlockCommand; use codex_cli::SeatbeltCommand; +use codex_cli::login::run_login_status; use codex_cli::login::run_login_with_chatgpt; use codex_cli::proto; use codex_common::CliConfigOverrides; @@ -43,7 +44,7 @@ enum Subcommand { #[clap(visible_alias = "e")] Exec(ExecCli), - /// Login with ChatGPT. + /// Manage login. Login(LoginCommand), /// Experimental: run Codex as an MCP server. @@ -90,6 +91,15 @@ enum DebugCommand { struct LoginCommand { #[clap(skip)] config_overrides: CliConfigOverrides, + + #[command(subcommand)] + action: Option, +} + +#[derive(Debug, clap::Subcommand)] +enum LoginSubcommand { + /// Show login status. + Status, } fn main() -> anyhow::Result<()> { @@ -118,7 +128,14 @@ async fn cli_main(codex_linux_sandbox_exe: Option) -> anyhow::Result<() } Some(Subcommand::Login(mut login_cli)) => { prepend_config_flags(&mut login_cli.config_overrides, cli.config_overrides); - run_login_with_chatgpt(login_cli.config_overrides).await; + match login_cli.action { + Some(LoginSubcommand::Status) => { + run_login_status(login_cli.config_overrides).await; + } + None => { + run_login_with_chatgpt(login_cli.config_overrides).await; + } + } } Some(Subcommand::Proto(mut proto_cli)) => { prepend_config_flags(&mut proto_cli.config_overrides, cli.config_overrides); From 221ebfcccc3ddd957d5903b8c09e4d02e62387e7 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Wed, 30 Jul 2025 16:45:08 -0700 Subject: [PATCH 009/100] fix: run apply_patch calls through the sandbox (#1705) Building on the work of https://github.com/openai/codex/pull/1702, this changes how a shell call to `apply_patch` is handled. Previously, a shell call to `apply_patch` was always handled in-process, never leveraging a sandbox. To determine whether the `apply_patch` operation could be auto-approved, the `is_write_patch_constrained_to_writable_paths()` function would check if all the paths listed in the paths were writable. If so, the agent would apply the changes listed in the patch. Unfortunately, this approach afforded a loophole: symlinks! * For a soft link, we could fix this issue by tracing the link and checking whether the target is in the set of writable paths, however... * ...For a hard link, things are not as simple. We can run `stat FILE` to see if the number of links is greater than 1, but then we would have to do something potentially expensive like `find . -inum ` to find the other paths for `FILE`. Further, even if this worked, this approach runs the risk of a [TOCTOU](https://en.wikipedia.org/wiki/Time-of-check_to_time-of-use) race condition, so it is not robust. The solution, implemented in this PR, is to take the virtual execution of the `apply_patch` CLI into an _actual_ execution using `codex --codex-run-as-apply-patch PATCH`, which we can run under the sandbox the user specified, just like any other `shell` call. This, of course, assumes that the sandbox prevents writing through symlinks as a mechanism to write to folders that are not in the writable set configured by the sandbox. I verified this by testing the following on both Mac and Linux: ```shell #!/usr/bin/env bash set -euo pipefail # Can running a command in SANDBOX_DIR write a file in EXPLOIT_DIR? # Codex is run in SANDBOX_DIR, so writes should be constrianed to this directory. SANDBOX_DIR=$(mktemp -d -p "$HOME" sandboxtesttemp.XXXXXX) # EXPLOIT_DIR is outside of SANDBOX_DIR, so let's see if we can write to it. EXPLOIT_DIR=$(mktemp -d -p "$HOME" sandboxtesttemp.XXXXXX) echo "SANDBOX_DIR: $SANDBOX_DIR" echo "EXPLOIT_DIR: $EXPLOIT_DIR" cleanup() { # Only remove if it looks sane and still exists [[ -n "${SANDBOX_DIR:-}" && -d "$SANDBOX_DIR" ]] && rm -rf -- "$SANDBOX_DIR" [[ -n "${EXPLOIT_DIR:-}" && -d "$EXPLOIT_DIR" ]] && rm -rf -- "$EXPLOIT_DIR" } trap cleanup EXIT echo "I am the original content" > "${EXPLOIT_DIR}/original.txt" # Drop the -s to test hard links. ln -s "${EXPLOIT_DIR}/original.txt" "${SANDBOX_DIR}/link-to-original.txt" cat "${SANDBOX_DIR}/link-to-original.txt" if [[ "$(uname)" == "Linux" ]]; then SANDBOX_SUBCOMMAND=landlock else SANDBOX_SUBCOMMAND=seatbelt fi # Attempt the exploit cd "${SANDBOX_DIR}" codex debug "${SANDBOX_SUBCOMMAND}" bash -lc "echo pwned > ./link-to-original.txt" || true cat "${EXPLOIT_DIR}/original.txt" ``` Admittedly, this change merits a proper integration test, but I think I will have to do that in a follow-up PR. --- codex-rs/apply-patch/src/lib.rs | 72 ++++++++++++---- codex-rs/apply-patch/src/parser.rs | 54 ++++++++---- codex-rs/arg0/src/lib.rs | 6 +- codex-rs/core/src/apply_patch.rs | 74 +++++++++++----- codex-rs/core/src/codex.rs | 131 +++++++++++++++++++++-------- codex-rs/core/src/lib.rs | 1 + codex-rs/core/src/safety.rs | 13 ++- codex-rs/exec/tests/apply_patch.rs | 3 +- 8 files changed, 257 insertions(+), 97 deletions(-) diff --git a/codex-rs/apply-patch/src/lib.rs b/codex-rs/apply-patch/src/lib.rs index c81241d0da..8d42be9a92 100644 --- a/codex-rs/apply-patch/src/lib.rs +++ b/codex-rs/apply-patch/src/lib.rs @@ -58,16 +58,24 @@ impl PartialEq for IoError { #[derive(Debug, PartialEq)] pub enum MaybeApplyPatch { - Body(Vec), + Body(ApplyPatchArgs), ShellParseError(ExtractHeredocError), PatchParseError(ParseError), NotApplyPatch, } +/// Both the raw PATCH argument to `apply_patch` as well as the PATCH argument +/// parsed into hunks. +#[derive(Debug, PartialEq)] +pub struct ApplyPatchArgs { + pub patch: String, + pub hunks: Vec, +} + pub fn maybe_parse_apply_patch(argv: &[String]) -> MaybeApplyPatch { match argv { [cmd, body] if cmd == "apply_patch" => match parse_patch(body) { - Ok(hunks) => MaybeApplyPatch::Body(hunks), + Ok(source) => MaybeApplyPatch::Body(source), Err(e) => MaybeApplyPatch::PatchParseError(e), }, [bash, flag, script] @@ -77,7 +85,7 @@ pub fn maybe_parse_apply_patch(argv: &[String]) -> MaybeApplyPatch { { match extract_heredoc_body_from_apply_patch_command(script) { Ok(body) => match parse_patch(&body) { - Ok(hunks) => MaybeApplyPatch::Body(hunks), + Ok(source) => MaybeApplyPatch::Body(source), Err(e) => MaybeApplyPatch::PatchParseError(e), }, Err(e) => MaybeApplyPatch::ShellParseError(e), @@ -116,11 +124,19 @@ pub enum MaybeApplyPatchVerified { NotApplyPatch, } -#[derive(Debug, PartialEq)] /// ApplyPatchAction is the result of parsing an `apply_patch` command. By /// construction, all paths should be absolute paths. +#[derive(Debug, PartialEq)] pub struct ApplyPatchAction { changes: HashMap, + + /// The raw patch argument that can be used with `apply_patch` as an exec + /// call. i.e., if the original arg was parsed in "lenient" mode with a + /// heredoc, this should be the value without the heredoc wrapper. + pub patch: String, + + /// The working directory that was used to resolve relative paths in the patch. + pub cwd: PathBuf, } impl ApplyPatchAction { @@ -140,8 +156,28 @@ impl ApplyPatchAction { panic!("path must be absolute"); } + #[allow(clippy::expect_used)] + let filename = path + .file_name() + .expect("path should not be empty") + .to_string_lossy(); + let patch = format!( + r#"*** Begin Patch +*** Update File: {filename} +@@ ++ {content} +*** End Patch"#, + ); let changes = HashMap::from([(path.to_path_buf(), ApplyPatchFileChange::Add { content })]); - Self { changes } + #[allow(clippy::expect_used)] + Self { + changes, + cwd: path + .parent() + .expect("path should have parent") + .to_path_buf(), + patch, + } } } @@ -149,7 +185,7 @@ impl ApplyPatchAction { /// patch. pub fn maybe_parse_apply_patch_verified(argv: &[String], cwd: &Path) -> MaybeApplyPatchVerified { match maybe_parse_apply_patch(argv) { - MaybeApplyPatch::Body(hunks) => { + MaybeApplyPatch::Body(ApplyPatchArgs { patch, hunks }) => { let mut changes = HashMap::new(); for hunk in hunks { let path = hunk.resolve_path(cwd); @@ -183,7 +219,11 @@ pub fn maybe_parse_apply_patch_verified(argv: &[String], cwd: &Path) -> MaybeApp } } } - MaybeApplyPatchVerified::Body(ApplyPatchAction { changes }) + MaybeApplyPatchVerified::Body(ApplyPatchAction { + changes, + patch, + cwd: cwd.to_path_buf(), + }) } MaybeApplyPatch::ShellParseError(e) => MaybeApplyPatchVerified::ShellParseError(e), MaybeApplyPatch::PatchParseError(e) => MaybeApplyPatchVerified::CorrectnessError(e.into()), @@ -264,7 +304,7 @@ pub fn apply_patch( stderr: &mut impl std::io::Write, ) -> Result<(), ApplyPatchError> { let hunks = match parse_patch(patch) { - Ok(hunks) => hunks, + Ok(source) => source.hunks, Err(e) => { match &e { InvalidPatchError(message) => { @@ -652,7 +692,7 @@ mod tests { ]); match maybe_parse_apply_patch(&args) { - MaybeApplyPatch::Body(hunks) => { + MaybeApplyPatch::Body(ApplyPatchArgs { hunks, patch: _ }) => { assert_eq!( hunks, vec![Hunk::AddFile { @@ -679,7 +719,7 @@ PATCH"#, ]); match maybe_parse_apply_patch(&args) { - MaybeApplyPatch::Body(hunks) => { + MaybeApplyPatch::Body(ApplyPatchArgs { hunks, patch: _ }) => { assert_eq!( hunks, vec![Hunk::AddFile { @@ -954,7 +994,7 @@ PATCH"#, )); let patch = parse_patch(&patch).unwrap(); - let update_file_chunks = match patch.as_slice() { + let update_file_chunks = match patch.hunks.as_slice() { [Hunk::UpdateFile { chunks, .. }] => chunks, _ => panic!("Expected a single UpdateFile hunk"), }; @@ -992,7 +1032,7 @@ PATCH"#, )); let patch = parse_patch(&patch).unwrap(); - let chunks = match patch.as_slice() { + let chunks = match patch.hunks.as_slice() { [Hunk::UpdateFile { chunks, .. }] => chunks, _ => panic!("Expected a single UpdateFile hunk"), }; @@ -1029,7 +1069,7 @@ PATCH"#, )); let patch = parse_patch(&patch).unwrap(); - let chunks = match patch.as_slice() { + let chunks = match patch.hunks.as_slice() { [Hunk::UpdateFile { chunks, .. }] => chunks, _ => panic!("Expected a single UpdateFile hunk"), }; @@ -1064,7 +1104,7 @@ PATCH"#, )); let patch = parse_patch(&patch).unwrap(); - let chunks = match patch.as_slice() { + let chunks = match patch.hunks.as_slice() { [Hunk::UpdateFile { chunks, .. }] => chunks, _ => panic!("Expected a single UpdateFile hunk"), }; @@ -1110,7 +1150,7 @@ PATCH"#, // Extract chunks then build the unified diff. let parsed = parse_patch(&patch).unwrap(); - let chunks = match parsed.as_slice() { + let chunks = match parsed.hunks.as_slice() { [Hunk::UpdateFile { chunks, .. }] => chunks, _ => panic!("Expected a single UpdateFile hunk"), }; @@ -1193,6 +1233,8 @@ g new_content: "updated session directory content\n".to_string(), }, )]), + patch: argv[1].clone(), + cwd: session_dir.path().to_path_buf(), }) ); } diff --git a/codex-rs/apply-patch/src/parser.rs b/codex-rs/apply-patch/src/parser.rs index d07691a49d..44c5b14619 100644 --- a/codex-rs/apply-patch/src/parser.rs +++ b/codex-rs/apply-patch/src/parser.rs @@ -22,6 +22,7 @@ //! //! The parser below is a little more lenient than the explicit spec and allows for //! leading/trailing whitespace around patch markers. +use crate::ApplyPatchArgs; use std::path::Path; use std::path::PathBuf; @@ -102,7 +103,7 @@ pub struct UpdateFileChunk { pub is_end_of_file: bool, } -pub fn parse_patch(patch: &str) -> Result, ParseError> { +pub fn parse_patch(patch: &str) -> Result { let mode = if PARSE_IN_STRICT_MODE { ParseMode::Strict } else { @@ -150,7 +151,7 @@ enum ParseMode { Lenient, } -fn parse_patch_text(patch: &str, mode: ParseMode) -> Result, ParseError> { +fn parse_patch_text(patch: &str, mode: ParseMode) -> Result { let lines: Vec<&str> = patch.trim().lines().collect(); let lines: &[&str] = match check_patch_boundaries_strict(&lines) { Ok(()) => &lines, @@ -173,7 +174,8 @@ fn parse_patch_text(patch: &str, mode: ParseMode) -> Result, ParseErro line_number += hunk_lines; remaining_lines = &remaining_lines[hunk_lines..] } - Ok(hunks) + let patch = lines.join("\n"); + Ok(ApplyPatchArgs { hunks, patch }) } /// Checks the start and end lines of the patch text for `apply_patch`, @@ -425,6 +427,7 @@ fn parse_update_file_chunk( } #[test] +#[allow(clippy::unwrap_used)] fn test_parse_patch() { assert_eq!( parse_patch_text("bad", ParseMode::Strict), @@ -455,8 +458,10 @@ fn test_parse_patch() { "*** Begin Patch\n\ *** End Patch", ParseMode::Strict - ), - Ok(Vec::new()) + ) + .unwrap() + .hunks, + Vec::new() ); assert_eq!( parse_patch_text( @@ -472,8 +477,10 @@ fn test_parse_patch() { + return 123\n\ *** End Patch", ParseMode::Strict - ), - Ok(vec![ + ) + .unwrap() + .hunks, + vec![ AddFile { path: PathBuf::from("path/add.py"), contents: "abc\ndef\n".to_string() @@ -491,7 +498,7 @@ fn test_parse_patch() { is_end_of_file: false }] } - ]) + ] ); // Update hunk followed by another hunk (Add File). assert_eq!( @@ -504,8 +511,10 @@ fn test_parse_patch() { +content\n\ *** End Patch", ParseMode::Strict - ), - Ok(vec![ + ) + .unwrap() + .hunks, + vec![ UpdateFile { path: PathBuf::from("file.py"), move_path: None, @@ -520,7 +529,7 @@ fn test_parse_patch() { path: PathBuf::from("other.py"), contents: "content\n".to_string() } - ]) + ] ); // Update hunk without an explicit @@ header for the first chunk should parse. @@ -533,8 +542,10 @@ fn test_parse_patch() { +bar *** End Patch"#, ParseMode::Strict - ), - Ok(vec![UpdateFile { + ) + .unwrap() + .hunks, + vec![UpdateFile { path: PathBuf::from("file2.py"), move_path: None, chunks: vec![UpdateFileChunk { @@ -543,7 +554,7 @@ fn test_parse_patch() { new_lines: vec!["import foo".to_string(), "bar".to_string()], is_end_of_file: false, }], - }]) + }] ); } @@ -574,7 +585,10 @@ fn test_parse_patch_lenient() { ); assert_eq!( parse_patch_text(&patch_text_in_heredoc, ParseMode::Lenient), - Ok(expected_patch.clone()) + Ok(ApplyPatchArgs { + hunks: expected_patch.clone(), + patch: patch_text.to_string() + }) ); let patch_text_in_single_quoted_heredoc = format!("<<'EOF'\n{patch_text}\nEOF\n"); @@ -584,7 +598,10 @@ fn test_parse_patch_lenient() { ); assert_eq!( parse_patch_text(&patch_text_in_single_quoted_heredoc, ParseMode::Lenient), - Ok(expected_patch.clone()) + Ok(ApplyPatchArgs { + hunks: expected_patch.clone(), + patch: patch_text.to_string() + }) ); let patch_text_in_double_quoted_heredoc = format!("<<\"EOF\"\n{patch_text}\nEOF\n"); @@ -594,7 +611,10 @@ fn test_parse_patch_lenient() { ); assert_eq!( parse_patch_text(&patch_text_in_double_quoted_heredoc, ParseMode::Lenient), - Ok(expected_patch.clone()) + Ok(ApplyPatchArgs { + hunks: expected_patch.clone(), + patch: patch_text.to_string() + }) ); let patch_text_in_mismatched_quotes_heredoc = format!("<<\"EOF'\n{patch_text}\nEOF\n"); diff --git a/codex-rs/arg0/src/lib.rs b/codex-rs/arg0/src/lib.rs index d7109176a5..c097ebc11c 100644 --- a/codex-rs/arg0/src/lib.rs +++ b/codex-rs/arg0/src/lib.rs @@ -2,6 +2,8 @@ use std::future::Future; use std::path::Path; use std::path::PathBuf; +use codex_core::CODEX_APPLY_PATCH_ARG1; + /// While we want to deploy the Codex CLI as a single executable for simplicity, /// we also want to expose some of its functionality as distinct CLIs, so we use /// the "arg0 trick" to determine which CLI to dispatch. This effectively allows @@ -43,7 +45,7 @@ where } let argv1 = args.next().unwrap_or_default(); - if argv1 == "--codex-run-as-apply-patch" { + if argv1 == CODEX_APPLY_PATCH_ARG1 { let patch_arg = args.next().and_then(|s| s.to_str().map(|s| s.to_owned())); let exit_code = match patch_arg { Some(patch_arg) => { @@ -55,7 +57,7 @@ where } } None => { - eprintln!("Error: --codex-run-as-apply-patch requires a UTF-8 PATCH argument."); + eprintln!("Error: {CODEX_APPLY_PATCH_ARG1} requires a UTF-8 PATCH argument."); 1 } }; diff --git a/codex-rs/core/src/apply_patch.rs b/codex-rs/core/src/apply_patch.rs index 44af72c746..f116c790ab 100644 --- a/codex-rs/core/src/apply_patch.rs +++ b/codex-rs/core/src/apply_patch.rs @@ -18,12 +18,34 @@ use std::collections::HashMap; use std::path::Path; use std::path::PathBuf; +pub const CODEX_APPLY_PATCH_ARG1: &str = "--codex-run-as-apply-patch"; + +pub(crate) enum InternalApplyPatchInvocation { + /// The `apply_patch` call was handled programmatically, without any sort + /// of sandbox, because the user explicitly approved it. This is the + /// result to use with the `shell` function call that contained `apply_patch`. + Output(ResponseInputItem), + + /// The `apply_patch` call was auto-approved, which means that, on the + /// surface, it appears to be safe, but it should be run in a sandbox if the + /// user has configured one because a path being written could be a hard + /// link to a file outside the writable folders, so only the sandbox can + /// faithfully prevent the write in that case. + DelegateToExec(ApplyPatchAction), +} + +impl From for InternalApplyPatchInvocation { + fn from(item: ResponseInputItem) -> Self { + InternalApplyPatchInvocation::Output(item) + } +} + pub(crate) async fn apply_patch( sess: &Session, - sub_id: String, - call_id: String, + sub_id: &str, + call_id: &str, action: ApplyPatchAction, -) -> ResponseInputItem { +) -> InternalApplyPatchInvocation { let writable_roots_snapshot = { #[allow(clippy::unwrap_used)] let guard = sess.writable_roots.lock().unwrap(); @@ -36,34 +58,38 @@ pub(crate) async fn apply_patch( &writable_roots_snapshot, &sess.cwd, ) { - SafetyCheck::AutoApprove { .. } => true, + SafetyCheck::AutoApprove { .. } => { + return InternalApplyPatchInvocation::DelegateToExec(action); + } SafetyCheck::AskUser => { // Compute a readable summary of path changes to include in the // approval request so the user can make an informed decision. let rx_approve = sess - .request_patch_approval(sub_id.clone(), call_id.clone(), &action, None, None) + .request_patch_approval(sub_id.to_owned(), call_id.to_owned(), &action, None, None) .await; match rx_approve.await.unwrap_or_default() { ReviewDecision::Approved | ReviewDecision::ApprovedForSession => false, ReviewDecision::Denied | ReviewDecision::Abort => { return ResponseInputItem::FunctionCallOutput { - call_id, + call_id: call_id.to_owned(), output: FunctionCallOutputPayload { content: "patch rejected by user".to_string(), success: Some(false), }, - }; + } + .into(); } } } SafetyCheck::Reject { reason } => { return ResponseInputItem::FunctionCallOutput { - call_id, + call_id: call_id.to_owned(), output: FunctionCallOutputPayload { content: format!("patch rejected: {reason}"), success: Some(false), }, - }; + } + .into(); } }; @@ -83,8 +109,8 @@ pub(crate) async fn apply_patch( let rx = sess .request_patch_approval( - sub_id.clone(), - call_id.clone(), + sub_id.to_owned(), + call_id.to_owned(), &action, reason.clone(), Some(root.clone()), @@ -96,12 +122,13 @@ pub(crate) async fn apply_patch( ReviewDecision::Approved | ReviewDecision::ApprovedForSession ) { return ResponseInputItem::FunctionCallOutput { - call_id, + call_id: call_id.to_owned(), output: FunctionCallOutputPayload { content: "patch rejected by user".to_string(), success: Some(false), }, - }; + } + .into(); } // user approved, extend writable roots for this session @@ -112,9 +139,9 @@ pub(crate) async fn apply_patch( let _ = sess .tx_event .send(Event { - id: sub_id.clone(), + id: sub_id.to_owned(), msg: EventMsg::PatchApplyBegin(PatchApplyBeginEvent { - call_id: call_id.clone(), + call_id: call_id.to_owned(), auto_approved, changes: convert_apply_patch_to_protocol(&action), }), @@ -173,8 +200,8 @@ pub(crate) async fn apply_patch( )); let rx = sess .request_patch_approval( - sub_id.clone(), - call_id.clone(), + sub_id.to_owned(), + call_id.to_owned(), &action, reason.clone(), Some(root.clone()), @@ -204,9 +231,9 @@ pub(crate) async fn apply_patch( let _ = sess .tx_event .send(Event { - id: sub_id.clone(), + id: sub_id.to_owned(), msg: EventMsg::PatchApplyEnd(PatchApplyEndEvent { - call_id: call_id.clone(), + call_id: call_id.to_owned(), stdout: String::from_utf8_lossy(&stdout).to_string(), stderr: String::from_utf8_lossy(&stderr).to_string(), success: success_flag, @@ -214,22 +241,23 @@ pub(crate) async fn apply_patch( }) .await; - match result { + let item = match result { Ok(_) => ResponseInputItem::FunctionCallOutput { - call_id, + call_id: call_id.to_owned(), output: FunctionCallOutputPayload { content: String::from_utf8_lossy(&stdout).to_string(), success: None, }, }, Err(e) => ResponseInputItem::FunctionCallOutput { - call_id, + call_id: call_id.to_owned(), output: FunctionCallOutputPayload { content: format!("error: {e:#}, stderr: {}", String::from_utf8_lossy(&stderr)), success: Some(false), }, }, - } + }; + InternalApplyPatchInvocation::Output(item) } /// Return the first path in `hunks` that is NOT under any of the diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 92ca7bf88b..cd92739c72 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -4,6 +4,7 @@ use std::borrow::Cow; use std::collections::HashMap; use std::collections::HashSet; +use std::path::Path; use std::path::PathBuf; use std::sync::Arc; use std::sync::Mutex; @@ -30,6 +31,8 @@ use tracing::trace; use tracing::warn; use uuid::Uuid; +use crate::apply_patch::CODEX_APPLY_PATCH_ARG1; +use crate::apply_patch::InternalApplyPatchInvocation; use crate::apply_patch::convert_apply_patch_to_protocol; use crate::apply_patch::get_writable_roots; use crate::apply_patch::{self}; @@ -81,6 +84,7 @@ use crate::protocol::TaskCompleteEvent; use crate::rollout::RolloutRecorder; use crate::safety::SafetyCheck; use crate::safety::assess_command_safety; +use crate::safety::assess_safety_for_untrusted_command; use crate::shell; use crate::user_notification::UserNotification; use crate::util::backoff; @@ -354,13 +358,19 @@ impl Session { } } - async fn notify_exec_command_begin(&self, sub_id: &str, call_id: &str, params: &ExecParams) { + async fn notify_exec_command_begin( + &self, + sub_id: &str, + call_id: &str, + command_for_display: Vec, + command_cwd: &Path, + ) { let event = Event { id: sub_id.to_string(), msg: EventMsg::ExecCommandBegin(ExecCommandBeginEvent { call_id: call_id.to_string(), - command: params.command.clone(), - cwd: params.cwd.clone(), + command: command_for_display, + cwd: command_cwd.to_path_buf(), }), }; let _ = self.tx_event.send(event).await; @@ -1420,38 +1430,77 @@ async fn handle_container_exec_with_params( call_id: String, ) -> ResponseInputItem { // check if this was a patch, and apply it if so - match maybe_parse_apply_patch_verified(¶ms.command, ¶ms.cwd) { - MaybeApplyPatchVerified::Body(changes) => { - return apply_patch::apply_patch(sess, sub_id, call_id, changes).await; - } - MaybeApplyPatchVerified::CorrectnessError(parse_error) => { - // It looks like an invocation of `apply_patch`, but we - // could not resolve it into a patch that would apply - // cleanly. Return to model for resample. - return ResponseInputItem::FunctionCallOutput { - call_id, - output: FunctionCallOutputPayload { - content: format!("error: {parse_error:#}"), - success: None, - }, - }; - } - MaybeApplyPatchVerified::ShellParseError(error) => { - trace!("Failed to parse shell command, {error:?}"); - } - MaybeApplyPatchVerified::NotApplyPatch => (), - } + let apply_patch_action_for_exec = + match maybe_parse_apply_patch_verified(¶ms.command, ¶ms.cwd) { + MaybeApplyPatchVerified::Body(changes) => { + match apply_patch::apply_patch(sess, &sub_id, &call_id, changes).await { + InternalApplyPatchInvocation::Output(item) => return item, + InternalApplyPatchInvocation::DelegateToExec(action) => Some(action), + } + } + MaybeApplyPatchVerified::CorrectnessError(parse_error) => { + // It looks like an invocation of `apply_patch`, but we + // could not resolve it into a patch that would apply + // cleanly. Return to model for resample. + return ResponseInputItem::FunctionCallOutput { + call_id, + output: FunctionCallOutputPayload { + content: format!("error: {parse_error:#}"), + success: None, + }, + }; + } + MaybeApplyPatchVerified::ShellParseError(error) => { + trace!("Failed to parse shell command, {error:?}"); + None + } + MaybeApplyPatchVerified::NotApplyPatch => None, + }; - // safety checks - let safety = { - let state = sess.state.lock().unwrap(); - assess_command_safety( - ¶ms.command, - sess.approval_policy, - &sess.sandbox_policy, - &state.approved_commands, - ) + let (params, safety, command_for_display) = match apply_patch_action_for_exec { + Some(ApplyPatchAction { patch, cwd, .. }) => { + let path_to_codex = std::env::current_exe() + .ok() + .map(|p| p.to_string_lossy().to_string()); + let Some(path_to_codex) = path_to_codex else { + return ResponseInputItem::FunctionCallOutput { + call_id, + output: FunctionCallOutputPayload { + content: "failed to determine path to codex executable".to_string(), + success: None, + }, + }; + }; + + let params = ExecParams { + command: vec![ + path_to_codex, + CODEX_APPLY_PATCH_ARG1.to_string(), + patch.clone(), + ], + cwd, + timeout_ms: params.timeout_ms, + env: HashMap::new(), + }; + let safety = + assess_safety_for_untrusted_command(sess.approval_policy, &sess.sandbox_policy); + (params, safety, vec!["apply_patch".to_string(), patch]) + } + None => { + let safety = { + let state = sess.state.lock().unwrap(); + assess_command_safety( + ¶ms.command, + sess.approval_policy, + &sess.sandbox_policy, + &state.approved_commands, + ) + }; + let command_for_display = params.command.clone(); + (params, safety, command_for_display) + } }; + let sandbox_type = match safety { SafetyCheck::AutoApprove { sandbox_type } => sandbox_type, SafetyCheck::AskUser => { @@ -1496,7 +1545,7 @@ async fn handle_container_exec_with_params( } }; - sess.notify_exec_command_begin(&sub_id, &call_id, ¶ms) + sess.notify_exec_command_begin(&sub_id, &call_id, command_for_display.clone(), ¶ms.cwd) .await; let params = maybe_run_with_user_profile(params, sess); @@ -1537,7 +1586,16 @@ async fn handle_container_exec_with_params( } } Err(CodexErr::Sandbox(error)) => { - handle_sandbox_error(error, sandbox_type, params, sess, sub_id, call_id).await + handle_sandbox_error( + error, + sandbox_type, + params, + command_for_display, + sess, + sub_id, + call_id, + ) + .await } Err(e) => { // Handle non-sandbox errors @@ -1556,6 +1614,7 @@ async fn handle_sandbox_error( error: SandboxErr, sandbox_type: SandboxType, params: ExecParams, + command_for_display: Vec, sess: &Session, sub_id: String, call_id: String, @@ -1605,7 +1664,7 @@ async fn handle_sandbox_error( sess.notify_background_event(&sub_id, "retrying command without sandbox") .await; - sess.notify_exec_command_begin(&sub_id, &call_id, ¶ms) + sess.notify_exec_command_begin(&sub_id, &call_id, command_for_display, ¶ms.cwd) .await; // This is an escalated retry; the policy will not be diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index ffe64d7c90..054abd742a 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -43,4 +43,5 @@ pub mod shell; mod user_notification; pub mod util; +pub use apply_patch::CODEX_APPLY_PATCH_ARG1; pub use client_common::model_supports_reasoning_summaries; diff --git a/codex-rs/core/src/safety.rs b/codex-rs/core/src/safety.rs index 6a3ff29901..f9bc27e058 100644 --- a/codex-rs/core/src/safety.rs +++ b/codex-rs/core/src/safety.rs @@ -75,9 +75,6 @@ pub fn assess_command_safety( sandbox_policy: &SandboxPolicy, approved: &HashSet>, ) -> SafetyCheck { - use AskForApproval::*; - use SandboxPolicy::*; - // A command is "trusted" because either: // - it belongs to a set of commands we consider "safe" by default, or // - the user has explicitly approved the command for this session @@ -97,6 +94,16 @@ pub fn assess_command_safety( }; } + assess_safety_for_untrusted_command(approval_policy, sandbox_policy) +} + +pub(crate) fn assess_safety_for_untrusted_command( + approval_policy: AskForApproval, + sandbox_policy: &SandboxPolicy, +) -> SafetyCheck { + use AskForApproval::*; + use SandboxPolicy::*; + match (approval_policy, sandbox_policy) { (UnlessTrusted, _) => { // Even though the user may have opted into DangerFullAccess, diff --git a/codex-rs/exec/tests/apply_patch.rs b/codex-rs/exec/tests/apply_patch.rs index 69ac1b8c0a..f65d32e1c8 100644 --- a/codex-rs/exec/tests/apply_patch.rs +++ b/codex-rs/exec/tests/apply_patch.rs @@ -1,5 +1,6 @@ use anyhow::Context; use assert_cmd::prelude::*; +use codex_core::CODEX_APPLY_PATCH_ARG1; use std::fs; use std::process::Command; use tempfile::tempdir; @@ -16,7 +17,7 @@ fn test_standalone_exec_cli_can_use_apply_patch() -> anyhow::Result<()> { Command::cargo_bin("codex-exec") .context("should find binary for codex-exec")? - .arg("--codex-run-as-apply-patch") + .arg(CODEX_APPLY_PATCH_ARG1) .arg( r#"*** Begin Patch *** Update File: source.txt From f2134f66332bdba9c26f6b32725ec2aeb484a5c9 Mon Sep 17 00:00:00 2001 From: Jeremy Rose <172423086+nornagon-openai@users.noreply.github.com> Date: Wed, 30 Jul 2025 17:06:55 -0700 Subject: [PATCH 010/100] resizable viewport (#1732) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Proof of concept for a resizable viewport. The general approach here is to duplicate the `Terminal` struct from ratatui, but with our own logic. This is a "light fork" in that we are still using all the base ratatui functions (`Buffer`, `Widget` and so on), but we're doing our own bookkeeping at the top level to determine where to draw everything. This approach could use improvement—e.g, when the window is resized to a smaller size, if the UI wraps, we don't correctly clear out the artifacts from wrapping. This is possible with a little work (i.e. tracking what parts of our UI would have been wrapped), but this behavior is at least at par with the existing behavior. https://github.com/user-attachments/assets/4eb17689-09fd-4daa-8315-c7ebc654986d cc @joshka who might have Thoughts™ --- NOTICE | 4 + codex-rs/tui/src/app.rs | 40 ++ codex-rs/tui/src/bottom_pane/chat_composer.rs | 13 +- codex-rs/tui/src/bottom_pane/command_popup.rs | 2 +- .../tui/src/bottom_pane/file_search_popup.rs | 2 +- codex-rs/tui/src/bottom_pane/mod.rs | 4 + codex-rs/tui/src/chatwidget.rs | 4 + codex-rs/tui/src/custom_terminal.rs | 588 ++++++++++++++++++ codex-rs/tui/src/insert_history.rs | 15 +- codex-rs/tui/src/lib.rs | 1 + codex-rs/tui/src/tui.rs | 18 +- 11 files changed, 668 insertions(+), 23 deletions(-) create mode 100644 codex-rs/tui/src/custom_terminal.rs diff --git a/NOTICE b/NOTICE index ad09ca421e..2805899d56 100644 --- a/NOTICE +++ b/NOTICE @@ -1,2 +1,6 @@ OpenAI Codex Copyright 2025 OpenAI + +This project includes code derived from [Ratatui](https://github.com/ratatui/ratatui), licensed under the MIT license. +Copyright (c) 2016-2022 Florian Dehau +Copyright (c) 2023-2025 The Ratatui Developers diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index 6823a83a50..13ceabd7aa 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -12,6 +12,8 @@ use codex_core::protocol::Event; use color_eyre::eyre::Result; use crossterm::event::KeyCode; use crossterm::event::KeyEvent; +use ratatui::layout::Offset; +use ratatui::prelude::Backend; use std::path::PathBuf; use std::sync::Arc; use std::sync::atomic::AtomicBool; @@ -321,6 +323,44 @@ impl App<'_> { fn draw_next_frame(&mut self, terminal: &mut tui::Tui) -> Result<()> { // TODO: add a throttle to avoid redrawing too often + let screen_size = terminal.size()?; + let last_known_screen_size = terminal.last_known_screen_size; + if screen_size != last_known_screen_size { + let cursor_pos = terminal.get_cursor_position()?; + let last_known_cursor_pos = terminal.last_known_cursor_pos; + if cursor_pos.y != last_known_cursor_pos.y { + // The terminal was resized. The only point of reference we have for where our viewport + // was moved is the cursor position. + // NB this assumes that the cursor was not wrapped as part of the resize. + let cursor_delta = cursor_pos.y as i32 - last_known_cursor_pos.y as i32; + + let new_viewport_area = terminal.viewport_area.offset(Offset { + x: 0, + y: cursor_delta, + }); + terminal.set_viewport_area(new_viewport_area); + terminal.clear()?; + } + } + + let size = terminal.size()?; + let desired_height = match &self.app_state { + AppState::Chat { widget } => widget.desired_height(), + AppState::GitWarning { .. } => 10, + }; + let mut area = terminal.viewport_area; + area.height = desired_height; + area.width = size.width; + if area.bottom() > size.height { + terminal + .backend_mut() + .scroll_region_up(0..area.top(), area.bottom() - size.height)?; + area.y = size.height - area.height; + } + if area != terminal.viewport_area { + terminal.clear()?; + terminal.set_viewport_area(area); + } match &mut self.app_state { AppState::Chat { widget } => { terminal.draw(|frame| frame.render_widget_ref(&**widget, frame.area()))?; diff --git a/codex-rs/tui/src/bottom_pane/chat_composer.rs b/codex-rs/tui/src/bottom_pane/chat_composer.rs index b15d81f8f5..4d313f14a5 100644 --- a/codex-rs/tui/src/bottom_pane/chat_composer.rs +++ b/codex-rs/tui/src/bottom_pane/chat_composer.rs @@ -71,6 +71,15 @@ impl ChatComposer<'_> { this } + pub fn desired_height(&self) -> u16 { + 2 + self.textarea.lines().len() as u16 + + match &self.active_popup { + ActivePopup::None => 0u16, + ActivePopup::Command(c) => c.calculate_required_height(), + ActivePopup::File(c) => c.calculate_required_height(), + } + } + /// Returns true if the composer currently contains no user input. pub(crate) fn is_empty(&self) -> bool { self.textarea.is_empty() @@ -651,7 +660,7 @@ impl WidgetRef for &ChatComposer<'_> { fn render_ref(&self, area: Rect, buf: &mut Buffer) { match &self.active_popup { ActivePopup::Command(popup) => { - let popup_height = popup.calculate_required_height(&area); + let popup_height = popup.calculate_required_height(); // Split the provided rect so that the popup is rendered at the // *top* and the textarea occupies the remaining space below. @@ -673,7 +682,7 @@ impl WidgetRef for &ChatComposer<'_> { self.textarea.render(textarea_rect, buf); } ActivePopup::File(popup) => { - let popup_height = popup.calculate_required_height(&area); + let popup_height = popup.calculate_required_height(); let popup_rect = Rect { x: area.x, diff --git a/codex-rs/tui/src/bottom_pane/command_popup.rs b/codex-rs/tui/src/bottom_pane/command_popup.rs index fd865047ef..da3b3a8253 100644 --- a/codex-rs/tui/src/bottom_pane/command_popup.rs +++ b/codex-rs/tui/src/bottom_pane/command_popup.rs @@ -71,7 +71,7 @@ impl CommandPopup { /// Determine the preferred height of the popup. This is the number of /// rows required to show **at most** `MAX_POPUP_ROWS` commands plus the /// table/border overhead (one line at the top and one at the bottom). - pub(crate) fn calculate_required_height(&self, _area: &Rect) -> u16 { + pub(crate) fn calculate_required_height(&self) -> u16 { let matches = self.filtered_commands(); let row_count = matches.len().clamp(1, MAX_POPUP_ROWS) as u16; // Account for the border added by the Block that wraps the table. diff --git a/codex-rs/tui/src/bottom_pane/file_search_popup.rs b/codex-rs/tui/src/bottom_pane/file_search_popup.rs index 34eb59e4b2..e15f8690ae 100644 --- a/codex-rs/tui/src/bottom_pane/file_search_popup.rs +++ b/codex-rs/tui/src/bottom_pane/file_search_popup.rs @@ -109,7 +109,7 @@ impl FileSearchPopup { } /// Preferred height (rows) including border. - pub(crate) fn calculate_required_height(&self, _area: &Rect) -> u16 { + pub(crate) fn calculate_required_height(&self) -> u16 { // Row count depends on whether we already have matches. If no matches // yet (e.g. initial search or query with no results) reserve a single // row so the popup is still visible. When matches are present we show diff --git a/codex-rs/tui/src/bottom_pane/mod.rs b/codex-rs/tui/src/bottom_pane/mod.rs index 4ec1ba4b3e..2ca858d8ce 100644 --- a/codex-rs/tui/src/bottom_pane/mod.rs +++ b/codex-rs/tui/src/bottom_pane/mod.rs @@ -64,6 +64,10 @@ impl BottomPane<'_> { } } + pub fn desired_height(&self) -> u16 { + self.composer.desired_height() + } + /// Forward a key event to the active view or the composer. pub fn handle_key_event(&mut self, key_event: KeyEvent) -> InputResult { if let Some(mut view) = self.active_view.take() { diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index fde6978634..33e3ee11e4 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -143,6 +143,10 @@ impl ChatWidget<'_> { } } + pub fn desired_height(&self) -> u16 { + self.bottom_pane.desired_height() + } + pub(crate) fn handle_key_event(&mut self, key_event: KeyEvent) { self.bottom_pane.clear_ctrl_c_quit_hint(); diff --git a/codex-rs/tui/src/custom_terminal.rs b/codex-rs/tui/src/custom_terminal.rs new file mode 100644 index 0000000000..1ada679fc1 --- /dev/null +++ b/codex-rs/tui/src/custom_terminal.rs @@ -0,0 +1,588 @@ +// This is derived from `ratatui::Terminal`, which is licensed under the following terms: +// +// The MIT License (MIT) +// Copyright (c) 2016-2022 Florian Dehau +// Copyright (c) 2023-2025 The Ratatui Developers +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +use std::io; + +use ratatui::backend::Backend; +use ratatui::backend::ClearType; +use ratatui::buffer::Buffer; +use ratatui::layout::Position; +use ratatui::layout::Rect; +use ratatui::layout::Size; +use ratatui::widgets::StatefulWidget; +use ratatui::widgets::StatefulWidgetRef; +use ratatui::widgets::Widget; +use ratatui::widgets::WidgetRef; + +#[derive(Debug, Hash)] +pub struct Frame<'a> { + /// Where should the cursor be after drawing this frame? + /// + /// If `None`, the cursor is hidden and its position is controlled by the backend. If `Some((x, + /// y))`, the cursor is shown and placed at `(x, y)` after the call to `Terminal::draw()`. + pub(crate) cursor_position: Option, + + /// The area of the viewport + pub(crate) viewport_area: Rect, + + /// The buffer that is used to draw the current frame + pub(crate) buffer: &'a mut Buffer, + + /// The frame count indicating the sequence number of this frame. + pub(crate) count: usize, +} + +#[allow(dead_code)] +impl Frame<'_> { + /// The area of the current frame + /// + /// This is guaranteed not to change during rendering, so may be called multiple times. + /// + /// If your app listens for a resize event from the backend, it should ignore the values from + /// the event for any calculations that are used to render the current frame and use this value + /// instead as this is the area of the buffer that is used to render the current frame. + pub const fn area(&self) -> Rect { + self.viewport_area + } + + /// Render a [`Widget`] to the current buffer using [`Widget::render`]. + /// + /// Usually the area argument is the size of the current frame or a sub-area of the current + /// frame (which can be obtained using [`Layout`] to split the total area). + /// + /// # Example + /// + /// ```rust + /// # use ratatui::{backend::TestBackend, Terminal}; + /// # let backend = TestBackend::new(5, 5); + /// # let mut terminal = Terminal::new(backend).unwrap(); + /// # let mut frame = terminal.get_frame(); + /// use ratatui::{layout::Rect, widgets::Block}; + /// + /// let block = Block::new(); + /// let area = Rect::new(0, 0, 5, 5); + /// frame.render_widget(block, area); + /// ``` + /// + /// [`Layout`]: crate::layout::Layout + pub fn render_widget(&mut self, widget: W, area: Rect) { + widget.render(area, self.buffer); + } + + /// Render a [`WidgetRef`] to the current buffer using [`WidgetRef::render_ref`]. + /// + /// Usually the area argument is the size of the current frame or a sub-area of the current + /// frame (which can be obtained using [`Layout`] to split the total area). + /// + /// # Example + /// + /// ```rust + /// # #[cfg(feature = "unstable-widget-ref")] { + /// # use ratatui::{backend::TestBackend, Terminal}; + /// # let backend = TestBackend::new(5, 5); + /// # let mut terminal = Terminal::new(backend).unwrap(); + /// # let mut frame = terminal.get_frame(); + /// use ratatui::{layout::Rect, widgets::Block}; + /// + /// let block = Block::new(); + /// let area = Rect::new(0, 0, 5, 5); + /// frame.render_widget_ref(block, area); + /// # } + /// ``` + #[allow(clippy::needless_pass_by_value)] + pub fn render_widget_ref(&mut self, widget: W, area: Rect) { + widget.render_ref(area, self.buffer); + } + + /// Render a [`StatefulWidget`] to the current buffer using [`StatefulWidget::render`]. + /// + /// Usually the area argument is the size of the current frame or a sub-area of the current + /// frame (which can be obtained using [`Layout`] to split the total area). + /// + /// The last argument should be an instance of the [`StatefulWidget::State`] associated to the + /// given [`StatefulWidget`]. + /// + /// # Example + /// + /// ```rust + /// # use ratatui::{backend::TestBackend, Terminal}; + /// # let backend = TestBackend::new(5, 5); + /// # let mut terminal = Terminal::new(backend).unwrap(); + /// # let mut frame = terminal.get_frame(); + /// use ratatui::{ + /// layout::Rect, + /// widgets::{List, ListItem, ListState}, + /// }; + /// + /// let mut state = ListState::default().with_selected(Some(1)); + /// let list = List::new(vec![ListItem::new("Item 1"), ListItem::new("Item 2")]); + /// let area = Rect::new(0, 0, 5, 5); + /// frame.render_stateful_widget(list, area, &mut state); + /// ``` + /// + /// [`Layout`]: crate::layout::Layout + pub fn render_stateful_widget(&mut self, widget: W, area: Rect, state: &mut W::State) + where + W: StatefulWidget, + { + widget.render(area, self.buffer, state); + } + + /// Render a [`StatefulWidgetRef`] to the current buffer using + /// [`StatefulWidgetRef::render_ref`]. + /// + /// Usually the area argument is the size of the current frame or a sub-area of the current + /// frame (which can be obtained using [`Layout`] to split the total area). + /// + /// The last argument should be an instance of the [`StatefulWidgetRef::State`] associated to + /// the given [`StatefulWidgetRef`]. + /// + /// # Example + /// + /// ```rust + /// # #[cfg(feature = "unstable-widget-ref")] { + /// # use ratatui::{backend::TestBackend, Terminal}; + /// # let backend = TestBackend::new(5, 5); + /// # let mut terminal = Terminal::new(backend).unwrap(); + /// # let mut frame = terminal.get_frame(); + /// use ratatui::{ + /// layout::Rect, + /// widgets::{List, ListItem, ListState}, + /// }; + /// + /// let mut state = ListState::default().with_selected(Some(1)); + /// let list = List::new(vec![ListItem::new("Item 1"), ListItem::new("Item 2")]); + /// let area = Rect::new(0, 0, 5, 5); + /// frame.render_stateful_widget_ref(list, area, &mut state); + /// # } + /// ``` + #[allow(clippy::needless_pass_by_value)] + pub fn render_stateful_widget_ref(&mut self, widget: W, area: Rect, state: &mut W::State) + where + W: StatefulWidgetRef, + { + widget.render_ref(area, self.buffer, state); + } + + /// After drawing this frame, make the cursor visible and put it at the specified (x, y) + /// coordinates. If this method is not called, the cursor will be hidden. + /// + /// Note that this will interfere with calls to [`Terminal::hide_cursor`], + /// [`Terminal::show_cursor`], and [`Terminal::set_cursor_position`]. Pick one of the APIs and + /// stick with it. + /// + /// [`Terminal::hide_cursor`]: crate::Terminal::hide_cursor + /// [`Terminal::show_cursor`]: crate::Terminal::show_cursor + /// [`Terminal::set_cursor_position`]: crate::Terminal::set_cursor_position + pub fn set_cursor_position>(&mut self, position: P) { + self.cursor_position = Some(position.into()); + } + + /// Gets the buffer that this `Frame` draws into as a mutable reference. + pub fn buffer_mut(&mut self) -> &mut Buffer { + self.buffer + } + + /// Returns the current frame count. + /// + /// This method provides access to the frame count, which is a sequence number indicating + /// how many frames have been rendered up to (but not including) this one. It can be used + /// for purposes such as animation, performance tracking, or debugging. + /// + /// Each time a frame has been rendered, this count is incremented, + /// providing a consistent way to reference the order and number of frames processed by the + /// terminal. When count reaches its maximum value (`usize::MAX`), it wraps around to zero. + /// + /// This count is particularly useful when dealing with dynamic content or animations where the + /// state of the display changes over time. By tracking the frame count, developers can + /// synchronize updates or changes to the content with the rendering process. + /// + /// # Examples + /// + /// ```rust + /// # use ratatui::{backend::TestBackend, Terminal}; + /// # let backend = TestBackend::new(5, 5); + /// # let mut terminal = Terminal::new(backend).unwrap(); + /// # let mut frame = terminal.get_frame(); + /// let current_count = frame.count(); + /// println!("Current frame count: {}", current_count); + /// ``` + pub const fn count(&self) -> usize { + self.count + } +} + +#[derive(Debug, Default, Clone, Eq, PartialEq, Hash)] +pub struct Terminal +where + B: Backend, +{ + /// The backend used to interface with the terminal + backend: B, + /// Holds the results of the current and previous draw calls. The two are compared at the end + /// of each draw pass to output the necessary updates to the terminal + buffers: [Buffer; 2], + /// Index of the current buffer in the previous array + current: usize, + /// Whether the cursor is currently hidden + hidden_cursor: bool, + /// Area of the viewport + pub viewport_area: Rect, + /// Last known size of the terminal. Used to detect if the internal buffers have to be resized. + pub last_known_screen_size: Size, + /// Last known position of the cursor. Used to find the new area when the viewport is inlined + /// and the terminal resized. + pub last_known_cursor_pos: Position, + /// Number of frames rendered up until current time. + frame_count: usize, +} + +impl Drop for Terminal +where + B: Backend, +{ + #[allow(clippy::print_stderr)] + fn drop(&mut self) { + // Attempt to restore the cursor state + if self.hidden_cursor { + if let Err(err) = self.show_cursor() { + eprintln!("Failed to show the cursor: {err}"); + } + } + } +} + +impl Terminal +where + B: Backend, +{ + /// Creates a new [`Terminal`] with the given [`Backend`] and [`TerminalOptions`]. + /// + /// # Example + /// + /// ```rust + /// use std::io::stdout; + /// + /// use ratatui::{backend::CrosstermBackend, layout::Rect, Terminal, TerminalOptions, Viewport}; + /// + /// let backend = CrosstermBackend::new(stdout()); + /// let viewport = Viewport::Fixed(Rect::new(0, 0, 10, 10)); + /// let terminal = Terminal::with_options(backend, TerminalOptions { viewport })?; + /// # std::io::Result::Ok(()) + /// ``` + pub fn with_options(mut backend: B) -> io::Result { + let screen_size = backend.size()?; + let cursor_pos = backend.get_cursor_position()?; + Ok(Self { + backend, + buffers: [ + Buffer::empty(Rect::new(0, 0, 0, 0)), + Buffer::empty(Rect::new(0, 0, 0, 0)), + ], + current: 0, + hidden_cursor: false, + viewport_area: Rect::new(0, cursor_pos.y, 0, 0), + last_known_screen_size: screen_size, + last_known_cursor_pos: cursor_pos, + frame_count: 0, + }) + } + + /// Get a Frame object which provides a consistent view into the terminal state for rendering. + pub fn get_frame(&mut self) -> Frame { + let count = self.frame_count; + Frame { + cursor_position: None, + viewport_area: self.viewport_area, + buffer: self.current_buffer_mut(), + count, + } + } + + /// Gets the current buffer as a mutable reference. + pub fn current_buffer_mut(&mut self) -> &mut Buffer { + &mut self.buffers[self.current] + } + + /// Gets the backend + pub const fn backend(&self) -> &B { + &self.backend + } + + /// Gets the backend as a mutable reference + pub fn backend_mut(&mut self) -> &mut B { + &mut self.backend + } + + /// Obtains a difference between the previous and the current buffer and passes it to the + /// current backend for drawing. + pub fn flush(&mut self) -> io::Result<()> { + let previous_buffer = &self.buffers[1 - self.current]; + let current_buffer = &self.buffers[self.current]; + let updates = previous_buffer.diff(current_buffer); + if let Some((col, row, _)) = updates.last() { + self.last_known_cursor_pos = Position { x: *col, y: *row }; + } + self.backend.draw(updates.into_iter()) + } + + /// Updates the Terminal so that internal buffers match the requested area. + /// + /// Requested area will be saved to remain consistent when rendering. This leads to a full clear + /// of the screen. + pub fn resize(&mut self, screen_size: Size) -> io::Result<()> { + self.last_known_screen_size = screen_size; + Ok(()) + } + + /// Sets the viewport area. + pub fn set_viewport_area(&mut self, area: Rect) { + self.buffers[self.current].resize(area); + self.buffers[1 - self.current].resize(area); + self.viewport_area = area; + } + + /// Queries the backend for size and resizes if it doesn't match the previous size. + pub fn autoresize(&mut self) -> io::Result<()> { + let screen_size = self.size()?; + if screen_size != self.last_known_screen_size { + self.resize(screen_size)?; + } + Ok(()) + } + + /// Draws a single frame to the terminal. + /// + /// Returns a [`CompletedFrame`] if successful, otherwise a [`std::io::Error`]. + /// + /// If the render callback passed to this method can fail, use [`try_draw`] instead. + /// + /// Applications should call `draw` or [`try_draw`] in a loop to continuously render the + /// terminal. These methods are the main entry points for drawing to the terminal. + /// + /// [`try_draw`]: Terminal::try_draw + /// + /// This method will: + /// + /// - autoresize the terminal if necessary + /// - call the render callback, passing it a [`Frame`] reference to render to + /// - flush the current internal state by copying the current buffer to the backend + /// - move the cursor to the last known position if it was set during the rendering closure + /// + /// The render callback should fully render the entire frame when called, including areas that + /// are unchanged from the previous frame. This is because each frame is compared to the + /// previous frame to determine what has changed, and only the changes are written to the + /// terminal. If the render callback does not fully render the frame, the terminal will not be + /// in a consistent state. + /// + /// # Examples + /// + /// ``` + /// # let backend = ratatui::backend::TestBackend::new(10, 10); + /// # let mut terminal = ratatui::Terminal::new(backend)?; + /// use ratatui::{layout::Position, widgets::Paragraph}; + /// + /// // with a closure + /// terminal.draw(|frame| { + /// let area = frame.area(); + /// frame.render_widget(Paragraph::new("Hello World!"), area); + /// frame.set_cursor_position(Position { x: 0, y: 0 }); + /// })?; + /// + /// // or with a function + /// terminal.draw(render)?; + /// + /// fn render(frame: &mut ratatui::Frame) { + /// frame.render_widget(Paragraph::new("Hello World!"), frame.area()); + /// } + /// # std::io::Result::Ok(()) + /// ``` + pub fn draw(&mut self, render_callback: F) -> io::Result<()> + where + F: FnOnce(&mut Frame), + { + self.try_draw(|frame| { + render_callback(frame); + io::Result::Ok(()) + }) + } + + /// Tries to draw a single frame to the terminal. + /// + /// Returns [`Result::Ok`] containing a [`CompletedFrame`] if successful, otherwise + /// [`Result::Err`] containing the [`std::io::Error`] that caused the failure. + /// + /// This is the equivalent of [`Terminal::draw`] but the render callback is a function or + /// closure that returns a `Result` instead of nothing. + /// + /// Applications should call `try_draw` or [`draw`] in a loop to continuously render the + /// terminal. These methods are the main entry points for drawing to the terminal. + /// + /// [`draw`]: Terminal::draw + /// + /// This method will: + /// + /// - autoresize the terminal if necessary + /// - call the render callback, passing it a [`Frame`] reference to render to + /// - flush the current internal state by copying the current buffer to the backend + /// - move the cursor to the last known position if it was set during the rendering closure + /// - return a [`CompletedFrame`] with the current buffer and the area of the terminal + /// + /// The render callback passed to `try_draw` can return any [`Result`] with an error type that + /// can be converted into an [`std::io::Error`] using the [`Into`] trait. This makes it possible + /// to use the `?` operator to propagate errors that occur during rendering. If the render + /// callback returns an error, the error will be returned from `try_draw` as an + /// [`std::io::Error`] and the terminal will not be updated. + /// + /// The [`CompletedFrame`] returned by this method can be useful for debugging or testing + /// purposes, but it is often not used in regular applicationss. + /// + /// The render callback should fully render the entire frame when called, including areas that + /// are unchanged from the previous frame. This is because each frame is compared to the + /// previous frame to determine what has changed, and only the changes are written to the + /// terminal. If the render function does not fully render the frame, the terminal will not be + /// in a consistent state. + /// + /// # Examples + /// + /// ```should_panic + /// # use ratatui::layout::Position;; + /// # let backend = ratatui::backend::TestBackend::new(10, 10); + /// # let mut terminal = ratatui::Terminal::new(backend)?; + /// use std::io; + /// + /// use ratatui::widgets::Paragraph; + /// + /// // with a closure + /// terminal.try_draw(|frame| { + /// let value: u8 = "not a number".parse().map_err(io::Error::other)?; + /// let area = frame.area(); + /// frame.render_widget(Paragraph::new("Hello World!"), area); + /// frame.set_cursor_position(Position { x: 0, y: 0 }); + /// io::Result::Ok(()) + /// })?; + /// + /// // or with a function + /// terminal.try_draw(render)?; + /// + /// fn render(frame: &mut ratatui::Frame) -> io::Result<()> { + /// let value: u8 = "not a number".parse().map_err(io::Error::other)?; + /// frame.render_widget(Paragraph::new("Hello World!"), frame.area()); + /// Ok(()) + /// } + /// # io::Result::Ok(()) + /// ``` + pub fn try_draw(&mut self, render_callback: F) -> io::Result<()> + where + F: FnOnce(&mut Frame) -> Result<(), E>, + E: Into, + { + // Autoresize - otherwise we get glitches if shrinking or potential desync between widgets + // and the terminal (if growing), which may OOB. + self.autoresize()?; + + let mut frame = self.get_frame(); + + render_callback(&mut frame).map_err(Into::into)?; + + // We can't change the cursor position right away because we have to flush the frame to + // stdout first. But we also can't keep the frame around, since it holds a &mut to + // Buffer. Thus, we're taking the important data out of the Frame and dropping it. + let cursor_position = frame.cursor_position; + + // Draw to stdout + self.flush()?; + + match cursor_position { + None => self.hide_cursor()?, + Some(position) => { + self.show_cursor()?; + self.set_cursor_position(position)?; + } + } + + self.swap_buffers(); + + // Flush + self.backend.flush()?; + + // increment frame count before returning from draw + self.frame_count = self.frame_count.wrapping_add(1); + + Ok(()) + } + + /// Hides the cursor. + pub fn hide_cursor(&mut self) -> io::Result<()> { + self.backend.hide_cursor()?; + self.hidden_cursor = true; + Ok(()) + } + + /// Shows the cursor. + pub fn show_cursor(&mut self) -> io::Result<()> { + self.backend.show_cursor()?; + self.hidden_cursor = false; + Ok(()) + } + + /// Gets the current cursor position. + /// + /// This is the position of the cursor after the last draw call. + #[allow(dead_code)] + pub fn get_cursor_position(&mut self) -> io::Result { + self.backend.get_cursor_position() + } + + /// Sets the cursor position. + pub fn set_cursor_position>(&mut self, position: P) -> io::Result<()> { + let position = position.into(); + self.backend.set_cursor_position(position)?; + self.last_known_cursor_pos = position; + Ok(()) + } + + /// Clear the terminal and force a full redraw on the next draw call. + pub fn clear(&mut self) -> io::Result<()> { + if self.viewport_area.is_empty() { + return Ok(()); + } + self.backend + .set_cursor_position(self.viewport_area.as_position())?; + self.backend.clear_region(ClearType::AfterCursor)?; + // Reset the back buffer to make sure the next update will redraw everything. + self.buffers[1 - self.current].reset(); + Ok(()) + } + + /// Clears the inactive buffer and swaps it with the current buffer + pub fn swap_buffers(&mut self) { + self.buffers[1 - self.current].reset(); + self.current = 1 - self.current; + } + + /// Queries the real size of the backend. + pub fn size(&self) -> io::Result { + self.backend.size() + } +} diff --git a/codex-rs/tui/src/insert_history.rs b/codex-rs/tui/src/insert_history.rs index 32d0b4b297..54faf4beb8 100644 --- a/codex-rs/tui/src/insert_history.rs +++ b/codex-rs/tui/src/insert_history.rs @@ -4,6 +4,7 @@ use std::io::Write; use crate::tui; use crossterm::Command; +use crossterm::cursor::MoveTo; use crossterm::queue; use crossterm::style::Color as CColor; use crossterm::style::Colors; @@ -12,7 +13,6 @@ use crossterm::style::SetAttribute; use crossterm::style::SetBackgroundColor; use crossterm::style::SetColors; use crossterm::style::SetForegroundColor; -use ratatui::layout::Position; use ratatui::layout::Size; use ratatui::prelude::Backend; use ratatui::style::Color; @@ -23,6 +23,7 @@ use ratatui::text::Span; /// Insert `lines` above the viewport. pub(crate) fn insert_history_lines(terminal: &mut tui::Tui, lines: Vec) { let screen_size = terminal.backend().size().unwrap_or(Size::new(0, 0)); + let cursor_pos = terminal.get_cursor_position().ok(); let mut area = terminal.get_frame().area(); @@ -60,9 +61,10 @@ pub(crate) fn insert_history_lines(terminal: &mut tui::Tui, lines: Vec) { // └──────────────────────────────┘ queue!(std::io::stdout(), SetScrollRegion(1..area.top())).ok(); - terminal - .set_cursor_position(Position::new(0, cursor_top)) - .ok(); + // NB: we are using MoveTo instead of set_cursor_position here to avoid messing with the + // terminal's last_known_cursor_position, which hopefully will still be accurate after we + // fetch/restore the cursor position. insert_history_lines should be cursor-position-neutral :) + queue!(std::io::stdout(), MoveTo(0, cursor_top)).ok(); for line in lines { queue!(std::io::stdout(), Print("\r\n")).ok(); @@ -70,6 +72,11 @@ pub(crate) fn insert_history_lines(terminal: &mut tui::Tui, lines: Vec) { } queue!(std::io::stdout(), ResetScrollRegion).ok(); + + // Restore the cursor position to where it was before we started. + if let Some(cursor_pos) = cursor_pos { + queue!(std::io::stdout(), MoveTo(cursor_pos.x, cursor_pos.y)).ok(); + } } fn wrapped_line_count(lines: &[Line], width: u16) -> u16 { diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index 424b5ac2fc..351fab4df8 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -25,6 +25,7 @@ mod bottom_pane; mod chatwidget; mod citation_regex; mod cli; +mod custom_terminal; mod exec_command; mod file_search; mod get_git_diff; diff --git a/codex-rs/tui/src/tui.rs b/codex-rs/tui/src/tui.rs index 66ae1cfb96..1b215961ab 100644 --- a/codex-rs/tui/src/tui.rs +++ b/codex-rs/tui/src/tui.rs @@ -5,14 +5,13 @@ use std::io::stdout; use codex_core::config::Config; use crossterm::event::DisableBracketedPaste; use crossterm::event::EnableBracketedPaste; -use ratatui::Terminal; -use ratatui::TerminalOptions; -use ratatui::Viewport; use ratatui::backend::CrosstermBackend; use ratatui::crossterm::execute; use ratatui::crossterm::terminal::disable_raw_mode; use ratatui::crossterm::terminal::enable_raw_mode; +use crate::custom_terminal::Terminal; + /// A type alias for the terminal type used in this application pub type Tui = Terminal>; @@ -23,19 +22,8 @@ pub fn init(_config: &Config) -> Result { enable_raw_mode()?; set_panic_hook(); - // Reserve a fixed number of lines for the interactive viewport (composer, - // status, popups). History is injected above using `insert_before`. This - // is an initial step of the refactor – later the height can become - // dynamic. For now a conservative default keeps enough room for the - // multi‑line composer while not occupying the whole screen. - const BOTTOM_VIEWPORT_HEIGHT: u16 = 8; let backend = CrosstermBackend::new(stdout()); - let tui = Terminal::with_options( - backend, - TerminalOptions { - viewport: Viewport::Inline(BOTTOM_VIEWPORT_HEIGHT), - }, - )?; + let tui = Terminal::with_options(backend)?; Ok(tui) } From 35010812c7e8b40c3d921c6d3148db2a88d5a4a6 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Wed, 30 Jul 2025 17:49:07 -0700 Subject: [PATCH 011/100] chore: add support for a new label, codex-rust-review (#1744) The goal of this change is to try an experiment where we try to get AI to take on more of the code review load. The idea is that once you believe your PR is ready for review, please add the `codex-rust-review` label (as opposed to the `codex-review` label). Admittedly the corresponding prompt currently represents my personal biases in terms of code review, but we should massage it over time to represent the team's preferences. --- .github/codex/labels/codex-rust-review.md | 23 +++++++++++++++++++++++ .github/workflows/codex.yml | 2 +- 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 .github/codex/labels/codex-rust-review.md diff --git a/.github/codex/labels/codex-rust-review.md b/.github/codex/labels/codex-rust-review.md new file mode 100644 index 0000000000..2c2893a1fe --- /dev/null +++ b/.github/codex/labels/codex-rust-review.md @@ -0,0 +1,23 @@ +Review this PR and respond with a very concise final message, formatted in Markdown. + +There should be a summary of the changes (1-2 sentences) and a few bullet points if necessary. + +Then provide the **review** (1-2 sentences plus bullet points, friendly tone). + +Things to look out for when doing the review: + +- **Make sure the pull request body explains the motivation behind the change.** If the author has failed to do this, call it out, and if you think you can deduce the motivation behind the change, propose copy. +- Ideally, the PR body also contains a small summary of the change. For small changes, the PR title may be sufficient. +- Each PR should ideally do one conceptual thing. For example, if a PR does a refactoring as well as introducing a new feature, push back and suggest the refactoring be done in a separate PR. This makes things easier for the reviewer, as refactoring changes can often be far-reaching, yet quick to review. +- If the nature of the change seems to have a visual component (which is often the case for changes to `codex-rs/tui`), recommend including a screenshot or video to demonstrate the change, if appropriate. +- Rust files should generally be organized such that the public parts of the API appear near the top of the file and helper functions go below. This is analagous to the "inverted pyramid" structure that is favored in journalism. +- Encourage the use of small enums or the newtype pattern in Rust if it helps readability without adding significant cognitive load or lines of code. +- Be wary of large files and offer suggestions for how to break things into more reasonably-sized files. +- When modifying a `Cargo.toml` file, make sure that dependency lists stay alphabetically sorted. Also consider whether a new dependency is added to the appropriate place (e.g., `[dependencies]` versus `[dev-dependencies]`) +- If you see opportunities for the changes in a diff to use more idiomatic Rust, please make specific recommendations. For example, favor the use of expressions over `return`. +- When introducing new code, be on the lookout for code that duplicates existing code. When found, propose a way to refactor the existing code such that it should be reused. +- Each create in the Cargo workspace in `codex-rs` has a specific purpose: make a note if you believe new code is not introduced in the correct crate. +- When possible, try to keep the `core` crate as small as possible. Non-core but shared logic is often a good candidate for `codex-rs/common`. +- References to existing GitHub issues and PRs are encouraged, where appropriate, though you likely do not have network access, so may not be able to help here. + +{CODEX_ACTION_GITHUB_EVENT_PATH} contains the JSON that triggered this GitHub workflow. It contains the `base` and `head` refs that define this PR. Both refs are available locally. diff --git a/.github/workflows/codex.yml b/.github/workflows/codex.yml index a0ac5b9740..18fe74cc85 100644 --- a/.github/workflows/codex.yml +++ b/.github/workflows/codex.yml @@ -20,7 +20,7 @@ jobs: (github.event_name == 'issues' && ( (github.event.action == 'labeled' && (github.event.label.name == 'codex-attempt' || github.event.label.name == 'codex-triage')) )) || - (github.event_name == 'pull_request' && github.event.action == 'labeled' && github.event.label.name == 'codex-review') + (github.event_name == 'pull_request' && github.event.action == 'labeled' && (github.event.label.name == 'codex-review' || github.event.label.name == 'codex-rust-review')) runs-on: ubuntu-latest permissions: contents: write # can push or create branches From 51b6bdefbeb47adf9fb6f7fe4a03461376ab56a8 Mon Sep 17 00:00:00 2001 From: pakrym-oai Date: Wed, 30 Jul 2025 18:37:00 -0700 Subject: [PATCH 012/100] Auto format toml (#1745) Add recommended extension and configure it to auto format prompt. --- .vscode/extensions.json | 5 +++++ .vscode/settings.json | 8 +++++++- codex-rs/Cargo.toml | 3 +-- codex-rs/ansi-escape/Cargo.toml | 4 ++-- codex-rs/apply-patch/Cargo.toml | 2 +- codex-rs/arg0/Cargo.toml | 2 +- codex-rs/chatgpt/Cargo.toml | 6 +++--- codex-rs/cli/Cargo.toml | 4 ++-- codex-rs/common/Cargo.toml | 6 +++--- codex-rs/core/Cargo.toml | 8 ++++---- codex-rs/exec/Cargo.toml | 4 ++-- codex-rs/file-search/Cargo.toml | 2 +- codex-rs/linux-sandbox/Cargo.toml | 4 ++-- codex-rs/login/Cargo.toml | 2 +- codex-rs/mcp-server/Cargo.toml | 10 +++++----- codex-rs/mcp-types/Cargo.toml | 2 +- codex-rs/tui/Cargo.toml | 4 ++-- 17 files changed, 43 insertions(+), 33 deletions(-) create mode 100644 .vscode/extensions.json diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000000..dd5dac527d --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,5 @@ +{ + "recommendations": [ + "tamasfe.even-better-toml", + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json index f66a12583c..1712f5989b 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -6,5 +6,11 @@ "[rust]": { "editor.defaultFormatter": "rust-lang.rust-analyzer", "editor.formatOnSave": true, - } + }, + "[toml]": { + "editor.defaultFormatter": "tamasfe.even-better-toml", + "editor.formatOnSave": true, + }, + "evenBetterToml.formatter.reorderArrays": true, + "evenBetterToml.formatter.reorderKeys": true, } diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index 51b2b5cc12..0f8085c7e5 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -1,5 +1,4 @@ [workspace] -resolver = "2" members = [ "ansi-escape", "apply-patch", @@ -17,6 +16,7 @@ members = [ "mcp-types", "tui", ] +resolver = "2" [workspace.package] version = "0.0.0" @@ -45,4 +45,3 @@ codegen-units = 1 [patch.crates-io] # ratatui = { path = "../../ratatui" } ratatui = { git = "https://github.com/nornagon/ratatui", branch = "nornagon-v0.29.0-patch" } - diff --git a/codex-rs/ansi-escape/Cargo.toml b/codex-rs/ansi-escape/Cargo.toml index 9092c77c9c..ada675380d 100644 --- a/codex-rs/ansi-escape/Cargo.toml +++ b/codex-rs/ansi-escape/Cargo.toml @@ -1,7 +1,7 @@ [package] +edition = "2024" name = "codex-ansi-escape" version = { workspace = true } -edition = "2024" [lib] name = "codex_ansi_escape" @@ -10,7 +10,7 @@ path = "src/lib.rs" [dependencies] ansi-to-tui = "7.0.0" ratatui = { version = "0.29.0", features = [ - "unstable-widget-ref", "unstable-rendered-line-info", + "unstable-widget-ref", ] } tracing = { version = "0.1.41", features = ["log"] } diff --git a/codex-rs/apply-patch/Cargo.toml b/codex-rs/apply-patch/Cargo.toml index 5b95d4fa15..622f53ce71 100644 --- a/codex-rs/apply-patch/Cargo.toml +++ b/codex-rs/apply-patch/Cargo.toml @@ -1,7 +1,7 @@ [package] +edition = "2024" name = "codex-apply-patch" version = { workspace = true } -edition = "2024" [lib] name = "codex_apply_patch" diff --git a/codex-rs/arg0/Cargo.toml b/codex-rs/arg0/Cargo.toml index 7c55ac0d96..d668ffeff9 100644 --- a/codex-rs/arg0/Cargo.toml +++ b/codex-rs/arg0/Cargo.toml @@ -1,7 +1,7 @@ [package] +edition = "2024" name = "codex-arg0" version = { workspace = true } -edition = "2024" [lib] name = "codex_arg0" diff --git a/codex-rs/chatgpt/Cargo.toml b/codex-rs/chatgpt/Cargo.toml index e07543f4e8..903dc14b51 100644 --- a/codex-rs/chatgpt/Cargo.toml +++ b/codex-rs/chatgpt/Cargo.toml @@ -1,7 +1,7 @@ [package] +edition = "2024" name = "codex-chatgpt" version = { workspace = true } -edition = "2024" [lints] workspace = true @@ -9,12 +9,12 @@ workspace = true [dependencies] anyhow = "1" clap = { version = "4", features = ["derive"] } -serde = { version = "1", features = ["derive"] } -serde_json = "1" codex-common = { path = "../common", features = ["cli"] } codex-core = { path = "../core" } codex-login = { path = "../login" } reqwest = { version = "0.12", features = ["json", "stream"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" tokio = { version = "1", features = ["full"] } [dev-dependencies] diff --git a/codex-rs/cli/Cargo.toml b/codex-rs/cli/Cargo.toml index ab98764bed..0f370691cf 100644 --- a/codex-rs/cli/Cargo.toml +++ b/codex-rs/cli/Cargo.toml @@ -1,7 +1,7 @@ [package] +edition = "2024" name = "codex-cli" version = { workspace = true } -edition = "2024" [[bin]] name = "codex" @@ -20,8 +20,8 @@ clap = { version = "4", features = ["derive"] } clap_complete = "4" codex-arg0 = { path = "../arg0" } codex-chatgpt = { path = "../chatgpt" } -codex-core = { path = "../core" } codex-common = { path = "../common", features = ["cli"] } +codex-core = { path = "../core" } codex-exec = { path = "../exec" } codex-login = { path = "../login" } codex-mcp-server = { path = "../mcp-server" } diff --git a/codex-rs/common/Cargo.toml b/codex-rs/common/Cargo.toml index 3b843181cf..1723098b8a 100644 --- a/codex-rs/common/Cargo.toml +++ b/codex-rs/common/Cargo.toml @@ -1,7 +1,7 @@ [package] +edition = "2024" name = "codex-common" version = { workspace = true } -edition = "2024" [lints] workspace = true @@ -9,11 +9,11 @@ workspace = true [dependencies] clap = { version = "4", features = ["derive", "wrap_help"], optional = true } codex-core = { path = "../core" } -toml = { version = "0.9", optional = true } serde = { version = "1", optional = true } +toml = { version = "0.9", optional = true } [features] # Separate feature so that `clap` is not a mandatory dependency. -cli = ["clap", "toml", "serde"] +cli = ["clap", "serde", "toml"] elapsed = [] sandbox_summary = [] diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index 5ebb5ef63d..ecc904cd5e 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -1,7 +1,7 @@ [package] +edition = "2024" name = "codex-core" version = { workspace = true } -edition = "2024" [lib] name = "codex_core" @@ -15,10 +15,10 @@ anyhow = "1" async-channel = "2.3.1" base64 = "0.22" bytes = "1.10.1" -codex-apply-patch = { path = "../apply-patch" } -codex-mcp-client = { path = "../mcp-client" } chrono = { version = "0.4", features = ["serde"] } +codex-apply-patch = { path = "../apply-patch" } codex-login = { path = "../login" } +codex-mcp-client = { path = "../mcp-client" } dirs = "6" env-flags = "0.1.1" eventsource-stream = "0.2.3" @@ -49,8 +49,8 @@ tracing = { version = "0.1.41", features = ["log"] } tree-sitter = "0.25.8" tree-sitter-bash = "0.25.0" uuid = { version = "1", features = ["serde", "v4"] } -wildmatch = "2.4.0" whoami = "1.6.0" +wildmatch = "2.4.0" [target.'cfg(target_os = "linux")'.dependencies] diff --git a/codex-rs/exec/Cargo.toml b/codex-rs/exec/Cargo.toml index ced771f238..cd521410b1 100644 --- a/codex-rs/exec/Cargo.toml +++ b/codex-rs/exec/Cargo.toml @@ -1,7 +1,7 @@ [package] +edition = "2024" name = "codex-exec" version = { workspace = true } -edition = "2024" [[bin]] name = "codex-exec" @@ -19,12 +19,12 @@ anyhow = "1" chrono = "0.4.40" clap = { version = "4", features = ["derive"] } codex-arg0 = { path = "../arg0" } -codex-core = { path = "../core" } codex-common = { path = "../common", features = [ "cli", "elapsed", "sandbox_summary", ] } +codex-core = { path = "../core" } owo-colors = "4.2.0" serde_json = "1" shlex = "1.3.0" diff --git a/codex-rs/file-search/Cargo.toml b/codex-rs/file-search/Cargo.toml index bb5b80b2cf..3f70377183 100644 --- a/codex-rs/file-search/Cargo.toml +++ b/codex-rs/file-search/Cargo.toml @@ -1,7 +1,7 @@ [package] +edition = "2024" name = "codex-file-search" version = { workspace = true } -edition = "2024" [[bin]] name = "codex-file-search" diff --git a/codex-rs/linux-sandbox/Cargo.toml b/codex-rs/linux-sandbox/Cargo.toml index 4b173ea17a..ea7052c409 100644 --- a/codex-rs/linux-sandbox/Cargo.toml +++ b/codex-rs/linux-sandbox/Cargo.toml @@ -1,7 +1,7 @@ [package] +edition = "2024" name = "codex-linux-sandbox" version = { workspace = true } -edition = "2024" [[bin]] name = "codex-linux-sandbox" @@ -19,8 +19,8 @@ anyhow = "1" clap = { version = "4", features = ["derive"] } codex-common = { path = "../common", features = ["cli"] } codex-core = { path = "../core" } -libc = "0.2.172" landlock = "0.4.1" +libc = "0.2.172" seccompiler = "0.5.0" [target.'cfg(target_os = "linux")'.dev-dependencies] diff --git a/codex-rs/login/Cargo.toml b/codex-rs/login/Cargo.toml index e6eba6fd4f..e10666b092 100644 --- a/codex-rs/login/Cargo.toml +++ b/codex-rs/login/Cargo.toml @@ -1,7 +1,7 @@ [package] +edition = "2024" name = "codex-login" version = { workspace = true } -edition = "2024" [lints] workspace = true diff --git a/codex-rs/mcp-server/Cargo.toml b/codex-rs/mcp-server/Cargo.toml index 19cf4db538..2f618808c1 100644 --- a/codex-rs/mcp-server/Cargo.toml +++ b/codex-rs/mcp-server/Cargo.toml @@ -1,7 +1,7 @@ [package] +edition = "2024" name = "codex-mcp-server" version = { workspace = true } -edition = "2024" [[bin]] name = "codex-mcp-server" @@ -23,9 +23,7 @@ schemars = "0.8.22" serde = { version = "1", features = ["derive"] } serde_json = "1" shlex = "1.3.0" -toml = "0.9" -tracing = { version = "0.1.41", features = ["log"] } -tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] } +strum_macros = "0.27.2" tokio = { version = "1", features = [ "io-std", "macros", @@ -33,8 +31,10 @@ tokio = { version = "1", features = [ "rt-multi-thread", "signal", ] } +toml = "0.9" +tracing = { version = "0.1.41", features = ["log"] } +tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] } uuid = { version = "1", features = ["serde", "v4"] } -strum_macros = "0.27.2" [dev-dependencies] assert_cmd = "2" diff --git a/codex-rs/mcp-types/Cargo.toml b/codex-rs/mcp-types/Cargo.toml index 81ac2d9761..db849d5f0e 100644 --- a/codex-rs/mcp-types/Cargo.toml +++ b/codex-rs/mcp-types/Cargo.toml @@ -1,7 +1,7 @@ [package] +edition = "2024" name = "mcp-types" version = { workspace = true } -edition = "2024" [lints] workspace = true diff --git a/codex-rs/tui/Cargo.toml b/codex-rs/tui/Cargo.toml index 2f150921fb..63d287ca11 100644 --- a/codex-rs/tui/Cargo.toml +++ b/codex-rs/tui/Cargo.toml @@ -1,7 +1,7 @@ [package] +edition = "2024" name = "codex-tui" version = { workspace = true } -edition = "2024" [[bin]] name = "codex-tui" @@ -20,12 +20,12 @@ base64 = "0.22.1" clap = { version = "4", features = ["derive"] } codex-ansi-escape = { path = "../ansi-escape" } codex-arg0 = { path = "../arg0" } -codex-core = { path = "../core" } codex-common = { path = "../common", features = [ "cli", "elapsed", "sandbox_summary", ] } +codex-core = { path = "../core" } codex-file-search = { path = "../file-search" } codex-login = { path = "../login" } color-eyre = "0.6.3" From defeafb279d3f58f38e64d7c7c855c9edf7a5cf1 Mon Sep 17 00:00:00 2001 From: pap-openai Date: Thu, 31 Jul 2025 04:23:56 +0100 Subject: [PATCH 013/100] add keyboard enhancements to support shift_return (#1743) For terminal that supports [keyboard enhancements](https://docs.rs/libcrossterm/latest/crossterm/enum.KeyboardEnhancementFlags.html), adds the enhancements (enabling [kitty keyboard protocol](https://sw.kovidgoyal.net/kitty/keyboard-protocol/)) to support shift+enter listener. Those users (users with terminals listed on [KPP](https://sw.kovidgoyal.net/kitty/keyboard-protocol/)) should be able to press shift+return for new line --------- Co-authored-by: easong-openai --- codex-rs/tui/src/bottom_pane/chat_composer.rs | 14 ++++++++++++++ codex-rs/tui/src/tui.rs | 15 +++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/codex-rs/tui/src/bottom_pane/chat_composer.rs b/codex-rs/tui/src/bottom_pane/chat_composer.rs index 4d313f14a5..9c52057cc5 100644 --- a/codex-rs/tui/src/bottom_pane/chat_composer.rs +++ b/codex-rs/tui/src/bottom_pane/chat_composer.rs @@ -469,6 +469,20 @@ impl ChatComposer<'_> { self.textarea.insert_newline(); (InputResult::None, true) } + Input { + key: Key::Char('d'), + ctrl: true, + alt: false, + shift: false, + } => { + self.textarea.input(Input { + key: Key::Delete, + ctrl: false, + alt: false, + shift: false, + }); + (InputResult::None, true) + } input => self.handle_input_basic(input), } } diff --git a/codex-rs/tui/src/tui.rs b/codex-rs/tui/src/tui.rs index 1b215961ab..268483cbcf 100644 --- a/codex-rs/tui/src/tui.rs +++ b/codex-rs/tui/src/tui.rs @@ -5,6 +5,9 @@ use std::io::stdout; use codex_core::config::Config; use crossterm::event::DisableBracketedPaste; use crossterm::event::EnableBracketedPaste; +use crossterm::event::KeyboardEnhancementFlags; +use crossterm::event::PopKeyboardEnhancementFlags; +use crossterm::event::PushKeyboardEnhancementFlags; use ratatui::backend::CrosstermBackend; use ratatui::crossterm::execute; use ratatui::crossterm::terminal::disable_raw_mode; @@ -20,6 +23,17 @@ pub fn init(_config: &Config) -> Result { execute!(stdout(), EnableBracketedPaste)?; enable_raw_mode()?; + // Enable keyboard enhancement flags so modifiers for keys like Enter are disambiguated. + // chat_composer.rs is using a keyboard event listener to enter for any modified keys + // to create a new line that require this. + execute!( + stdout(), + PushKeyboardEnhancementFlags( + KeyboardEnhancementFlags::DISAMBIGUATE_ESCAPE_CODES + | KeyboardEnhancementFlags::REPORT_EVENT_TYPES + | KeyboardEnhancementFlags::REPORT_ALTERNATE_KEYS + ) + )?; set_panic_hook(); let backend = CrosstermBackend::new(stdout()); @@ -37,6 +51,7 @@ fn set_panic_hook() { /// Restore the terminal to its original state pub fn restore() -> Result<()> { + execute!(stdout(), PopKeyboardEnhancementFlags)?; execute!(stdout(), DisableBracketedPaste)?; disable_raw_mode()?; Ok(()) From d86270696e7bda0c321c7e981edbc7562ad3d330 Mon Sep 17 00:00:00 2001 From: Jeremy Rose <172423086+nornagon-openai@users.noreply.github.com> Date: Thu, 31 Jul 2025 00:43:21 -0700 Subject: [PATCH 014/100] streamline ui (#1733) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Simplify and improve many UI elements. * Remove all-around borders in most places. These interact badly with terminal resizing and look heavy. Prefer left-side-only borders. * Make the viewport adjust to the size of its contents. * / and @ autocomplete boxes appear below the prompt, instead of above it. * Restyle the keyboard shortcut hints & move them to the left. * Restyle the approval dialog. * Use synchronized rendering to avoid flashing during rerenders. https://github.com/user-attachments/assets/96f044af-283b-411c-b7fc-5e6b8a433c20 Screenshot 2025-07-30 at 5 29 20 PM --- codex-rs/tui/src/app.rs | 21 +++- .../src/bottom_pane/approval_modal_view.rs | 4 + .../tui/src/bottom_pane/bottom_pane_view.rs | 3 + codex-rs/tui/src/bottom_pane/chat_composer.rs | 100 ++++++++++-------- codex-rs/tui/src/bottom_pane/command_popup.rs | 48 ++++----- .../tui/src/bottom_pane/file_search_popup.rs | 28 +++-- codex-rs/tui/src/bottom_pane/mod.rs | 7 +- ...mposer__tests__backspace_after_pastes.snap | 20 ++-- ...tom_pane__chat_composer__tests__empty.snap | 20 ++-- ...tom_pane__chat_composer__tests__large.snap | 20 ++-- ...chat_composer__tests__multiple_pastes.snap | 20 ++-- ...tom_pane__chat_composer__tests__small.snap | 20 ++-- .../src/bottom_pane/status_indicator_view.rs | 4 + codex-rs/tui/src/chatwidget.rs | 32 ++++-- codex-rs/tui/src/history_cell.rs | 9 +- codex-rs/tui/src/slash_command.rs | 4 + codex-rs/tui/src/status_indicator_widget.rs | 8 +- codex-rs/tui/src/user_approval_widget.rs | 37 ++++--- 18 files changed, 232 insertions(+), 173 deletions(-) diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index 13ceabd7aa..ad4bc24fd4 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -9,7 +9,10 @@ use crate::slash_command::SlashCommand; use crate::tui; use codex_core::config::Config; use codex_core::protocol::Event; +use codex_core::protocol::EventMsg; +use codex_core::protocol::ExecApprovalRequestEvent; use color_eyre::eyre::Result; +use crossterm::SynchronizedUpdate; use crossterm::event::KeyCode; use crossterm::event::KeyEvent; use ratatui::layout::Offset; @@ -201,7 +204,7 @@ impl App<'_> { self.schedule_redraw(); } AppEvent::Redraw => { - self.draw_next_frame(terminal)?; + std::io::stdout().sync_update(|_| self.draw_next_frame(terminal))??; } AppEvent::KeyEvent(key_event) => { match key_event { @@ -297,6 +300,18 @@ impl App<'_> { widget.add_diff_output(text); } } + #[cfg(debug_assertions)] + SlashCommand::TestApproval => { + self.app_event_tx.send(AppEvent::CodexEvent(Event { + id: "1".to_string(), + msg: EventMsg::ExecApprovalRequest(ExecApprovalRequestEvent { + call_id: "1".to_string(), + command: vec!["git".into(), "apply".into()], + cwd: self.config.cwd.clone(), + reason: Some("test".to_string()), + }), + })); + } }, AppEvent::StartFileSearch(query) => { self.file_search.on_user_query(query); @@ -321,8 +336,6 @@ impl App<'_> { } fn draw_next_frame(&mut self, terminal: &mut tui::Tui) -> Result<()> { - // TODO: add a throttle to avoid redrawing too often - let screen_size = terminal.size()?; let last_known_screen_size = terminal.last_known_screen_size; if screen_size != last_known_screen_size { @@ -345,7 +358,7 @@ impl App<'_> { let size = terminal.size()?; let desired_height = match &self.app_state { - AppState::Chat { widget } => widget.desired_height(), + AppState::Chat { widget } => widget.desired_height(size.width), AppState::GitWarning { .. } => 10, }; let mut area = terminal.viewport_area; diff --git a/codex-rs/tui/src/bottom_pane/approval_modal_view.rs b/codex-rs/tui/src/bottom_pane/approval_modal_view.rs index 376135ef31..4cd952f9eb 100644 --- a/codex-rs/tui/src/bottom_pane/approval_modal_view.rs +++ b/codex-rs/tui/src/bottom_pane/approval_modal_view.rs @@ -57,6 +57,10 @@ impl<'a> BottomPaneView<'a> for ApprovalModalView<'a> { self.current.is_complete() && self.queue.is_empty() } + fn desired_height(&self, width: u16) -> u16 { + self.current.desired_height(width) + } + fn render(&self, area: Rect, buf: &mut Buffer) { (&self.current).render_ref(area, buf); } diff --git a/codex-rs/tui/src/bottom_pane/bottom_pane_view.rs b/codex-rs/tui/src/bottom_pane/bottom_pane_view.rs index 96922d94e7..a5616371d2 100644 --- a/codex-rs/tui/src/bottom_pane/bottom_pane_view.rs +++ b/codex-rs/tui/src/bottom_pane/bottom_pane_view.rs @@ -28,6 +28,9 @@ pub(crate) trait BottomPaneView<'a> { CancellationEvent::Ignored } + /// Return the desired height of the view. + fn desired_height(&self, width: u16) -> u16; + /// Render the view: this will be displayed in place of the composer. fn render(&self, area: Rect, buf: &mut Buffer); diff --git a/codex-rs/tui/src/bottom_pane/chat_composer.rs b/codex-rs/tui/src/bottom_pane/chat_composer.rs index 9c52057cc5..3bc573a003 100644 --- a/codex-rs/tui/src/bottom_pane/chat_composer.rs +++ b/codex-rs/tui/src/bottom_pane/chat_composer.rs @@ -1,11 +1,13 @@ use codex_core::protocol::TokenUsage; use crossterm::event::KeyEvent; use ratatui::buffer::Buffer; -use ratatui::layout::Alignment; use ratatui::layout::Rect; +use ratatui::style::Color; use ratatui::style::Style; +use ratatui::style::Styled; use ratatui::style::Stylize; use ratatui::text::Line; +use ratatui::text::Span; use ratatui::widgets::BorderType; use ratatui::widgets::Borders; use ratatui::widgets::Widget; @@ -22,7 +24,7 @@ use crate::app_event::AppEvent; use crate::app_event_sender::AppEventSender; use codex_file_search::FileMatch; -const BASE_PLACEHOLDER_TEXT: &str = "send a message"; +const BASE_PLACEHOLDER_TEXT: &str = "..."; /// If the pasted content exceeds this number of characters, replace it with a /// placeholder in the UI. const LARGE_PASTE_CHAR_THRESHOLD: usize = 1000; @@ -72,9 +74,9 @@ impl ChatComposer<'_> { } pub fn desired_height(&self) -> u16 { - 2 + self.textarea.lines().len() as u16 + self.textarea.lines().len().max(1) as u16 + match &self.active_popup { - ActivePopup::None => 0u16, + ActivePopup::None => 1u16, ActivePopup::Command(c) => c.calculate_required_height(), ActivePopup::File(c) => c.calculate_required_height(), } @@ -635,37 +637,17 @@ impl ChatComposer<'_> { } fn update_border(&mut self, has_focus: bool) { - struct BlockState { - right_title: Line<'static>, - border_style: Style, - } - - let bs = if has_focus { - if self.ctrl_c_quit_hint { - BlockState { - right_title: Line::from("Ctrl+C to quit").alignment(Alignment::Right), - border_style: Style::default(), - } - } else { - BlockState { - right_title: Line::from("Enter to send | Ctrl+D to quit | Ctrl+J for newline") - .alignment(Alignment::Right), - border_style: Style::default(), - } - } + let border_style = if has_focus { + Style::default().fg(Color::Cyan) } else { - BlockState { - right_title: Line::from(""), - border_style: Style::default().dim(), - } + Style::default().dim() }; self.textarea.set_block( ratatui::widgets::Block::default() - .title_bottom(bs.right_title) - .borders(Borders::ALL) - .border_type(BorderType::Rounded) - .border_style(bs.border_style), + .borders(Borders::LEFT) + .border_type(BorderType::QuadrantOutside) + .border_style(border_style), ); } } @@ -677,19 +659,19 @@ impl WidgetRef for &ChatComposer<'_> { let popup_height = popup.calculate_required_height(); // Split the provided rect so that the popup is rendered at the - // *top* and the textarea occupies the remaining space below. - let popup_rect = Rect { + // **bottom** and the textarea occupies the remaining space above. + let popup_height = popup_height.min(area.height); + let textarea_rect = Rect { x: area.x, y: area.y, width: area.width, - height: popup_height.min(area.height), + height: area.height.saturating_sub(popup_height), }; - - let textarea_rect = Rect { + let popup_rect = Rect { x: area.x, - y: area.y + popup_rect.height, + y: area.y + textarea_rect.height, width: area.width, - height: area.height.saturating_sub(popup_rect.height), + height: popup_height, }; popup.render(popup_rect, buf); @@ -698,25 +680,51 @@ impl WidgetRef for &ChatComposer<'_> { ActivePopup::File(popup) => { let popup_height = popup.calculate_required_height(); - let popup_rect = Rect { + let popup_height = popup_height.min(area.height); + let textarea_rect = Rect { x: area.x, y: area.y, width: area.width, - height: popup_height.min(area.height), - }; - - let textarea_rect = Rect { - x: area.x, - y: area.y + popup_rect.height, - width: area.width, height: area.height.saturating_sub(popup_height), }; + let popup_rect = Rect { + x: area.x, + y: area.y + textarea_rect.height, + width: area.width, + height: popup_height, + }; popup.render(popup_rect, buf); self.textarea.render(textarea_rect, buf); } ActivePopup::None => { - self.textarea.render(area, buf); + let mut textarea_rect = area; + textarea_rect.height = textarea_rect.height.saturating_sub(1); + self.textarea.render(textarea_rect, buf); + let mut bottom_line_rect = area; + bottom_line_rect.y += textarea_rect.height; + bottom_line_rect.height = 1; + let key_hint_style = Style::default().fg(Color::Cyan); + let hint = if self.ctrl_c_quit_hint { + vec![ + Span::from(" "), + "Ctrl+C again".set_style(key_hint_style), + Span::from(" to quit"), + ] + } else { + vec![ + Span::from(" "), + "⏎".set_style(key_hint_style), + Span::from(" send "), + "Shift+⏎".set_style(key_hint_style), + Span::from(" newline "), + "Ctrl+C".set_style(key_hint_style), + Span::from(" quit"), + ] + }; + Line::from(hint) + .style(Style::default().dim()) + .render_ref(bottom_line_rect, buf); } } } diff --git a/codex-rs/tui/src/bottom_pane/command_popup.rs b/codex-rs/tui/src/bottom_pane/command_popup.rs index da3b3a8253..364a8472dc 100644 --- a/codex-rs/tui/src/bottom_pane/command_popup.rs +++ b/codex-rs/tui/src/bottom_pane/command_popup.rs @@ -3,9 +3,9 @@ use ratatui::layout::Rect; use ratatui::style::Color; use ratatui::style::Style; use ratatui::style::Stylize; -use ratatui::widgets::Block; -use ratatui::widgets::BorderType; -use ratatui::widgets::Borders; +use ratatui::symbols::border::QUADRANT_LEFT_HALF; +use ratatui::text::Line; +use ratatui::text::Span; use ratatui::widgets::Cell; use ratatui::widgets::Row; use ratatui::widgets::Table; @@ -72,11 +72,7 @@ impl CommandPopup { /// rows required to show **at most** `MAX_POPUP_ROWS` commands plus the /// table/border overhead (one line at the top and one at the bottom). pub(crate) fn calculate_required_height(&self) -> u16 { - let matches = self.filtered_commands(); - let row_count = matches.len().clamp(1, MAX_POPUP_ROWS) as u16; - // Account for the border added by the Block that wraps the table. - // 2 = one line at the top, one at the bottom. - row_count + 2 + self.filtered_commands().len().clamp(1, MAX_POPUP_ROWS) as u16 } /// Return the list of commands that match the current filter. Matching is @@ -158,18 +154,19 @@ impl WidgetRef for CommandPopup { let default_style = Style::default(); let command_style = Style::default().fg(Color::LightBlue); for (idx, cmd) in visible_matches.iter().enumerate() { - let (cmd_style, desc_style) = if Some(idx) == self.selected_idx { - ( - command_style.bg(Color::DarkGray), - default_style.bg(Color::DarkGray), - ) - } else { - (command_style, default_style) - }; - rows.push(Row::new(vec![ - Cell::from(format!("/{}", cmd.command())).style(cmd_style), - Cell::from(cmd.description().to_string()).style(desc_style), + Cell::from(Line::from(vec![ + if Some(idx) == self.selected_idx { + Span::styled( + "›", + Style::default().bg(Color::DarkGray).fg(Color::LightCyan), + ) + } else { + Span::styled(QUADRANT_LEFT_HALF, Style::default().fg(Color::DarkGray)) + }, + Span::styled(format!("/{}", cmd.command()), command_style), + ])), + Cell::from(cmd.description().to_string()).style(default_style), ])); } } @@ -180,12 +177,13 @@ impl WidgetRef for CommandPopup { rows, [Constraint::Length(FIRST_COLUMN_WIDTH), Constraint::Min(10)], ) - .column_spacing(0) - .block( - Block::default() - .borders(Borders::ALL) - .border_type(BorderType::Rounded), - ); + .column_spacing(0); + // .block( + // Block::default() + // .borders(Borders::LEFT) + // .border_type(BorderType::QuadrantOutside) + // .border_style(Style::default().fg(Color::DarkGray)), + // ); table.render(area, buf); } diff --git a/codex-rs/tui/src/bottom_pane/file_search_popup.rs b/codex-rs/tui/src/bottom_pane/file_search_popup.rs index e15f8690ae..ac6c91cf47 100644 --- a/codex-rs/tui/src/bottom_pane/file_search_popup.rs +++ b/codex-rs/tui/src/bottom_pane/file_search_popup.rs @@ -115,12 +115,8 @@ impl FileSearchPopup { // row so the popup is still visible. When matches are present we show // up to MAX_RESULTS regardless of the waiting flag so the list // remains stable while a newer search is in-flight. - let rows = if self.matches.is_empty() { - 1 - } else { - self.matches.len().clamp(1, MAX_RESULTS) - } as u16; - rows + 2 // border + + self.matches.len().clamp(1, MAX_RESULTS) as u16 } } @@ -128,7 +124,14 @@ impl WidgetRef for &FileSearchPopup { fn render_ref(&self, area: Rect, buf: &mut Buffer) { // Prepare rows. let rows: Vec = if self.matches.is_empty() { - vec![Row::new(vec![Cell::from(" no matches ")])] + vec![Row::new(vec![ + Cell::from(if self.waiting { + "(searching …)" + } else { + "no matches" + }) + .style(Style::new().add_modifier(Modifier::ITALIC | Modifier::DIM)), + ])] } else { self.matches .iter() @@ -169,17 +172,12 @@ impl WidgetRef for &FileSearchPopup { .collect() }; - let mut title = format!(" @{} ", self.pending_query); - if self.waiting { - title.push_str(" (searching …)"); - } - let table = Table::new(rows, vec![Constraint::Percentage(100)]) .block( Block::default() - .borders(Borders::ALL) - .border_type(BorderType::Rounded) - .title(title), + .borders(Borders::LEFT) + .border_type(BorderType::QuadrantOutside) + .border_style(Style::default().fg(Color::DarkGray)), ) .widths([Constraint::Percentage(100)]); diff --git a/codex-rs/tui/src/bottom_pane/mod.rs b/codex-rs/tui/src/bottom_pane/mod.rs index 2ca858d8ce..2710a3e997 100644 --- a/codex-rs/tui/src/bottom_pane/mod.rs +++ b/codex-rs/tui/src/bottom_pane/mod.rs @@ -64,8 +64,11 @@ impl BottomPane<'_> { } } - pub fn desired_height(&self) -> u16 { - self.composer.desired_height() + pub fn desired_height(&self, width: u16) -> u16 { + self.active_view + .as_ref() + .map(|v| v.desired_height(width)) + .unwrap_or(self.composer.desired_height()) } /// Forward a key event to the active view or the composer. diff --git a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__backspace_after_pastes.snap b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__backspace_after_pastes.snap index fa604c862b..4f155dab30 100644 --- a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__backspace_after_pastes.snap +++ b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__backspace_after_pastes.snap @@ -2,13 +2,13 @@ source: tui/src/bottom_pane/chat_composer.rs expression: terminal.backend() --- -"╭──────────────────────────────────────────────────────────────────────────────────────────────────╮" -"│[Pasted Content 1002 chars][Pasted Content 1004 chars] │" -"│ │" -"│ │" -"│ │" -"│ │" -"│ │" -"│ │" -"│ │" -"╰───────────────────────────────────────────────Enter to send | Ctrl+D to quit | Ctrl+J for newline╯" +"▌[Pasted Content 1002 chars][Pasted Content 1004 chars] " +"▌ " +"▌ " +"▌ " +"▌ " +"▌ " +"▌ " +"▌ " +"▌ " +" ⏎ send Shift+⏎ newline Ctrl+C quit " diff --git a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__empty.snap b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__empty.snap index a89076d8aa..4e8371f177 100644 --- a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__empty.snap +++ b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__empty.snap @@ -2,13 +2,13 @@ source: tui/src/bottom_pane/chat_composer.rs expression: terminal.backend() --- -"╭──────────────────────────────────────────────────────────────────────────────────────────────────╮" -"│ send a message │" -"│ │" -"│ │" -"│ │" -"│ │" -"│ │" -"│ │" -"│ │" -"╰───────────────────────────────────────────────Enter to send | Ctrl+D to quit | Ctrl+J for newline╯" +"▌ ... " +"▌ " +"▌ " +"▌ " +"▌ " +"▌ " +"▌ " +"▌ " +"▌ " +" ⏎ send Shift+⏎ newline Ctrl+C quit " diff --git a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__large.snap b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__large.snap index 39a62da400..80fea40d5f 100644 --- a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__large.snap +++ b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__large.snap @@ -2,13 +2,13 @@ source: tui/src/bottom_pane/chat_composer.rs expression: terminal.backend() --- -"╭──────────────────────────────────────────────────────────────────────────────────────────────────╮" -"│[Pasted Content 1005 chars] │" -"│ │" -"│ │" -"│ │" -"│ │" -"│ │" -"│ │" -"│ │" -"╰───────────────────────────────────────────────Enter to send | Ctrl+D to quit | Ctrl+J for newline╯" +"▌[Pasted Content 1005 chars] " +"▌ " +"▌ " +"▌ " +"▌ " +"▌ " +"▌ " +"▌ " +"▌ " +" ⏎ send Shift+⏎ newline Ctrl+C quit " diff --git a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__multiple_pastes.snap b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__multiple_pastes.snap index cd94095431..26e8d26733 100644 --- a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__multiple_pastes.snap +++ b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__multiple_pastes.snap @@ -2,13 +2,13 @@ source: tui/src/bottom_pane/chat_composer.rs expression: terminal.backend() --- -"╭──────────────────────────────────────────────────────────────────────────────────────────────────╮" -"│[Pasted Content 1003 chars][Pasted Content 1007 chars] another short paste │" -"│ │" -"│ │" -"│ │" -"│ │" -"│ │" -"│ │" -"│ │" -"╰───────────────────────────────────────────────Enter to send | Ctrl+D to quit | Ctrl+J for newline╯" +"▌[Pasted Content 1003 chars][Pasted Content 1007 chars] another short paste " +"▌ " +"▌ " +"▌ " +"▌ " +"▌ " +"▌ " +"▌ " +"▌ " +" ⏎ send Shift+⏎ newline Ctrl+C quit " diff --git a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__small.snap b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__small.snap index e6b55e36d8..0f1b9e6426 100644 --- a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__small.snap +++ b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__small.snap @@ -2,13 +2,13 @@ source: tui/src/bottom_pane/chat_composer.rs expression: terminal.backend() --- -"╭──────────────────────────────────────────────────────────────────────────────────────────────────╮" -"│short │" -"│ │" -"│ │" -"│ │" -"│ │" -"│ │" -"│ │" -"│ │" -"╰───────────────────────────────────────────────Enter to send | Ctrl+D to quit | Ctrl+J for newline╯" +"▌short " +"▌ " +"▌ " +"▌ " +"▌ " +"▌ " +"▌ " +"▌ " +"▌ " +" ⏎ send Shift+⏎ newline Ctrl+C quit " diff --git a/codex-rs/tui/src/bottom_pane/status_indicator_view.rs b/codex-rs/tui/src/bottom_pane/status_indicator_view.rs index f8c06ec5e5..a944271e45 100644 --- a/codex-rs/tui/src/bottom_pane/status_indicator_view.rs +++ b/codex-rs/tui/src/bottom_pane/status_indicator_view.rs @@ -33,6 +33,10 @@ impl BottomPaneView<'_> for StatusIndicatorView { true } + fn desired_height(&self, width: u16) -> u16 { + self.view.desired_height(width) + } + fn render(&self, area: ratatui::layout::Rect, buf: &mut Buffer) { self.view.render_ref(area, buf); } diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 33e3ee11e4..3ee724e687 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; use std::path::PathBuf; use std::sync::Arc; use std::time::Duration; @@ -44,6 +45,12 @@ use crate::history_cell::PatchEventType; use crate::user_approval_widget::ApprovalRequest; use codex_file_search::FileMatch; +struct RunningCommand { + command: Vec, + #[allow(dead_code)] + cwd: PathBuf, +} + pub(crate) struct ChatWidget<'a> { app_event_tx: AppEventSender, codex_op_tx: UnboundedSender, @@ -56,6 +63,7 @@ pub(crate) struct ChatWidget<'a> { // We wait for the final AgentMessage event and then emit the full text // at once into scrollback so the history contains a single message. answer_buffer: String, + running_commands: HashMap, } struct UserMessage { @@ -140,11 +148,12 @@ impl ChatWidget<'_> { token_usage: TokenUsage::default(), reasoning_buffer: String::new(), answer_buffer: String::new(), + running_commands: HashMap::new(), } } - pub fn desired_height(&self) -> u16 { - self.bottom_pane.desired_height() + pub fn desired_height(&self, width: u16) -> u16 { + self.bottom_pane.desired_height(width) } pub(crate) fn handle_key_event(&mut self, key_event: KeyEvent) { @@ -343,12 +352,18 @@ impl ChatWidget<'_> { self.request_redraw(); } EventMsg::ExecCommandBegin(ExecCommandBeginEvent { - call_id: _, + call_id, command, - cwd: _, + cwd, }) => { + self.running_commands.insert( + call_id, + RunningCommand { + command: command.clone(), + cwd: cwd.clone(), + }, + ); self.add_to_history(HistoryCell::new_active_exec_command(command)); - self.request_redraw(); } EventMsg::PatchApplyBegin(PatchApplyBeginEvent { call_id: _, @@ -361,7 +376,6 @@ impl ChatWidget<'_> { PatchEventType::ApplyBegin { auto_approved }, changes, )); - self.request_redraw(); } EventMsg::ExecCommandEnd(ExecCommandEndEvent { call_id, @@ -369,8 +383,9 @@ impl ChatWidget<'_> { stdout, stderr, }) => { + let cmd = self.running_commands.remove(&call_id); self.add_to_history(HistoryCell::new_completed_exec_command( - call_id, + cmd.map(|cmd| cmd.command).unwrap_or_else(|| vec![call_id]), CommandOutput { exit_code, stdout, @@ -384,7 +399,6 @@ impl ChatWidget<'_> { invocation, }) => { self.add_to_history(HistoryCell::new_active_mcp_tool_call(invocation)); - self.request_redraw(); } EventMsg::McpToolCallEnd(McpToolCallEndEvent { call_id: _, @@ -419,7 +433,6 @@ impl ChatWidget<'_> { } event => { self.add_to_history(HistoryCell::new_background_event(format!("{event:?}"))); - self.request_redraw(); } } } @@ -436,7 +449,6 @@ impl ChatWidget<'_> { pub(crate) fn add_diff_output(&mut self, diff_output: String) { self.add_to_history(HistoryCell::new_diff_output(diff_output.clone())); - self.request_redraw(); } /// Forward file-search results to the bottom pane. diff --git a/codex-rs/tui/src/history_cell.rs b/codex-rs/tui/src/history_cell.rs index 04279a01f8..956a0cc7ef 100644 --- a/codex-rs/tui/src/history_cell.rs +++ b/codex-rs/tui/src/history_cell.rs @@ -1,4 +1,4 @@ -use crate::exec_command::escape_command; +use crate::exec_command::strip_bash_lc_and_escape; use crate::markdown::append_markdown; use crate::text_block::TextBlock; use crate::text_formatting::format_and_truncate_tool_result; @@ -246,7 +246,7 @@ impl HistoryCell { } pub(crate) fn new_active_exec_command(command: Vec) -> Self { - let command_escaped = escape_command(&command); + let command_escaped = strip_bash_lc_and_escape(&command); let lines: Vec> = vec![ Line::from(vec!["command".magenta(), " running...".dim()]), @@ -259,7 +259,7 @@ impl HistoryCell { } } - pub(crate) fn new_completed_exec_command(command: String, output: CommandOutput) -> Self { + pub(crate) fn new_completed_exec_command(command: Vec, output: CommandOutput) -> Self { let CommandOutput { exit_code, stdout, @@ -283,7 +283,8 @@ impl HistoryCell { let src = if exit_code == 0 { stdout } else { stderr }; - lines.push(Line::from(format!("$ {command}"))); + let cmdline = strip_bash_lc_and_escape(&command); + lines.push(Line::from(format!("$ {cmdline}"))); let mut lines_iter = src.lines(); for raw in lines_iter.by_ref().take(TOOL_CALL_MAX_LINES) { lines.push(ansi_escape_line(raw).dim()); diff --git a/codex-rs/tui/src/slash_command.rs b/codex-rs/tui/src/slash_command.rs index 603eb721cd..7df1bcbdec 100644 --- a/codex-rs/tui/src/slash_command.rs +++ b/codex-rs/tui/src/slash_command.rs @@ -15,6 +15,8 @@ pub enum SlashCommand { New, Diff, Quit, + #[cfg(debug_assertions)] + TestApproval, } impl SlashCommand { @@ -26,6 +28,8 @@ impl SlashCommand { SlashCommand::Diff => { "Show git diff of the working directory (including untracked files)" } + #[cfg(debug_assertions)] + SlashCommand::TestApproval => "Test approval request", } } diff --git a/codex-rs/tui/src/status_indicator_widget.rs b/codex-rs/tui/src/status_indicator_widget.rs index 973ef09818..7e6d267481 100644 --- a/codex-rs/tui/src/status_indicator_widget.rs +++ b/codex-rs/tui/src/status_indicator_widget.rs @@ -73,6 +73,10 @@ impl StatusIndicatorWidget { } } + pub fn desired_height(&self, _width: u16) -> u16 { + 1 + } + /// Update the line that is displayed in the widget. pub(crate) fn update_text(&mut self, text: String) { self.text = text.replace(['\n', '\r'], " "); @@ -91,8 +95,8 @@ impl WidgetRef for StatusIndicatorWidget { let widget_style = Style::default(); let block = Block::default() .padding(Padding::new(1, 0, 0, 0)) - .borders(Borders::ALL) - .border_type(BorderType::Rounded) + .borders(Borders::LEFT) + .border_type(BorderType::QuadrantOutside) .border_style(widget_style.dim()); // Animated 3‑dot pattern inside brackets. The *active* dot is bold // white, the others are dim. diff --git a/codex-rs/tui/src/user_approval_widget.rs b/codex-rs/tui/src/user_approval_widget.rs index a161c2c399..855a7ea3db 100644 --- a/codex-rs/tui/src/user_approval_widget.rs +++ b/codex-rs/tui/src/user_approval_widget.rs @@ -17,13 +17,11 @@ use ratatui::layout::Rect; use ratatui::prelude::*; use ratatui::text::Line; use ratatui::text::Span; -use ratatui::widgets::Block; -use ratatui::widgets::BorderType; -use ratatui::widgets::Borders; use ratatui::widgets::List; use ratatui::widgets::Paragraph; use ratatui::widgets::Widget; use ratatui::widgets::WidgetRef; +use ratatui::widgets::Wrap; use tui_input::Input; use tui_input::backend::crossterm::EventHandler; @@ -134,10 +132,9 @@ impl UserApprovalWidget<'_> { None => cwd.display().to_string(), }; let mut contents: Vec = vec![ - Line::from("Shell Command".bold()), - Line::from(""), Line::from(vec![ - format!("{cwd_str}$").dim(), + Span::from(cwd_str).dim(), + Span::from("$"), Span::from(format!(" {cmd}")), ]), Line::from(""), @@ -147,7 +144,7 @@ impl UserApprovalWidget<'_> { contents.push(Line::from("")); } contents.extend(vec![Line::from("Allow command?"), Line::from("")]); - Paragraph::new(contents) + Paragraph::new(contents).wrap(Wrap { trim: false }) } ApprovalRequest::ApplyPatch { reason, grant_root, .. @@ -313,21 +310,21 @@ impl UserApprovalWidget<'_> { pub(crate) fn is_complete(&self) -> bool { self.done } + + pub(crate) fn desired_height(&self, width: u16) -> u16 { + self.get_confirmation_prompt_height(width - 2) + SELECT_OPTIONS.len() as u16 + 2 + } } const PLAIN: Style = Style::new(); -const BLUE_FG: Style = Style::new().fg(Color::Blue); +const BLUE_FG: Style = Style::new().fg(Color::LightCyan); impl WidgetRef for &UserApprovalWidget<'_> { fn render_ref(&self, area: Rect, buf: &mut Buffer) { // Take the area, wrap it in a block with a border, and divide up the // remaining area into two chunks: one for the confirmation prompt and // one for the response. - let outer = Block::default() - .title("Review") - .borders(Borders::ALL) - .border_type(BorderType::Rounded); - let inner = outer.inner(area); + let inner = area.inner(Margin::new(0, 2)); // Determine how many rows we can allocate for the static confirmation // prompt while *always* keeping enough space for the interactive @@ -384,8 +381,18 @@ impl WidgetRef for &UserApprovalWidget<'_> { } }; - outer.render(area, buf); + let border = ("◢◤") + .repeat((area.width / 2).into()) + .fg(Color::LightYellow); + + border.render_ref(area, buf); + Paragraph::new(" Execution Request ".bold().black().on_light_yellow()) + .alignment(Alignment::Center) + .render_ref(area, buf); + self.confirmation_prompt.clone().render(prompt_chunk, buf); - Widget::render(List::new(lines), response_chunk, buf); + List::new(lines).render_ref(response_chunk, buf); + + border.render_ref(Rect::new(0, area.y + area.height - 1, area.width, 1), buf); } } From be0cd3430053441027654630ccd4db736235ecda Mon Sep 17 00:00:00 2001 From: Jeremy Rose <172423086+nornagon-openai@users.noreply.github.com> Date: Thu, 31 Jul 2025 09:17:59 -0700 Subject: [PATCH 015/100] fix git tests (#1747) the git tests were failing on my local machine due to gpg signing config in my ~/.gitconfig. tests should not be affected by ~/.gitconfig, so configure them to ignore it. --- codex-rs/chatgpt/tests/apply_command_e2e.rs | 9 +++++++++ codex-rs/core/src/git_info.rs | 9 +++++++++ codex-rs/core/tests/cli_stream.rs | 11 +++++++++++ 3 files changed, 29 insertions(+) diff --git a/codex-rs/chatgpt/tests/apply_command_e2e.rs b/codex-rs/chatgpt/tests/apply_command_e2e.rs index 45c33bedb4..f1a35e1521 100644 --- a/codex-rs/chatgpt/tests/apply_command_e2e.rs +++ b/codex-rs/chatgpt/tests/apply_command_e2e.rs @@ -10,8 +10,13 @@ use tokio::process::Command; async fn create_temp_git_repo() -> anyhow::Result { let temp_dir = TempDir::new()?; let repo_path = temp_dir.path(); + let envs = vec![ + ("GIT_CONFIG_GLOBAL", "/dev/null"), + ("GIT_CONFIG_NOSYSTEM", "1"), + ]; let output = Command::new("git") + .envs(envs.clone()) .args(["init"]) .current_dir(repo_path) .output() @@ -25,12 +30,14 @@ async fn create_temp_git_repo() -> anyhow::Result { } Command::new("git") + .envs(envs.clone()) .args(["config", "user.email", "test@example.com"]) .current_dir(repo_path) .output() .await?; Command::new("git") + .envs(envs.clone()) .args(["config", "user.name", "Test User"]) .current_dir(repo_path) .output() @@ -39,12 +46,14 @@ async fn create_temp_git_repo() -> anyhow::Result { std::fs::write(repo_path.join("README.md"), "# Test Repo\n")?; Command::new("git") + .envs(envs.clone()) .args(["add", "README.md"]) .current_dir(repo_path) .output() .await?; let output = Command::new("git") + .envs(envs.clone()) .args(["commit", "-m", "Initial commit"]) .current_dir(repo_path) .output() diff --git a/codex-rs/core/src/git_info.rs b/codex-rs/core/src/git_info.rs index cf959d32d1..f5dc016e66 100644 --- a/codex-rs/core/src/git_info.rs +++ b/codex-rs/core/src/git_info.rs @@ -111,9 +111,14 @@ mod tests { // Helper function to create a test git repository async fn create_test_git_repo(temp_dir: &TempDir) -> PathBuf { let repo_path = temp_dir.path().to_path_buf(); + let envs = vec![ + ("GIT_CONFIG_GLOBAL", "/dev/null"), + ("GIT_CONFIG_NOSYSTEM", "1"), + ]; // Initialize git repo Command::new("git") + .envs(envs.clone()) .args(["init"]) .current_dir(&repo_path) .output() @@ -122,6 +127,7 @@ mod tests { // Configure git user (required for commits) Command::new("git") + .envs(envs.clone()) .args(["config", "user.name", "Test User"]) .current_dir(&repo_path) .output() @@ -129,6 +135,7 @@ mod tests { .expect("Failed to set git user name"); Command::new("git") + .envs(envs.clone()) .args(["config", "user.email", "test@example.com"]) .current_dir(&repo_path) .output() @@ -140,6 +147,7 @@ mod tests { fs::write(&test_file, "test content").expect("Failed to write test file"); Command::new("git") + .envs(envs.clone()) .args(["add", "."]) .current_dir(&repo_path) .output() @@ -147,6 +155,7 @@ mod tests { .expect("Failed to add files"); Command::new("git") + .envs(envs.clone()) .args(["commit", "-m", "Initial commit"]) .current_dir(&repo_path) .output() diff --git a/codex-rs/core/tests/cli_stream.rs b/codex-rs/core/tests/cli_stream.rs index 0ab7bd0bb2..ee0377fc10 100644 --- a/codex-rs/core/tests/cli_stream.rs +++ b/codex-rs/core/tests/cli_stream.rs @@ -460,9 +460,14 @@ async fn integration_git_info_unit_test() { // 1. Create temp directory for git repo let temp_dir = TempDir::new().unwrap(); let git_repo = temp_dir.path().to_path_buf(); + let envs = vec![ + ("GIT_CONFIG_GLOBAL", "/dev/null"), + ("GIT_CONFIG_NOSYSTEM", "1"), + ]; // 2. Initialize a git repository with some content let init_output = std::process::Command::new("git") + .envs(envs.clone()) .args(["init"]) .current_dir(&git_repo) .output() @@ -471,12 +476,14 @@ async fn integration_git_info_unit_test() { // Configure git user (required for commits) std::process::Command::new("git") + .envs(envs.clone()) .args(["config", "user.name", "Integration Test"]) .current_dir(&git_repo) .output() .unwrap(); std::process::Command::new("git") + .envs(envs.clone()) .args(["config", "user.email", "test@example.com"]) .current_dir(&git_repo) .output() @@ -487,12 +494,14 @@ async fn integration_git_info_unit_test() { std::fs::write(&test_file, "integration test content").unwrap(); std::process::Command::new("git") + .envs(envs.clone()) .args(["add", "."]) .current_dir(&git_repo) .output() .unwrap(); let commit_output = std::process::Command::new("git") + .envs(envs.clone()) .args(["commit", "-m", "Integration test commit"]) .current_dir(&git_repo) .output() @@ -501,6 +510,7 @@ async fn integration_git_info_unit_test() { // Create a branch to test branch detection std::process::Command::new("git") + .envs(envs.clone()) .args(["checkout", "-b", "integration-test-branch"]) .current_dir(&git_repo) .output() @@ -508,6 +518,7 @@ async fn integration_git_info_unit_test() { // Add a remote to test repository URL detection std::process::Command::new("git") + .envs(envs.clone()) .args([ "remote", "add", From 861ba8640356139ba15eebb89abc164a7b86d97d Mon Sep 17 00:00:00 2001 From: easong-openai Date: Thu, 31 Jul 2025 09:19:08 -0700 Subject: [PATCH 016/100] Show error message after panic (#1752) Previously we were swallowing errors and silently exiting, which isn't great for helping users help us. --- codex-rs/tui/src/lib.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index 351fab4df8..6b5fe7f7ae 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -176,9 +176,13 @@ fn run_ratatui_app( color_eyre::install()?; // Forward panic reports through tracing so they appear in the UI status - // line instead of interleaving raw panic output with the interface. - std::panic::set_hook(Box::new(|info| { + // line, but do not swallow the default/color-eyre panic handler. + // Chain to the previous hook so users still get a rich panic report + // (including backtraces) after we restore the terminal. + let prev_hook = std::panic::take_hook(); + std::panic::set_hook(Box::new(move |info| { tracing::error!("panic: {info}"); + prev_hook(info); })); let mut terminal = tui::init(&config)?; terminal.clear()?; From 96654a5d522fc885a19d38c1905087c7ae9b30b4 Mon Sep 17 00:00:00 2001 From: Jeremy Rose <172423086+nornagon-openai@users.noreply.github.com> Date: Thu, 31 Jul 2025 09:59:36 -0700 Subject: [PATCH 017/100] clamp render area to terminal size (#1758) this fixes a couple of panics that would happen when trying to render something larger than the terminal, or insert history lines when the top of the viewport is at y=0. --- codex-rs/tui/src/app.rs | 2 +- codex-rs/tui/src/insert_history.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index ad4bc24fd4..44c1875d40 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -362,7 +362,7 @@ impl App<'_> { AppState::GitWarning { .. } => 10, }; let mut area = terminal.viewport_area; - area.height = desired_height; + area.height = desired_height.min(size.height); area.width = size.width; if area.bottom() > size.height { terminal diff --git a/codex-rs/tui/src/insert_history.rs b/codex-rs/tui/src/insert_history.rs index 54faf4beb8..efd08a71c2 100644 --- a/codex-rs/tui/src/insert_history.rs +++ b/codex-rs/tui/src/insert_history.rs @@ -36,12 +36,12 @@ pub(crate) fn insert_history_lines(terminal: &mut tui::Tui, lines: Vec) { .backend_mut() .scroll_region_down(area.top()..screen_size.height, scroll_amount) .ok(); - let cursor_top = area.top() - 1; + let cursor_top = area.top().saturating_sub(1); area.y += scroll_amount; terminal.set_viewport_area(area); cursor_top } else { - area.top() - 1 + area.top().saturating_sub(1) }; // Limit the scroll region to the lines from the top of the screen to the From 549846b29ad52f6cb4f8560365a731966054a9b3 Mon Sep 17 00:00:00 2001 From: pakrym-oai Date: Thu, 31 Jul 2025 10:48:49 -0700 Subject: [PATCH 018/100] Add codex login --api-key (#1759) Allow setting the API key via `codex login --api-key` --- codex-rs/Cargo.lock | 1 + codex-rs/chatgpt/src/chatgpt_token.rs | 2 +- codex-rs/cli/src/login.rs | 32 +++- codex-rs/cli/src/main.rs | 10 +- codex-rs/cli/src/proto.rs | 2 +- codex-rs/core/src/codex_wrapper.rs | 2 +- codex-rs/core/tests/client.rs | 8 +- codex-rs/login/Cargo.toml | 3 + codex-rs/login/src/lib.rs | 202 +++++++++++++++++++++----- codex-rs/tui/src/lib.rs | 2 +- 10 files changed, 218 insertions(+), 46 deletions(-) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 120050c227..87d59b21be 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -793,6 +793,7 @@ dependencies = [ "reqwest", "serde", "serde_json", + "tempfile", "tokio", ] diff --git a/codex-rs/chatgpt/src/chatgpt_token.rs b/codex-rs/chatgpt/src/chatgpt_token.rs index 55ebc22a08..55b6886c59 100644 --- a/codex-rs/chatgpt/src/chatgpt_token.rs +++ b/codex-rs/chatgpt/src/chatgpt_token.rs @@ -18,7 +18,7 @@ pub fn set_chatgpt_token_data(value: TokenData) { /// Initialize the ChatGPT token from auth.json file pub async fn init_chatgpt_token_from_auth(codex_home: &Path) -> std::io::Result<()> { - let auth = codex_login::load_auth(codex_home)?; + let auth = codex_login::load_auth(codex_home, true)?; if let Some(auth) = auth { let token_data = auth.get_token_data().await?; set_chatgpt_token_data(token_data); diff --git a/codex-rs/cli/src/login.rs b/codex-rs/cli/src/login.rs index 390c310030..4fa13f0cc6 100644 --- a/codex-rs/cli/src/login.rs +++ b/codex-rs/cli/src/login.rs @@ -1,8 +1,12 @@ +use std::env; + use codex_common::CliConfigOverrides; use codex_core::config::Config; use codex_core::config::ConfigOverrides; use codex_login::AuthMode; +use codex_login::OPENAI_API_KEY_ENV_VAR; use codex_login::load_auth; +use codex_login::login_with_api_key; use codex_login::login_with_chatgpt; pub async fn run_login_with_chatgpt(cli_config_overrides: CliConfigOverrides) -> ! { @@ -21,14 +25,40 @@ pub async fn run_login_with_chatgpt(cli_config_overrides: CliConfigOverrides) -> } } +pub async fn run_login_with_api_key( + cli_config_overrides: CliConfigOverrides, + api_key: String, +) -> ! { + let config = load_config_or_exit(cli_config_overrides); + + match login_with_api_key(&config.codex_home, &api_key) { + Ok(_) => { + eprintln!("Successfully logged in"); + std::process::exit(0); + } + Err(e) => { + eprintln!("Error logging in: {e}"); + std::process::exit(1); + } + } +} + pub async fn run_login_status(cli_config_overrides: CliConfigOverrides) -> ! { let config = load_config_or_exit(cli_config_overrides); - match load_auth(&config.codex_home) { + match load_auth(&config.codex_home, true) { Ok(Some(auth)) => match auth.mode { AuthMode::ApiKey => { if let Some(api_key) = auth.api_key.as_deref() { eprintln!("Logged in using an API key - {}", safe_format_key(api_key)); + + if let Ok(env_api_key) = env::var(OPENAI_API_KEY_ENV_VAR) { + if env_api_key == api_key { + eprintln!( + " API loaded from OPENAI_API_KEY environment variable or .env file" + ); + } + } } else { eprintln!("Logged in using an API key"); } diff --git a/codex-rs/cli/src/main.rs b/codex-rs/cli/src/main.rs index c5fd69f9cd..27f8312193 100644 --- a/codex-rs/cli/src/main.rs +++ b/codex-rs/cli/src/main.rs @@ -8,6 +8,7 @@ use codex_chatgpt::apply_command::run_apply_command; use codex_cli::LandlockCommand; use codex_cli::SeatbeltCommand; use codex_cli::login::run_login_status; +use codex_cli::login::run_login_with_api_key; use codex_cli::login::run_login_with_chatgpt; use codex_cli::proto; use codex_common::CliConfigOverrides; @@ -92,6 +93,9 @@ struct LoginCommand { #[clap(skip)] config_overrides: CliConfigOverrides, + #[arg(long = "api-key", value_name = "API_KEY")] + api_key: Option, + #[command(subcommand)] action: Option, } @@ -133,7 +137,11 @@ async fn cli_main(codex_linux_sandbox_exe: Option) -> anyhow::Result<() run_login_status(login_cli.config_overrides).await; } None => { - run_login_with_chatgpt(login_cli.config_overrides).await; + if let Some(api_key) = login_cli.api_key { + run_login_with_api_key(login_cli.config_overrides, api_key).await; + } else { + run_login_with_chatgpt(login_cli.config_overrides).await; + } } } } diff --git a/codex-rs/cli/src/proto.rs b/codex-rs/cli/src/proto.rs index 291e1680f1..9f9a94ed4d 100644 --- a/codex-rs/cli/src/proto.rs +++ b/codex-rs/cli/src/proto.rs @@ -36,7 +36,7 @@ pub async fn run_main(opts: ProtoCli) -> anyhow::Result<()> { .map_err(anyhow::Error::msg)?; let config = Config::load_with_cli_overrides(overrides_vec, ConfigOverrides::default())?; - let auth = load_auth(&config.codex_home)?; + let auth = load_auth(&config.codex_home, true)?; let ctrl_c = notify_on_sigint(); let CodexSpawnOk { codex, .. } = Codex::spawn(config, auth, ctrl_c.clone()).await?; let codex = Arc::new(codex); diff --git a/codex-rs/core/src/codex_wrapper.rs b/codex-rs/core/src/codex_wrapper.rs index 1e26a9ebed..eeb4a7b470 100644 --- a/codex-rs/core/src/codex_wrapper.rs +++ b/codex-rs/core/src/codex_wrapper.rs @@ -26,7 +26,7 @@ pub struct CodexConversation { /// that callers can surface the information to the UI. pub async fn init_codex(config: Config) -> anyhow::Result { let ctrl_c = notify_on_sigint(); - let auth = load_auth(&config.codex_home)?; + let auth = load_auth(&config.codex_home, true)?; let CodexSpawnOk { codex, init_id, diff --git a/codex-rs/core/tests/client.rs b/codex-rs/core/tests/client.rs index 67d95cb8f6..1286928ec0 100644 --- a/codex-rs/core/tests/client.rs +++ b/codex-rs/core/tests/client.rs @@ -327,14 +327,14 @@ fn auth_from_token(id_token: String) -> CodexAuth { AuthMode::ChatGPT, PathBuf::new(), Some(AuthDotJson { - tokens: TokenData { + openai_api_key: None, + tokens: Some(TokenData { id_token, access_token: "Access Token".to_string(), refresh_token: "test".to_string(), account_id: None, - }, - last_refresh: Utc::now(), - openai_api_key: None, + }), + last_refresh: Some(Utc::now()), }), ) } diff --git a/codex-rs/login/Cargo.toml b/codex-rs/login/Cargo.toml index e10666b092..650291b3bc 100644 --- a/codex-rs/login/Cargo.toml +++ b/codex-rs/login/Cargo.toml @@ -18,3 +18,6 @@ tokio = { version = "1", features = [ "rt-multi-thread", "signal", ] } + +[dev-dependencies] +tempfile = "3" diff --git a/codex-rs/login/src/lib.rs b/codex-rs/login/src/lib.rs index 47dbbca9fb..2f0aeb60bd 100644 --- a/codex-rs/login/src/lib.rs +++ b/codex-rs/login/src/lib.rs @@ -20,7 +20,7 @@ use tokio::process::Command; const SOURCE_FOR_PYTHON_SERVER: &str = include_str!("./login_with_chatgpt.py"); const CLIENT_ID: &str = "app_EMoamEEZ73f0CkXaXp7hrann"; -const OPENAI_API_KEY_ENV_VAR: &str = "OPENAI_API_KEY"; +pub const OPENAI_API_KEY_ENV_VAR: &str = "OPENAI_API_KEY"; #[derive(Clone, Debug, PartialEq)] pub enum AuthMode { @@ -70,13 +70,16 @@ impl CodexAuth { pub async fn get_token_data(&self) -> Result { #[expect(clippy::unwrap_used)] let auth_dot_json = self.auth_dot_json.lock().unwrap().clone(); - match auth_dot_json { - Some(auth_dot_json) => { - if auth_dot_json.last_refresh < Utc::now() - chrono::Duration::days(28) { + Some(AuthDotJson { + tokens: Some(mut tokens), + last_refresh: Some(last_refresh), + .. + }) => { + if last_refresh < Utc::now() - chrono::Duration::days(28) { let refresh_response = tokio::time::timeout( Duration::from_secs(60), - try_refresh_token(auth_dot_json.tokens.refresh_token.clone()), + try_refresh_token(tokens.refresh_token.clone()), ) .await .map_err(|_| { @@ -92,13 +95,21 @@ impl CodexAuth { ) .await?; + tokens = updated_auth_dot_json + .tokens + .clone() + .ok_or(std::io::Error::other( + "Token data is not available after refresh.", + ))?; + #[expect(clippy::unwrap_used)] - let mut auth_dot_json = self.auth_dot_json.lock().unwrap(); - *auth_dot_json = Some(updated_auth_dot_json); + let mut auth_lock = self.auth_dot_json.lock().unwrap(); + *auth_lock = Some(updated_auth_dot_json); } - Ok(auth_dot_json.tokens.clone()) + + Ok(tokens) } - None => Err(std::io::Error::other("Token data is not available.")), + _ => Err(std::io::Error::other("Token data is not available.")), } } @@ -115,8 +126,8 @@ impl CodexAuth { } // Loads the available auth information from the auth.json or OPENAI_API_KEY environment variable. -pub fn load_auth(codex_home: &Path) -> std::io::Result> { - let auth_file = codex_home.join("auth.json"); +pub fn load_auth(codex_home: &Path, include_env_var: bool) -> std::io::Result> { + let auth_file = get_auth_file(codex_home); let auth_dot_json = try_read_auth_json(&auth_file).ok(); @@ -125,12 +136,21 @@ pub fn load_auth(codex_home: &Path) -> std::io::Result> { .and_then(|a| a.openai_api_key.clone()) .filter(|s| !s.is_empty()); - let openai_api_key = env::var(OPENAI_API_KEY_ENV_VAR) - .ok() - .filter(|s| !s.is_empty()) - .or(auth_json_api_key); + let openai_api_key = if include_env_var { + env::var(OPENAI_API_KEY_ENV_VAR) + .ok() + .filter(|s| !s.is_empty()) + .or(auth_json_api_key) + } else { + auth_json_api_key + }; - if openai_api_key.is_none() && auth_dot_json.is_none() { + let has_tokens = auth_dot_json + .as_ref() + .and_then(|a| a.tokens.as_ref()) + .is_some(); + + if openai_api_key.is_none() && !has_tokens { return Ok(None); } @@ -148,6 +168,10 @@ pub fn load_auth(codex_home: &Path) -> std::io::Result> { })) } +fn get_auth_file(codex_home: &Path) -> PathBuf { + codex_home.join("auth.json") +} + /// Run `python3 -c {{SOURCE_FOR_PYTHON_SERVER}}` with the CODEX_HOME /// environment variable set to the provided `codex_home` path. If the /// subprocess exits 0, read the OPENAI_API_KEY property out of @@ -187,6 +211,15 @@ pub async fn login_with_chatgpt(codex_home: &Path, capture_output: bool) -> std: } } +pub fn login_with_api_key(codex_home: &Path, api_key: &str) -> std::io::Result<()> { + let auth_dot_json = AuthDotJson { + openai_api_key: Some(api_key.to_string()), + tokens: None, + last_refresh: None, + }; + write_auth_json(&get_auth_file(codex_home), &auth_dot_json) +} + /// Attempt to read and refresh the `auth.json` file in the given `CODEX_HOME` directory. /// Returns the full AuthDotJson structure after refreshing if necessary. pub fn try_read_auth_json(auth_file: &Path) -> std::io::Result { @@ -198,35 +231,38 @@ pub fn try_read_auth_json(auth_file: &Path) -> std::io::Result { Ok(auth_dot_json) } -async fn update_tokens( - auth_file: &Path, - id_token: String, - access_token: Option, - refresh_token: Option, -) -> std::io::Result { +fn write_auth_json(auth_file: &Path, auth_dot_json: &AuthDotJson) -> std::io::Result<()> { + let json_data = serde_json::to_string_pretty(auth_dot_json)?; let mut options = OpenOptions::new(); options.truncate(true).write(true).create(true); #[cfg(unix)] { options.mode(0o600); } + let mut file = options.open(auth_file)?; + file.write_all(json_data.as_bytes())?; + file.flush()?; + Ok(()) +} + +async fn update_tokens( + auth_file: &Path, + id_token: String, + access_token: Option, + refresh_token: Option, +) -> std::io::Result { let mut auth_dot_json = try_read_auth_json(auth_file)?; - auth_dot_json.tokens.id_token = id_token.to_string(); + let tokens = auth_dot_json.tokens.get_or_insert_with(TokenData::default); + tokens.id_token = id_token.to_string(); if let Some(access_token) = access_token { - auth_dot_json.tokens.access_token = access_token.to_string(); + tokens.access_token = access_token.to_string(); } if let Some(refresh_token) = refresh_token { - auth_dot_json.tokens.refresh_token = refresh_token.to_string(); - } - auth_dot_json.last_refresh = Utc::now(); - - let json_data = serde_json::to_string_pretty(&auth_dot_json)?; - { - let mut file = options.open(auth_file)?; - file.write_all(json_data.as_bytes())?; - file.flush()?; + tokens.refresh_token = refresh_token.to_string(); } + auth_dot_json.last_refresh = Some(Utc::now()); + write_auth_json(auth_file, &auth_dot_json)?; Ok(auth_dot_json) } @@ -282,12 +318,14 @@ pub struct AuthDotJson { #[serde(rename = "OPENAI_API_KEY")] pub openai_api_key: Option, - pub tokens: TokenData, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tokens: Option, - pub last_refresh: DateTime, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_refresh: Option>, } -#[derive(Deserialize, Serialize, Clone, Debug, PartialEq)] +#[derive(Deserialize, Serialize, Clone, Debug, PartialEq, Default)] pub struct TokenData { /// This is a JWT. pub id_token: String, @@ -299,3 +337,95 @@ pub struct TokenData { pub account_id: Option, } + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::tempdir; + + #[test] + #[expect(clippy::unwrap_used)] + fn writes_api_key_and_loads_auth() { + let dir = tempdir().unwrap(); + login_with_api_key(dir.path(), "sk-test-key").unwrap(); + let auth = load_auth(dir.path(), false).unwrap().unwrap(); + assert_eq!(auth.mode, AuthMode::ApiKey); + assert_eq!(auth.api_key.as_deref(), Some("sk-test-key")); + } + + #[test] + #[expect(clippy::unwrap_used)] + fn loads_from_env_var_if_env_var_exists() { + let dir = tempdir().unwrap(); + + let env_var = std::env::var(OPENAI_API_KEY_ENV_VAR); + + if let Ok(env_var) = env_var { + let auth = load_auth(dir.path(), true).unwrap().unwrap(); + assert_eq!(auth.mode, AuthMode::ApiKey); + assert_eq!(auth.api_key, Some(env_var)); + } + } + + #[tokio::test] + #[expect(clippy::unwrap_used)] + async fn loads_token_data_from_auth_json() { + let dir = tempdir().unwrap(); + let auth_file = dir.path().join("auth.json"); + std::fs::write( + auth_file, + format!( + r#" + {{ + "OPENAI_API_KEY": null, + "tokens": {{ + "id_token": "test-id-token", + "access_token": "test-access-token", + "refresh_token": "test-refresh-token" + }}, + "last_refresh": "{}" + }} + "#, + Utc::now().to_rfc3339() + ), + ) + .unwrap(); + + let auth = load_auth(dir.path(), false).unwrap().unwrap(); + assert_eq!(auth.mode, AuthMode::ChatGPT); + assert_eq!(auth.api_key, None); + assert_eq!( + auth.get_token_data().await.unwrap(), + TokenData { + id_token: "test-id-token".to_string(), + access_token: "test-access-token".to_string(), + refresh_token: "test-refresh-token".to_string(), + account_id: None, + } + ); + } + + #[tokio::test] + #[expect(clippy::unwrap_used)] + async fn loads_api_key_from_auth_json() { + let dir = tempdir().unwrap(); + let auth_file = dir.path().join("auth.json"); + std::fs::write( + auth_file, + r#" + { + "OPENAI_API_KEY": "sk-test-key", + "tokens": null, + "last_refresh": null + } + "#, + ) + .unwrap(); + + let auth = load_auth(dir.path(), false).unwrap().unwrap(); + assert_eq!(auth.mode, AuthMode::ApiKey); + assert_eq!(auth.api_key, Some("sk-test-key".to_string())); + + assert!(auth.get_token_data().await.is_err()); + } +} diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index 6b5fe7f7ae..7e987f6ff4 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -226,7 +226,7 @@ fn should_show_login_screen(config: &Config) -> bool { // Reading the OpenAI API key is an async operation because it may need // to refresh the token. Block on it. let codex_home = config.codex_home.clone(); - match load_auth(&codex_home) { + match load_auth(&codex_home, true) { Ok(Some(_)) => false, Ok(None) => true, Err(err) => { From 06c786b2da8f5bc1b971883d16e0785665846900 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 31 Jul 2025 11:13:57 -0700 Subject: [PATCH 019/100] fix: ensure PatchApplyBeginEvent and PatchApplyEndEvent are dispatched reliably (#1760) This is a follow-up to https://github.com/openai/codex/pull/1705, as that PR inadvertently lost the logic where `PatchApplyBeginEvent` and `PatchApplyEndEvent` events were sent when patches were auto-approved. Though as part of this fix, I believe this also makes an important safety fix to `assess_patch_safety()`, as there was a case that returned `SandboxType::None`, which arguably is the thing we were trying to avoid in #1705. On a high level, we want there to be only one codepath where `apply_patch` happens, which should be unified with the patch to run `exec`, in general, so that sandboxing is applied consistently for both cases. Prior to this change, `apply_patch()` in `core` would either: * exit early, delegating to `exec()` to shell out to `apply_patch` using the appropriate sandbox * proceed to run the logic for `apply_patch` in memory https://github.com/openai/codex/blob/549846b29ad52f6cb4f8560365a731966054a9b3/codex-rs/core/src/apply_patch.rs#L61-L63 In this implementation, only the latter would dispatch `PatchApplyBeginEvent` and `PatchApplyEndEvent`, though the former would dispatch `ExecCommandBeginEvent` and `ExecCommandEndEvent` for the `apply_patch` call (or, more specifically, the `codex --codex-run-as-apply-patch PATCH` call). To unify things in this PR, we: * Eliminate the back half of the `apply_patch()` function, and instead have it also return with `DelegateToExec`, though we add an extra field to the return value, `user_explicitly_approved_this_action`. * In `codex.rs` where we process `DelegateToExec`, we use `SandboxType::None` when `user_explicitly_approved_this_action` is `true`. This means **we no longer run the apply_patch logic in memory**, as we always `exec()`. (Note this is what allowed us to delete so much code in `apply_patch.rs`.) * In `codex.rs`, we further update `notify_exec_command_begin()` and `notify_exec_command_end()` to take additional fields to determine what type of notification to send: `ExecCommand` or `PatchApply`. Admittedly, this PR also drops some of the functionality about giving the user the opportunity to expand the set of writable roots as part of approving the `apply_patch` command. I'm not sure how much that was used, and we should probably rethink how that works as we are currently tidying up the protocol to the TUI, in general. --- codex-rs/core/src/apply_patch.rs | 343 +++---------------------------- codex-rs/core/src/codex.rs | 226 +++++++++++++------- codex-rs/core/src/safety.rs | 12 +- 3 files changed, 193 insertions(+), 388 deletions(-) diff --git a/codex-rs/core/src/apply_patch.rs b/codex-rs/core/src/apply_patch.rs index f116c790ab..dc11aed023 100644 --- a/codex-rs/core/src/apply_patch.rs +++ b/codex-rs/core/src/apply_patch.rs @@ -1,19 +1,12 @@ use crate::codex::Session; use crate::models::FunctionCallOutputPayload; use crate::models::ResponseInputItem; -use crate::protocol::Event; -use crate::protocol::EventMsg; use crate::protocol::FileChange; -use crate::protocol::PatchApplyBeginEvent; -use crate::protocol::PatchApplyEndEvent; use crate::protocol::ReviewDecision; use crate::safety::SafetyCheck; use crate::safety::assess_patch_safety; -use anyhow::Context; -use codex_apply_patch::AffectedPaths; use codex_apply_patch::ApplyPatchAction; use codex_apply_patch::ApplyPatchFileChange; -use codex_apply_patch::print_summary; use std::collections::HashMap; use std::path::Path; use std::path::PathBuf; @@ -26,12 +19,18 @@ pub(crate) enum InternalApplyPatchInvocation { /// result to use with the `shell` function call that contained `apply_patch`. Output(ResponseInputItem), - /// The `apply_patch` call was auto-approved, which means that, on the - /// surface, it appears to be safe, but it should be run in a sandbox if the - /// user has configured one because a path being written could be a hard - /// link to a file outside the writable folders, so only the sandbox can - /// faithfully prevent the write in that case. - DelegateToExec(ApplyPatchAction), + /// The `apply_patch` call was approved, either automatically because it + /// appears that it should be allowed based on the user's sandbox policy + /// *or* because the user explicitly approved it. In either case, we use + /// exec with [`CODEX_APPLY_PATCH_ARG1`] to realize the `apply_patch` call, + /// but [`ApplyPatchExec::auto_approved`] is used to determine the sandbox + /// used with the `exec()`. + DelegateToExec(ApplyPatchExec), +} + +pub(crate) struct ApplyPatchExec { + pub(crate) action: ApplyPatchAction, + pub(crate) user_explicitly_approved_this_action: bool, } impl From for InternalApplyPatchInvocation { @@ -52,254 +51,57 @@ pub(crate) async fn apply_patch( guard.clone() }; - let auto_approved = match assess_patch_safety( + match assess_patch_safety( &action, sess.approval_policy, &writable_roots_snapshot, &sess.cwd, ) { SafetyCheck::AutoApprove { .. } => { - return InternalApplyPatchInvocation::DelegateToExec(action); + InternalApplyPatchInvocation::DelegateToExec(ApplyPatchExec { + action, + user_explicitly_approved_this_action: false, + }) } SafetyCheck::AskUser => { // Compute a readable summary of path changes to include in the // approval request so the user can make an informed decision. + // + // Note that it might be worth expanding this approval request to + // give the user the option to expand the set of writable roots so + // that similar patches can be auto-approved in the future during + // this session. let rx_approve = sess .request_patch_approval(sub_id.to_owned(), call_id.to_owned(), &action, None, None) .await; match rx_approve.await.unwrap_or_default() { - ReviewDecision::Approved | ReviewDecision::ApprovedForSession => false, + ReviewDecision::Approved | ReviewDecision::ApprovedForSession => { + InternalApplyPatchInvocation::DelegateToExec(ApplyPatchExec { + action, + user_explicitly_approved_this_action: true, + }) + } ReviewDecision::Denied | ReviewDecision::Abort => { - return ResponseInputItem::FunctionCallOutput { + ResponseInputItem::FunctionCallOutput { call_id: call_id.to_owned(), output: FunctionCallOutputPayload { content: "patch rejected by user".to_string(), success: Some(false), }, } - .into(); + .into() } } } - SafetyCheck::Reject { reason } => { - return ResponseInputItem::FunctionCallOutput { - call_id: call_id.to_owned(), - output: FunctionCallOutputPayload { - content: format!("patch rejected: {reason}"), - success: Some(false), - }, - } - .into(); - } - }; - - // Verify write permissions before touching the filesystem. - let writable_snapshot = { - #[allow(clippy::unwrap_used)] - sess.writable_roots.lock().unwrap().clone() - }; - - if let Some(offending) = first_offending_path(&action, &writable_snapshot, &sess.cwd) { - let root = offending.parent().unwrap_or(&offending).to_path_buf(); - - let reason = Some(format!( - "grant write access to {} for this session", - root.display() - )); - - let rx = sess - .request_patch_approval( - sub_id.to_owned(), - call_id.to_owned(), - &action, - reason.clone(), - Some(root.clone()), - ) - .await; - - if !matches!( - rx.await.unwrap_or_default(), - ReviewDecision::Approved | ReviewDecision::ApprovedForSession - ) { - return ResponseInputItem::FunctionCallOutput { - call_id: call_id.to_owned(), - output: FunctionCallOutputPayload { - content: "patch rejected by user".to_string(), - success: Some(false), - }, - } - .into(); - } - - // user approved, extend writable roots for this session - #[allow(clippy::unwrap_used)] - sess.writable_roots.lock().unwrap().push(root); - } - - let _ = sess - .tx_event - .send(Event { - id: sub_id.to_owned(), - msg: EventMsg::PatchApplyBegin(PatchApplyBeginEvent { - call_id: call_id.to_owned(), - auto_approved, - changes: convert_apply_patch_to_protocol(&action), - }), - }) - .await; - - let mut stdout = Vec::new(); - let mut stderr = Vec::new(); - // Enforce writable roots. If a write is blocked, collect offending root - // and prompt the user to extend permissions. - let mut result = apply_changes_from_apply_patch_and_report(&action, &mut stdout, &mut stderr); - - if let Err(err) = &result { - if err.kind() == std::io::ErrorKind::PermissionDenied { - // Determine first offending path. - let offending_opt = action - .changes() - .iter() - .flat_map(|(path, change)| match change { - ApplyPatchFileChange::Add { .. } => vec![path.as_ref()], - ApplyPatchFileChange::Delete => vec![path.as_ref()], - ApplyPatchFileChange::Update { - move_path: Some(move_path), - .. - } => { - vec![path.as_ref(), move_path.as_ref()] - } - ApplyPatchFileChange::Update { - move_path: None, .. - } => vec![path.as_ref()], - }) - .find_map(|path: &Path| { - // ApplyPatchAction promises to guarantee absolute paths. - if !path.is_absolute() { - panic!("apply_patch invariant failed: path is not absolute: {path:?}"); - } - - let writable = { - #[allow(clippy::unwrap_used)] - let roots = sess.writable_roots.lock().unwrap(); - roots.iter().any(|root| path.starts_with(root)) - }; - if writable { - None - } else { - Some(path.to_path_buf()) - } - }); - - if let Some(offending) = offending_opt { - let root = offending.parent().unwrap_or(&offending).to_path_buf(); - - let reason = Some(format!( - "grant write access to {} for this session", - root.display() - )); - let rx = sess - .request_patch_approval( - sub_id.to_owned(), - call_id.to_owned(), - &action, - reason.clone(), - Some(root.clone()), - ) - .await; - if matches!( - rx.await.unwrap_or_default(), - ReviewDecision::Approved | ReviewDecision::ApprovedForSession - ) { - // Extend writable roots. - #[allow(clippy::unwrap_used)] - sess.writable_roots.lock().unwrap().push(root); - stdout.clear(); - stderr.clear(); - result = apply_changes_from_apply_patch_and_report( - &action, - &mut stdout, - &mut stderr, - ); - } - } - } - } - - // Emit PatchApplyEnd event. - let success_flag = result.is_ok(); - let _ = sess - .tx_event - .send(Event { - id: sub_id.to_owned(), - msg: EventMsg::PatchApplyEnd(PatchApplyEndEvent { - call_id: call_id.to_owned(), - stdout: String::from_utf8_lossy(&stdout).to_string(), - stderr: String::from_utf8_lossy(&stderr).to_string(), - success: success_flag, - }), - }) - .await; - - let item = match result { - Ok(_) => ResponseInputItem::FunctionCallOutput { + SafetyCheck::Reject { reason } => ResponseInputItem::FunctionCallOutput { call_id: call_id.to_owned(), output: FunctionCallOutputPayload { - content: String::from_utf8_lossy(&stdout).to_string(), - success: None, - }, - }, - Err(e) => ResponseInputItem::FunctionCallOutput { - call_id: call_id.to_owned(), - output: FunctionCallOutputPayload { - content: format!("error: {e:#}, stderr: {}", String::from_utf8_lossy(&stderr)), + content: format!("patch rejected: {reason}"), success: Some(false), }, - }, - }; - InternalApplyPatchInvocation::Output(item) -} - -/// Return the first path in `hunks` that is NOT under any of the -/// `writable_roots` (after normalising). If all paths are acceptable, -/// returns None. -fn first_offending_path( - action: &ApplyPatchAction, - writable_roots: &[PathBuf], - cwd: &Path, -) -> Option { - let changes = action.changes(); - for (path, change) in changes { - let candidate = match change { - ApplyPatchFileChange::Add { .. } => path, - ApplyPatchFileChange::Delete => path, - ApplyPatchFileChange::Update { move_path, .. } => move_path.as_ref().unwrap_or(path), - }; - - let abs = if candidate.is_absolute() { - candidate.clone() - } else { - cwd.join(candidate) - }; - - let mut allowed = false; - for root in writable_roots { - let root_abs = if root.is_absolute() { - root.clone() - } else { - cwd.join(root) - }; - if abs.starts_with(&root_abs) { - allowed = true; - break; - } - } - - if !allowed { - return Some(candidate.clone()); } + .into(), } - None } pub(crate) fn convert_apply_patch_to_protocol( @@ -327,85 +129,6 @@ pub(crate) fn convert_apply_patch_to_protocol( result } -fn apply_changes_from_apply_patch_and_report( - action: &ApplyPatchAction, - stdout: &mut impl std::io::Write, - stderr: &mut impl std::io::Write, -) -> std::io::Result<()> { - match apply_changes_from_apply_patch(action) { - Ok(affected_paths) => { - print_summary(&affected_paths, stdout)?; - } - Err(err) => { - writeln!(stderr, "{err:?}")?; - } - } - - Ok(()) -} - -fn apply_changes_from_apply_patch(action: &ApplyPatchAction) -> anyhow::Result { - let mut added: Vec = Vec::new(); - let mut modified: Vec = Vec::new(); - let mut deleted: Vec = Vec::new(); - - let changes = action.changes(); - for (path, change) in changes { - match change { - ApplyPatchFileChange::Add { content } => { - if let Some(parent) = path.parent() { - if !parent.as_os_str().is_empty() { - std::fs::create_dir_all(parent).with_context(|| { - format!("Failed to create parent directories for {}", path.display()) - })?; - } - } - std::fs::write(path, content) - .with_context(|| format!("Failed to write file {}", path.display()))?; - added.push(path.clone()); - } - ApplyPatchFileChange::Delete => { - std::fs::remove_file(path) - .with_context(|| format!("Failed to delete file {}", path.display()))?; - deleted.push(path.clone()); - } - ApplyPatchFileChange::Update { - unified_diff: _unified_diff, - move_path, - new_content, - } => { - if let Some(move_path) = move_path { - if let Some(parent) = move_path.parent() { - if !parent.as_os_str().is_empty() { - std::fs::create_dir_all(parent).with_context(|| { - format!( - "Failed to create parent directories for {}", - move_path.display() - ) - })?; - } - } - - std::fs::rename(path, move_path) - .with_context(|| format!("Failed to rename file {}", path.display()))?; - std::fs::write(move_path, new_content)?; - modified.push(move_path.clone()); - deleted.push(path.clone()); - } else { - std::fs::write(path, new_content)?; - modified.push(path.clone()); - } - } - } - } - - Ok(AffectedPaths { - added, - modified, - deleted, - }) -} - pub(crate) fn get_writable_roots(cwd: &Path) -> Vec { let mut writable_roots = Vec::new(); if cfg!(target_os = "macos") { diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index cd92739c72..3dd1d513a2 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -4,7 +4,6 @@ use std::borrow::Cow; use std::collections::HashMap; use std::collections::HashSet; -use std::path::Path; use std::path::PathBuf; use std::sync::Arc; use std::sync::Mutex; @@ -31,6 +30,7 @@ use tracing::trace; use tracing::warn; use uuid::Uuid; +use crate::apply_patch::ApplyPatchExec; use crate::apply_patch::CODEX_APPLY_PATCH_ARG1; use crate::apply_patch::InternalApplyPatchInvocation; use crate::apply_patch::convert_apply_patch_to_protocol; @@ -74,8 +74,11 @@ use crate::protocol::EventMsg; use crate::protocol::ExecApprovalRequestEvent; use crate::protocol::ExecCommandBeginEvent; use crate::protocol::ExecCommandEndEvent; +use crate::protocol::FileChange; use crate::protocol::InputItem; use crate::protocol::Op; +use crate::protocol::PatchApplyBeginEvent; +use crate::protocol::PatchApplyEndEvent; use crate::protocol::ReviewDecision; use crate::protocol::SandboxPolicy; use crate::protocol::SessionConfiguredEvent; @@ -358,20 +361,32 @@ impl Session { } } - async fn notify_exec_command_begin( - &self, - sub_id: &str, - call_id: &str, - command_for_display: Vec, - command_cwd: &Path, - ) { + async fn notify_exec_command_begin(&self, exec_command_context: ExecCommandContext) { + let ExecCommandContext { + sub_id, + call_id, + command_for_display, + cwd, + apply_patch, + } = exec_command_context; + let msg = match apply_patch { + Some(ApplyPatchCommandContext { + user_explicitly_approved_this_action, + changes, + }) => EventMsg::PatchApplyBegin(PatchApplyBeginEvent { + call_id, + auto_approved: !user_explicitly_approved_this_action, + changes, + }), + None => EventMsg::ExecCommandBegin(ExecCommandBeginEvent { + call_id, + command: command_for_display.clone(), + cwd, + }), + }; let event = Event { id: sub_id.to_string(), - msg: EventMsg::ExecCommandBegin(ExecCommandBeginEvent { - call_id: call_id.to_string(), - command: command_for_display, - cwd: command_cwd.to_path_buf(), - }), + msg, }; let _ = self.tx_event.send(event).await; } @@ -383,18 +398,33 @@ impl Session { stdout: &str, stderr: &str, exit_code: i32, + is_apply_patch: bool, ) { + // Because stdout and stderr could each be up to 100 KiB, we send + // truncated versions. const MAX_STREAM_OUTPUT: usize = 5 * 1024; // 5KiB + let stdout = stdout.chars().take(MAX_STREAM_OUTPUT).collect(); + let stderr = stderr.chars().take(MAX_STREAM_OUTPUT).collect(); + + let msg = if is_apply_patch { + EventMsg::PatchApplyEnd(PatchApplyEndEvent { + call_id: call_id.to_string(), + stdout, + stderr, + success: exit_code == 0, + }) + } else { + EventMsg::ExecCommandEnd(ExecCommandEndEvent { + call_id: call_id.to_string(), + stdout, + stderr, + exit_code, + }) + }; + let event = Event { id: sub_id.to_string(), - // Because stdout and stderr could each be up to 100 KiB, we send - // truncated versions. - msg: EventMsg::ExecCommandEnd(ExecCommandEndEvent { - call_id: call_id.to_string(), - stdout: stdout.chars().take(MAX_STREAM_OUTPUT).collect(), - stderr: stderr.chars().take(MAX_STREAM_OUTPUT).collect(), - exit_code, - }), + msg, }; let _ = self.tx_event.send(event).await; } @@ -502,6 +532,21 @@ impl State { } } +#[derive(Clone, Debug)] +pub(crate) struct ExecCommandContext { + pub(crate) sub_id: String, + pub(crate) call_id: String, + pub(crate) command_for_display: Vec, + pub(crate) cwd: PathBuf, + pub(crate) apply_patch: Option, +} + +#[derive(Clone, Debug)] +pub(crate) struct ApplyPatchCommandContext { + pub(crate) user_explicitly_approved_this_action: bool, + pub(crate) changes: HashMap, +} + /// A series of Turns in response to user input. pub(crate) struct AgentTask { sess: Arc, @@ -1430,35 +1475,39 @@ async fn handle_container_exec_with_params( call_id: String, ) -> ResponseInputItem { // check if this was a patch, and apply it if so - let apply_patch_action_for_exec = - match maybe_parse_apply_patch_verified(¶ms.command, ¶ms.cwd) { - MaybeApplyPatchVerified::Body(changes) => { - match apply_patch::apply_patch(sess, &sub_id, &call_id, changes).await { - InternalApplyPatchInvocation::Output(item) => return item, - InternalApplyPatchInvocation::DelegateToExec(action) => Some(action), + let apply_patch_exec = match maybe_parse_apply_patch_verified(¶ms.command, ¶ms.cwd) { + MaybeApplyPatchVerified::Body(changes) => { + match apply_patch::apply_patch(sess, &sub_id, &call_id, changes).await { + InternalApplyPatchInvocation::Output(item) => return item, + InternalApplyPatchInvocation::DelegateToExec(apply_patch_exec) => { + Some(apply_patch_exec) } } - MaybeApplyPatchVerified::CorrectnessError(parse_error) => { - // It looks like an invocation of `apply_patch`, but we - // could not resolve it into a patch that would apply - // cleanly. Return to model for resample. - return ResponseInputItem::FunctionCallOutput { - call_id, - output: FunctionCallOutputPayload { - content: format!("error: {parse_error:#}"), - success: None, - }, - }; - } - MaybeApplyPatchVerified::ShellParseError(error) => { - trace!("Failed to parse shell command, {error:?}"); - None - } - MaybeApplyPatchVerified::NotApplyPatch => None, - }; + } + MaybeApplyPatchVerified::CorrectnessError(parse_error) => { + // It looks like an invocation of `apply_patch`, but we + // could not resolve it into a patch that would apply + // cleanly. Return to model for resample. + return ResponseInputItem::FunctionCallOutput { + call_id, + output: FunctionCallOutputPayload { + content: format!("error: {parse_error:#}"), + success: None, + }, + }; + } + MaybeApplyPatchVerified::ShellParseError(error) => { + trace!("Failed to parse shell command, {error:?}"); + None + } + MaybeApplyPatchVerified::NotApplyPatch => None, + }; - let (params, safety, command_for_display) = match apply_patch_action_for_exec { - Some(ApplyPatchAction { patch, cwd, .. }) => { + let (params, safety, command_for_display) = match &apply_patch_exec { + Some(ApplyPatchExec { + action: ApplyPatchAction { patch, cwd, .. }, + user_explicitly_approved_this_action, + }) => { let path_to_codex = std::env::current_exe() .ok() .map(|p| p.to_string_lossy().to_string()); @@ -1478,13 +1527,22 @@ async fn handle_container_exec_with_params( CODEX_APPLY_PATCH_ARG1.to_string(), patch.clone(), ], - cwd, + cwd: cwd.clone(), timeout_ms: params.timeout_ms, env: HashMap::new(), }; - let safety = - assess_safety_for_untrusted_command(sess.approval_policy, &sess.sandbox_policy); - (params, safety, vec!["apply_patch".to_string(), patch]) + let safety = if *user_explicitly_approved_this_action { + SafetyCheck::AutoApprove { + sandbox_type: SandboxType::None, + } + } else { + assess_safety_for_untrusted_command(sess.approval_policy, &sess.sandbox_policy) + }; + ( + params, + safety, + vec!["apply_patch".to_string(), patch.clone()], + ) } None => { let safety = { @@ -1545,7 +1603,22 @@ async fn handle_container_exec_with_params( } }; - sess.notify_exec_command_begin(&sub_id, &call_id, command_for_display.clone(), ¶ms.cwd) + let exec_command_context = ExecCommandContext { + sub_id: sub_id.clone(), + call_id: call_id.clone(), + command_for_display: command_for_display.clone(), + cwd: params.cwd.clone(), + apply_patch: apply_patch_exec.map( + |ApplyPatchExec { + action, + user_explicitly_approved_this_action, + }| ApplyPatchCommandContext { + user_explicitly_approved_this_action, + changes: convert_apply_patch_to_protocol(&action), + }, + ), + }; + sess.notify_exec_command_begin(exec_command_context.clone()) .await; let params = maybe_run_with_user_profile(params, sess); @@ -1567,8 +1640,15 @@ async fn handle_container_exec_with_params( duration, } = output; - sess.notify_exec_command_end(&sub_id, &call_id, &stdout, &stderr, exit_code) - .await; + sess.notify_exec_command_end( + &sub_id, + &call_id, + &stdout, + &stderr, + exit_code, + exec_command_context.apply_patch.is_some(), + ) + .await; let is_success = exit_code == 0; let content = format_exec_output( @@ -1586,16 +1666,7 @@ async fn handle_container_exec_with_params( } } Err(CodexErr::Sandbox(error)) => { - handle_sandbox_error( - error, - sandbox_type, - params, - command_for_display, - sess, - sub_id, - call_id, - ) - .await + handle_sandbox_error(params, exec_command_context, error, sandbox_type, sess).await } Err(e) => { // Handle non-sandbox errors @@ -1611,14 +1682,17 @@ async fn handle_container_exec_with_params( } async fn handle_sandbox_error( + params: ExecParams, + exec_command_context: ExecCommandContext, error: SandboxErr, sandbox_type: SandboxType, - params: ExecParams, - command_for_display: Vec, sess: &Session, - sub_id: String, - call_id: String, ) -> ResponseInputItem { + let call_id = exec_command_context.call_id.clone(); + let sub_id = exec_command_context.sub_id.clone(); + let cwd = exec_command_context.cwd.clone(); + let is_apply_patch = exec_command_context.apply_patch.is_some(); + // Early out if the user never wants to be asked for approval; just return to the model immediately if sess.approval_policy == AskForApproval::Never { return ResponseInputItem::FunctionCallOutput { @@ -1648,7 +1722,7 @@ async fn handle_sandbox_error( sub_id.clone(), call_id.clone(), params.command.clone(), - params.cwd.clone(), + cwd.clone(), Some("command failed; retry without sandbox?".to_string()), ) .await; @@ -1664,8 +1738,7 @@ async fn handle_sandbox_error( sess.notify_background_event(&sub_id, "retrying command without sandbox") .await; - sess.notify_exec_command_begin(&sub_id, &call_id, command_for_display, ¶ms.cwd) - .await; + sess.notify_exec_command_begin(exec_command_context).await; // This is an escalated retry; the policy will not be // examined and the sandbox has been set to `None`. @@ -1687,8 +1760,15 @@ async fn handle_sandbox_error( duration, } = retry_output; - sess.notify_exec_command_end(&sub_id, &call_id, &stdout, &stderr, exit_code) - .await; + sess.notify_exec_command_end( + &sub_id, + &call_id, + &stdout, + &stderr, + exit_code, + is_apply_patch, + ) + .await; let is_success = exit_code == 0; let content = format_exec_output( diff --git a/codex-rs/core/src/safety.rs b/codex-rs/core/src/safety.rs index f9bc27e058..224705f8f3 100644 --- a/codex-rs/core/src/safety.rs +++ b/codex-rs/core/src/safety.rs @@ -41,11 +41,13 @@ pub fn assess_patch_safety( } } - if is_write_patch_constrained_to_writable_paths(action, writable_roots, cwd) { - SafetyCheck::AutoApprove { - sandbox_type: SandboxType::None, - } - } else if policy == AskForApproval::OnFailure { + // Even though the patch *appears* to be constrained to writable paths, it + // is possible that paths in the patch are hard links to files outside the + // writable roots, so we should still run `apply_patch` in a sandbox in that + // case. + if is_write_patch_constrained_to_writable_paths(action, writable_roots, cwd) + || policy == AskForApproval::OnFailure + { // Only auto‑approve when we can actually enforce a sandbox. Otherwise // fall back to asking the user because the patch may touch arbitrary // paths outside the project. From 9aa11269a56828f6c2f43cd664dec278509e52c2 Mon Sep 17 00:00:00 2001 From: easong-openai Date: Thu, 31 Jul 2025 12:41:32 -0700 Subject: [PATCH 020/100] Fix double-scrolling in approval model (#1754) Previously, pressing up or down arrow in the new approval modal would be the equivalent of two up or down presses. --- codex-rs/tui/src/app.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index 44c1875d40..3013cb6df0 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -96,7 +96,9 @@ impl App<'_> { if let Ok(event) = crossterm::event::read() { match event { crossterm::event::Event::Key(key_event) => { - app_event_tx.send(AppEvent::KeyEvent(key_event)); + if key_event.kind == crossterm::event::KeyEventKind::Press { + app_event_tx.send(AppEvent::KeyEvent(key_event)); + } } crossterm::event::Event::Resize(_, _) => { app_event_tx.send(AppEvent::RequestRedraw); From 5a0ad5ab8f056e917063cab4bc5031a8b5e9f3ec Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 31 Jul 2025 13:11:47 -0700 Subject: [PATCH 021/100] chore: refactor exec.rs: create separate seatbelt.rs and spawn.rs files (#1762) At 550 lines, `exec.rs` was a bit large. In particular, I found it hard to locate the Seatbelt-related code quickly without a file with `seatbelt` in the name, so this refactors things so: - `spawn_command_under_seatbelt()` and dependent code moves to a new `seatbelt.rs` file - `spawn_child_async()` and dependent code moves to a new `spawn.rs` file --- codex-rs/cli/src/debug_sandbox.rs | 4 +- codex-rs/core/src/exec.rs | 188 +-------------------- codex-rs/core/src/lib.rs | 2 + codex-rs/core/src/seatbelt.rs | 96 +++++++++++ codex-rs/core/src/spawn.rs | 102 +++++++++++ codex-rs/core/tests/cli_stream.rs | 2 +- codex-rs/core/tests/client.rs | 2 +- codex-rs/core/tests/stream_no_completed.rs | 2 +- codex-rs/mcp-server/tests/codex_tool.rs | 2 +- codex-rs/mcp-server/tests/interrupt.rs | 2 +- 10 files changed, 210 insertions(+), 192 deletions(-) create mode 100644 codex-rs/core/src/seatbelt.rs create mode 100644 codex-rs/core/src/spawn.rs diff --git a/codex-rs/cli/src/debug_sandbox.rs b/codex-rs/cli/src/debug_sandbox.rs index 905b746168..7f0983cbc6 100644 --- a/codex-rs/cli/src/debug_sandbox.rs +++ b/codex-rs/cli/src/debug_sandbox.rs @@ -4,10 +4,10 @@ use codex_common::CliConfigOverrides; use codex_core::config::Config; use codex_core::config::ConfigOverrides; use codex_core::config_types::SandboxMode; -use codex_core::exec::StdioPolicy; use codex_core::exec::spawn_command_under_linux_sandbox; -use codex_core::exec::spawn_command_under_seatbelt; use codex_core::exec_env::create_env; +use codex_core::seatbelt::spawn_command_under_seatbelt; +use codex_core::spawn::StdioPolicy; use crate::LandlockCommand; use crate::SeatbeltCommand; diff --git a/codex-rs/core/src/exec.rs b/codex-rs/core/src/exec.rs index 230c4ec134..06416e6768 100644 --- a/codex-rs/core/src/exec.rs +++ b/codex-rs/core/src/exec.rs @@ -6,7 +6,6 @@ use std::io; use std::path::Path; use std::path::PathBuf; use std::process::ExitStatus; -use std::process::Stdio; use std::sync::Arc; use std::time::Duration; use std::time::Instant; @@ -15,14 +14,15 @@ use tokio::io::AsyncRead; use tokio::io::AsyncReadExt; use tokio::io::BufReader; use tokio::process::Child; -use tokio::process::Command; use tokio::sync::Notify; -use tracing::trace; use crate::error::CodexErr; use crate::error::Result; use crate::error::SandboxErr; use crate::protocol::SandboxPolicy; +use crate::seatbelt::spawn_command_under_seatbelt; +use crate::spawn::StdioPolicy; +use crate::spawn::spawn_child_async; // Maximum we send for each stream, which is either: // - 10KiB OR @@ -37,24 +37,6 @@ const DEFAULT_TIMEOUT_MS: u64 = 10_000; const SIGKILL_CODE: i32 = 9; const TIMEOUT_CODE: i32 = 64; -const MACOS_SEATBELT_BASE_POLICY: &str = include_str!("seatbelt_base_policy.sbpl"); - -/// When working with `sandbox-exec`, only consider `sandbox-exec` in `/usr/bin` -/// to defend against an attacker trying to inject a malicious version on the -/// PATH. If /usr/bin/sandbox-exec has been tampered with, then the attacker -/// already has root access. -const MACOS_PATH_TO_SEATBELT_EXECUTABLE: &str = "/usr/bin/sandbox-exec"; - -/// Experimental environment variable that will be set to some non-empty value -/// if both of the following are true: -/// -/// 1. The process was spawned by Codex as part of a shell tool call. -/// 2. SandboxPolicy.has_full_network_access() was false for the tool call. -/// -/// We may try to have just one environment variable for all sandboxing -/// attributes, so this may change in the future. -pub const CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR: &str = "CODEX_SANDBOX_NETWORK_DISABLED"; - #[derive(Debug, Clone)] pub struct ExecParams { pub command: Vec, @@ -168,27 +150,6 @@ pub async fn process_exec_tool_call( } } -pub async fn spawn_command_under_seatbelt( - command: Vec, - sandbox_policy: &SandboxPolicy, - cwd: PathBuf, - stdio_policy: StdioPolicy, - env: HashMap, -) -> std::io::Result { - let args = create_seatbelt_command_args(command, sandbox_policy, &cwd); - let arg0 = None; - spawn_child_async( - PathBuf::from(MACOS_PATH_TO_SEATBELT_EXECUTABLE), - args, - arg0, - cwd, - sandbox_policy, - stdio_policy, - env, - ) - .await -} - /// Spawn a shell tool command under the Linux Landlock+seccomp sandbox helper /// (codex-linux-sandbox). /// @@ -248,65 +209,6 @@ fn create_linux_sandbox_command_args( linux_cmd } -fn create_seatbelt_command_args( - command: Vec, - sandbox_policy: &SandboxPolicy, - cwd: &Path, -) -> Vec { - let (file_write_policy, extra_cli_args) = { - if sandbox_policy.has_full_disk_write_access() { - // Allegedly, this is more permissive than `(allow file-write*)`. - ( - r#"(allow file-write* (regex #"^/"))"#.to_string(), - Vec::::new(), - ) - } else { - let writable_roots = sandbox_policy.get_writable_roots_with_cwd(cwd); - let (writable_folder_policies, cli_args): (Vec, Vec) = writable_roots - .iter() - .enumerate() - .map(|(index, root)| { - let param_name = format!("WRITABLE_ROOT_{index}"); - let policy: String = format!("(subpath (param \"{param_name}\"))"); - let cli_arg = format!("-D{param_name}={}", root.to_string_lossy()); - (policy, cli_arg) - }) - .unzip(); - if writable_folder_policies.is_empty() { - ("".to_string(), Vec::::new()) - } else { - let file_write_policy = format!( - "(allow file-write*\n{}\n)", - writable_folder_policies.join(" ") - ); - (file_write_policy, cli_args) - } - } - }; - - let file_read_policy = if sandbox_policy.has_full_disk_read_access() { - "; allow read-only file operations\n(allow file-read*)" - } else { - "" - }; - - // TODO(mbolin): apply_patch calls must also honor the SandboxPolicy. - let network_policy = if sandbox_policy.has_full_network_access() { - "(allow network-outbound)\n(allow network-inbound)\n(allow system-socket)" - } else { - "" - }; - - let full_policy = format!( - "{MACOS_SEATBELT_BASE_POLICY}\n{file_read_policy}\n{file_write_policy}\n{network_policy}" - ); - let mut seatbelt_args: Vec = vec!["-p".to_string(), full_policy]; - seatbelt_args.extend(extra_cli_args); - seatbelt_args.push("--".to_string()); - seatbelt_args.extend(command); - seatbelt_args -} - #[derive(Debug)] pub struct RawExecToolCallOutput { pub exit_status: ExitStatus, @@ -352,90 +254,6 @@ async fn exec( consume_truncated_output(child, ctrl_c, timeout_ms).await } -#[derive(Debug, Clone, Copy)] -pub enum StdioPolicy { - RedirectForShellTool, - Inherit, -} - -/// Spawns the appropriate child process for the ExecParams and SandboxPolicy, -/// ensuring the args and environment variables used to create the `Command` -/// (and `Child`) honor the configuration. -/// -/// For now, we take `SandboxPolicy` as a parameter to spawn_child() because -/// we need to determine whether to set the -/// `CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR` environment variable. -async fn spawn_child_async( - program: PathBuf, - args: Vec, - #[cfg_attr(not(unix), allow(unused_variables))] arg0: Option<&str>, - cwd: PathBuf, - sandbox_policy: &SandboxPolicy, - stdio_policy: StdioPolicy, - env: HashMap, -) -> std::io::Result { - trace!( - "spawn_child_async: {program:?} {args:?} {arg0:?} {cwd:?} {sandbox_policy:?} {stdio_policy:?} {env:?}" - ); - - let mut cmd = Command::new(&program); - #[cfg(unix)] - cmd.arg0(arg0.map_or_else(|| program.to_string_lossy().to_string(), String::from)); - cmd.args(args); - cmd.current_dir(cwd); - cmd.env_clear(); - cmd.envs(env); - - if !sandbox_policy.has_full_network_access() { - cmd.env(CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR, "1"); - } - - // If this Codex process dies (including being killed via SIGKILL), we want - // any child processes that were spawned as part of a `"shell"` tool call - // to also be terminated. - - // This relies on prctl(2), so it only works on Linux. - #[cfg(target_os = "linux")] - unsafe { - cmd.pre_exec(|| { - // This prctl call effectively requests, "deliver SIGTERM when my - // current parent dies." - if libc::prctl(libc::PR_SET_PDEATHSIG, libc::SIGTERM) == -1 { - return Err(io::Error::last_os_error()); - } - - // Though if there was a race condition and this pre_exec() block is - // run _after_ the parent (i.e., the Codex process) has already - // exited, then the parent is the _init_ process (which will never - // die), so we should just terminate the child process now. - if libc::getppid() == 1 { - libc::raise(libc::SIGTERM); - } - Ok(()) - }); - } - - match stdio_policy { - StdioPolicy::RedirectForShellTool => { - // Do not create a file descriptor for stdin because otherwise some - // commands may hang forever waiting for input. For example, ripgrep has - // a heuristic where it may try to read from stdin as explained here: - // https://github.com/BurntSushi/ripgrep/blob/e2362d4d5185d02fa857bf381e7bd52e66fafc73/crates/core/flags/hiargs.rs#L1101-L1103 - cmd.stdin(Stdio::null()); - - cmd.stdout(Stdio::piped()).stderr(Stdio::piped()); - } - StdioPolicy::Inherit => { - // Inherit stdin, stdout, and stderr from the parent process. - cmd.stdin(Stdio::inherit()) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()); - } - } - - cmd.kill_on_drop(true).spawn() -} - /// Consumes the output of a child process, truncating it so it is suitable for /// use as the output of a `shell` tool call. Also enforces specified timeout. pub(crate) async fn consume_truncated_output( diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index 054abd742a..a33e185afb 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -39,7 +39,9 @@ mod project_doc; pub mod protocol; mod rollout; mod safety; +pub mod seatbelt; pub mod shell; +pub mod spawn; mod user_notification; pub mod util; diff --git a/codex-rs/core/src/seatbelt.rs b/codex-rs/core/src/seatbelt.rs new file mode 100644 index 0000000000..be2acb1bdc --- /dev/null +++ b/codex-rs/core/src/seatbelt.rs @@ -0,0 +1,96 @@ +use std::collections::HashMap; +use std::path::Path; +use std::path::PathBuf; +use tokio::process::Child; + +use crate::protocol::SandboxPolicy; +use crate::spawn::StdioPolicy; +use crate::spawn::spawn_child_async; + +const MACOS_SEATBELT_BASE_POLICY: &str = include_str!("seatbelt_base_policy.sbpl"); + +/// When working with `sandbox-exec`, only consider `sandbox-exec` in `/usr/bin` +/// to defend against an attacker trying to inject a malicious version on the +/// PATH. If /usr/bin/sandbox-exec has been tampered with, then the attacker +/// already has root access. +const MACOS_PATH_TO_SEATBELT_EXECUTABLE: &str = "/usr/bin/sandbox-exec"; + +pub async fn spawn_command_under_seatbelt( + command: Vec, + sandbox_policy: &SandboxPolicy, + cwd: PathBuf, + stdio_policy: StdioPolicy, + env: HashMap, +) -> std::io::Result { + let args = create_seatbelt_command_args(command, sandbox_policy, &cwd); + let arg0 = None; + spawn_child_async( + PathBuf::from(MACOS_PATH_TO_SEATBELT_EXECUTABLE), + args, + arg0, + cwd, + sandbox_policy, + stdio_policy, + env, + ) + .await +} + +fn create_seatbelt_command_args( + command: Vec, + sandbox_policy: &SandboxPolicy, + cwd: &Path, +) -> Vec { + let (file_write_policy, extra_cli_args) = { + if sandbox_policy.has_full_disk_write_access() { + // Allegedly, this is more permissive than `(allow file-write*)`. + ( + r#"(allow file-write* (regex #"^/"))"#.to_string(), + Vec::::new(), + ) + } else { + let writable_roots = sandbox_policy.get_writable_roots_with_cwd(cwd); + let (writable_folder_policies, cli_args): (Vec, Vec) = writable_roots + .iter() + .enumerate() + .map(|(index, root)| { + let param_name = format!("WRITABLE_ROOT_{index}"); + let policy: String = format!("(subpath (param \"{param_name}\"))"); + let cli_arg = format!("-D{param_name}={}", root.to_string_lossy()); + (policy, cli_arg) + }) + .unzip(); + if writable_folder_policies.is_empty() { + ("".to_string(), Vec::::new()) + } else { + let file_write_policy = format!( + "(allow file-write*\n{}\n)", + writable_folder_policies.join(" ") + ); + (file_write_policy, cli_args) + } + } + }; + + let file_read_policy = if sandbox_policy.has_full_disk_read_access() { + "; allow read-only file operations\n(allow file-read*)" + } else { + "" + }; + + // TODO(mbolin): apply_patch calls must also honor the SandboxPolicy. + let network_policy = if sandbox_policy.has_full_network_access() { + "(allow network-outbound)\n(allow network-inbound)\n(allow system-socket)" + } else { + "" + }; + + let full_policy = format!( + "{MACOS_SEATBELT_BASE_POLICY}\n{file_read_policy}\n{file_write_policy}\n{network_policy}" + ); + let mut seatbelt_args: Vec = vec!["-p".to_string(), full_policy]; + seatbelt_args.extend(extra_cli_args); + seatbelt_args.push("--".to_string()); + seatbelt_args.extend(command); + seatbelt_args +} diff --git a/codex-rs/core/src/spawn.rs b/codex-rs/core/src/spawn.rs new file mode 100644 index 0000000000..9fde26539b --- /dev/null +++ b/codex-rs/core/src/spawn.rs @@ -0,0 +1,102 @@ +use std::collections::HashMap; +use std::path::PathBuf; +use std::process::Stdio; +use tokio::process::Child; +use tokio::process::Command; +use tracing::trace; + +use crate::protocol::SandboxPolicy; + +/// Experimental environment variable that will be set to some non-empty value +/// if both of the following are true: +/// +/// 1. The process was spawned by Codex as part of a shell tool call. +/// 2. SandboxPolicy.has_full_network_access() was false for the tool call. +/// +/// We may try to have just one environment variable for all sandboxing +/// attributes, so this may change in the future. +pub const CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR: &str = "CODEX_SANDBOX_NETWORK_DISABLED"; + +#[derive(Debug, Clone, Copy)] +pub enum StdioPolicy { + RedirectForShellTool, + Inherit, +} + +/// Spawns the appropriate child process for the ExecParams and SandboxPolicy, +/// ensuring the args and environment variables used to create the `Command` +/// (and `Child`) honor the configuration. +/// +/// For now, we take `SandboxPolicy` as a parameter to spawn_child() because +/// we need to determine whether to set the +/// `CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR` environment variable. +pub(crate) async fn spawn_child_async( + program: PathBuf, + args: Vec, + #[cfg_attr(not(unix), allow(unused_variables))] arg0: Option<&str>, + cwd: PathBuf, + sandbox_policy: &SandboxPolicy, + stdio_policy: StdioPolicy, + env: HashMap, +) -> std::io::Result { + trace!( + "spawn_child_async: {program:?} {args:?} {arg0:?} {cwd:?} {sandbox_policy:?} {stdio_policy:?} {env:?}" + ); + + let mut cmd = Command::new(&program); + #[cfg(unix)] + cmd.arg0(arg0.map_or_else(|| program.to_string_lossy().to_string(), String::from)); + cmd.args(args); + cmd.current_dir(cwd); + cmd.env_clear(); + cmd.envs(env); + + if !sandbox_policy.has_full_network_access() { + cmd.env(CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR, "1"); + } + + // If this Codex process dies (including being killed via SIGKILL), we want + // any child processes that were spawned as part of a `"shell"` tool call + // to also be terminated. + + // This relies on prctl(2), so it only works on Linux. + #[cfg(target_os = "linux")] + unsafe { + cmd.pre_exec(|| { + // This prctl call effectively requests, "deliver SIGTERM when my + // current parent dies." + if libc::prctl(libc::PR_SET_PDEATHSIG, libc::SIGTERM) == -1 { + return Err(std::io::Error::last_os_error()); + } + + // Though if there was a race condition and this pre_exec() block is + // run _after_ the parent (i.e., the Codex process) has already + // exited, then the parent is the _init_ process (which will never + // die), so we should just terminate the child process now. + if libc::getppid() == 1 { + libc::raise(libc::SIGTERM); + } + Ok(()) + }); + } + + match stdio_policy { + StdioPolicy::RedirectForShellTool => { + // Do not create a file descriptor for stdin because otherwise some + // commands may hang forever waiting for input. For example, ripgrep has + // a heuristic where it may try to read from stdin as explained here: + // https://github.com/BurntSushi/ripgrep/blob/e2362d4d5185d02fa857bf381e7bd52e66fafc73/crates/core/flags/hiargs.rs#L1101-L1103 + cmd.stdin(Stdio::null()); + + cmd.stdout(Stdio::piped()).stderr(Stdio::piped()); + } + StdioPolicy::Inherit => { + // Inherit stdin, stdout, and stderr from the parent process. + cmd.stdin(Stdio::inherit()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()); + } + } + + cmd.kill_on_drop(true).spawn() +} diff --git a/codex-rs/core/tests/cli_stream.rs b/codex-rs/core/tests/cli_stream.rs index ee0377fc10..c8570577ce 100644 --- a/codex-rs/core/tests/cli_stream.rs +++ b/codex-rs/core/tests/cli_stream.rs @@ -1,7 +1,7 @@ #![expect(clippy::unwrap_used)] use assert_cmd::Command as AssertCommand; -use codex_core::exec::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR; +use codex_core::spawn::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR; use std::time::Duration; use std::time::Instant; use tempfile::TempDir; diff --git a/codex-rs/core/tests/client.rs b/codex-rs/core/tests/client.rs index 1286928ec0..ab8ad6833a 100644 --- a/codex-rs/core/tests/client.rs +++ b/codex-rs/core/tests/client.rs @@ -5,11 +5,11 @@ use codex_core::Codex; use codex_core::CodexSpawnOk; use codex_core::ModelProviderInfo; use codex_core::built_in_model_providers; -use codex_core::exec::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR; use codex_core::protocol::EventMsg; use codex_core::protocol::InputItem; use codex_core::protocol::Op; use codex_core::protocol::SessionConfiguredEvent; +use codex_core::spawn::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR; use codex_login::AuthDotJson; use codex_login::AuthMode; use codex_login::CodexAuth; diff --git a/codex-rs/core/tests/stream_no_completed.rs b/codex-rs/core/tests/stream_no_completed.rs index d2fc035569..3e30d93709 100644 --- a/codex-rs/core/tests/stream_no_completed.rs +++ b/codex-rs/core/tests/stream_no_completed.rs @@ -6,10 +6,10 @@ use std::time::Duration; use codex_core::Codex; use codex_core::CodexSpawnOk; use codex_core::ModelProviderInfo; -use codex_core::exec::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR; use codex_core::protocol::EventMsg; use codex_core::protocol::InputItem; use codex_core::protocol::Op; +use codex_core::spawn::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR; use codex_login::CodexAuth; use core_test_support::load_default_config_for_test; use core_test_support::load_sse_fixture; diff --git a/codex-rs/mcp-server/tests/codex_tool.rs b/codex-rs/mcp-server/tests/codex_tool.rs index 0f06483f24..7d59ff202e 100644 --- a/codex-rs/mcp-server/tests/codex_tool.rs +++ b/codex-rs/mcp-server/tests/codex_tool.rs @@ -3,9 +3,9 @@ use std::env; use std::path::Path; use std::path::PathBuf; -use codex_core::exec::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR; use codex_core::protocol::FileChange; use codex_core::protocol::ReviewDecision; +use codex_core::spawn::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR; use codex_mcp_server::CodexToolCallParam; use codex_mcp_server::ExecApprovalElicitRequestParams; use codex_mcp_server::ExecApprovalResponse; diff --git a/codex-rs/mcp-server/tests/interrupt.rs b/codex-rs/mcp-server/tests/interrupt.rs index 313bc7afab..b2767f5e58 100644 --- a/codex-rs/mcp-server/tests/interrupt.rs +++ b/codex-rs/mcp-server/tests/interrupt.rs @@ -3,7 +3,7 @@ use std::path::Path; -use codex_core::exec::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR; +use codex_core::spawn::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR; use codex_mcp_server::CodexToolCallParam; use mcp_types::JSONRPCResponse; use mcp_types::RequestId; From 6ce0a5875bbde55a00df054e7f0bceba681cf44d Mon Sep 17 00:00:00 2001 From: easong-openai Date: Thu, 31 Jul 2025 13:45:52 -0700 Subject: [PATCH 022/100] Initial planning tool (#1753) We need to optimize the prompt, but this causes the model to use the new planning_tool. image --- codex-rs/core/prompt.md | 9 +++ codex-rs/tui/src/chatwidget.rs | 4 ++ codex-rs/tui/src/history_cell.rs | 96 ++++++++++++++++++++++++++++++++ codex-rs/tui/src/lib.rs | 2 +- 4 files changed, 110 insertions(+), 1 deletion(-) diff --git a/codex-rs/core/prompt.md b/codex-rs/core/prompt.md index 66cd55b628..0a4578270a 100644 --- a/codex-rs/core/prompt.md +++ b/codex-rs/core/prompt.md @@ -96,3 +96,12 @@ You can invoke apply_patch like: ``` shell {"command":["apply_patch","*** Begin Patch\n*** Add File: hello.txt\n+Hello, world!\n*** End Patch\n"]} ``` + +Plan updates + +A tool named `update_plan` is available. Use it to keep an up‑to‑date, step‑by‑step plan for the task so you can follow your progress. When making your plans, keep in mind that you are a deployed coding agent - `update_plan` calls should not involve doing anything that you aren't capable of doing. For example, `update_plan` calls should NEVER contain tasks to merge your own pull requests. Only stop to ask the user if you genuinely need their feedback on a change. + +- At the start of the task, call `update_plan` with an initial plan: a short list of 1‑sentence steps with a `status` for each step (`pending`, `in_progress`, or `completed`). There should always be exactly one `in_progress` step until everything is done. +- Whenever you finish a step, call `update_plan` again, marking the finished step as `completed` and the next step as `in_progress`. +- If your plan needs to change, call `update_plan` with the revised steps and include an `explanation` describing the change. +- When all steps are complete, make a final `update_plan` call with all steps marked `completed`. diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 3ee724e687..50488f8b30 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -293,6 +293,10 @@ impl ChatWidget<'_> { self.add_to_history(HistoryCell::new_error_event(message.clone())); self.bottom_pane.set_task_running(false); } + EventMsg::PlanUpdate(update) => { + self.add_to_history(HistoryCell::new_plan_update(update)); + self.request_redraw(); + } EventMsg::ExecApprovalRequest(ExecApprovalRequestEvent { call_id: _, command, diff --git a/codex-rs/tui/src/history_cell.rs b/codex-rs/tui/src/history_cell.rs index 956a0cc7ef..0e9f25332d 100644 --- a/codex-rs/tui/src/history_cell.rs +++ b/codex-rs/tui/src/history_cell.rs @@ -9,6 +9,9 @@ use codex_common::summarize_sandbox_policy; use codex_core::WireApi; use codex_core::config::Config; use codex_core::model_supports_reasoning_summaries; +use codex_core::plan_tool::PlanItemArg; +use codex_core::plan_tool::StepStatus; +use codex_core::plan_tool::UpdatePlanArgs; use codex_core::protocol::FileChange; use codex_core::protocol::McpInvocation; use codex_core::protocol::SessionConfiguredEvent; @@ -109,6 +112,10 @@ pub(crate) enum HistoryCell { /// behaviour of `ActiveExecCommand` so the user sees *what* patch the /// model wants to apply before being prompted to approve or deny it. PendingPatch { view: TextBlock }, + + /// A human‑friendly rendering of the model's current plan and step + /// statuses provided via the `update_plan` tool. + PlanUpdate { view: TextBlock }, } const TOOL_CALL_MAX_LINES: usize = 5; @@ -130,6 +137,7 @@ impl HistoryCell { | HistoryCell::CompletedExecCommand { view } | HistoryCell::CompletedMcpToolCall { view } | HistoryCell::PendingPatch { view } + | HistoryCell::PlanUpdate { view } | HistoryCell::ActiveExecCommand { view, .. } | HistoryCell::ActiveMcpToolCall { view, .. } => { view.lines.iter().map(line_to_static).collect() @@ -477,6 +485,94 @@ impl HistoryCell { } } + /// Render a user‑friendly plan update with colourful status icons and a + /// simple progress indicator so users can follow along. + pub(crate) fn new_plan_update(update: UpdatePlanArgs) -> Self { + let UpdatePlanArgs { explanation, plan } = update; + + let mut lines: Vec> = Vec::new(); + + // Title + lines.push(Line::from("plan".magenta().bold())); + + if !plan.is_empty() { + // Progress bar – show completed/total with a visual bar + let total = plan.len(); + let completed = plan + .iter() + .filter(|p| matches!(p.status, StepStatus::Completed)) + .count(); + let width: usize = 20; + let filled = (completed * width + total / 2) / total; + let empty = width.saturating_sub(filled); + let mut bar_spans: Vec = Vec::new(); + if filled > 0 { + bar_spans.push(Span::styled( + "█".repeat(filled), + Style::default().fg(Color::Green), + )); + } + if empty > 0 { + bar_spans.push(Span::styled( + "░".repeat(empty), + Style::default().fg(Color::Gray), + )); + } + let progress_prefix = Span::raw("progress ["); + let progress_suffix = Span::raw("] "); + let fraction = Span::raw(format!("{completed}/{total}")); + let mut progress_line_spans = vec![progress_prefix]; + progress_line_spans.extend(bar_spans); + progress_line_spans.push(progress_suffix); + progress_line_spans.push(fraction); + lines.push(Line::from(progress_line_spans)); + } + + // Optional explanation/note from the model + if let Some(expl) = explanation.and_then(|s| { + let t = s.trim().to_string(); + if t.is_empty() { None } else { Some(t) } + }) { + lines.push(Line::from("note".gray().italic())); + for l in expl.lines() { + lines.push(Line::from(l.to_string()).gray()); + } + } + + // Steps (1‑based numbering) with fun, readable status icons + if plan.is_empty() { + lines.push(Line::from("(no steps provided)".gray().italic())); + } else { + for (idx, PlanItemArg { step, status }) in plan.into_iter().enumerate() { + let num = idx + 1; + let (icon, style): (&str, Style) = match status { + StepStatus::Completed => ("✓", Style::default().fg(Color::Green)), + StepStatus::InProgress => ( + "▶", + Style::default() + .fg(Color::Yellow) + .add_modifier(Modifier::BOLD), + ), + StepStatus::Pending => ("○", Style::default().fg(Color::Gray)), + }; + let prefix = vec![ + Span::raw(format!("{num:>2}. [")), + Span::styled(icon.to_string(), style), + Span::raw("] "), + ]; + let mut spans = prefix; + spans.push(Span::raw(step)); + lines.push(Line::from(spans)); + } + } + + lines.push(Line::from("")); + + HistoryCell::PlanUpdate { + view: TextBlock::new(lines), + } + } + /// Create a new `PendingPatch` cell that lists the file‑level summary of /// a proposed patch. The summary lines should already be formatted (e.g. /// "A path/to/file.rs"). diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index 7e987f6ff4..f0a0e9d833 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -75,7 +75,7 @@ pub async fn run_main( config_profile: cli.config_profile.clone(), codex_linux_sandbox_exe, base_instructions: None, - include_plan_tool: None, + include_plan_tool: Some(true), }; // Parse `-c` overrides from the CLI. let cli_kv_overrides = match cli.config_overrides.parse_overrides() { From 0935e6a875e22c33073f738e4b64292bc643bc90 Mon Sep 17 00:00:00 2001 From: pakrym-oai Date: Thu, 31 Jul 2025 15:40:19 -0700 Subject: [PATCH 023/100] Send account id when available (#1767) For users with multiple accounts we need to specify the account to use. --- codex-rs/core/src/client.rs | 8 +++++++- codex-rs/core/tests/client.rs | 4 +++- codex-rs/login/src/lib.rs | 11 +++++++++++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index 4e0e62c0f7..0f7f51c2fc 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -192,7 +192,7 @@ impl ModelClient { loop { attempt += 1; - let req_builder = self + let mut req_builder = self .client .post(format!("{base_url}/responses")) .header("OpenAI-Beta", "responses=experimental") @@ -201,6 +201,12 @@ impl ModelClient { .header(reqwest::header::ACCEPT, "text/event-stream") .json(&payload); + if auth.mode == AuthMode::ChatGPT { + if let Some(account_id) = auth.get_account_id().await { + req_builder = req_builder.header("chatgpt-account-id", account_id); + } + } + let req_builder = self.provider.apply_http_headers(req_builder); let res = req_builder.send().await; diff --git a/codex-rs/core/tests/client.rs b/codex-rs/core/tests/client.rs index ab8ad6833a..bfa29657d5 100644 --- a/codex-rs/core/tests/client.rs +++ b/codex-rs/core/tests/client.rs @@ -237,6 +237,7 @@ async fn chatgpt_auth_sends_correct_request() { let request_session_id = request.headers.get("session_id").unwrap(); let request_originator = request.headers.get("originator").unwrap(); let request_authorization = request.headers.get("authorization").unwrap(); + let request_chatgpt_account_id = request.headers.get("chatgpt-account-id").unwrap(); let request_body = request.body_json::().unwrap(); assert!(current_session_id.is_some()); @@ -249,6 +250,7 @@ async fn chatgpt_auth_sends_correct_request() { request_authorization.to_str().unwrap(), "Bearer Access Token" ); + assert_eq!(request_chatgpt_account_id.to_str().unwrap(), "account_id"); assert!(!request_body["store"].as_bool().unwrap()); assert!(request_body["stream"].as_bool().unwrap()); assert_eq!( @@ -332,7 +334,7 @@ fn auth_from_token(id_token: String) -> CodexAuth { id_token, access_token: "Access Token".to_string(), refresh_token: "test".to_string(), - account_id: None, + account_id: Some("account_id".to_string()), }), last_refresh: Some(Utc::now()), }), diff --git a/codex-rs/login/src/lib.rs b/codex-rs/login/src/lib.rs index 2f0aeb60bd..3d55c20264 100644 --- a/codex-rs/login/src/lib.rs +++ b/codex-rs/login/src/lib.rs @@ -123,6 +123,17 @@ impl CodexAuth { } } } + + pub async fn get_account_id(&self) -> Option { + match self.mode { + AuthMode::ApiKey => None, + AuthMode::ChatGPT => { + let token_data = self.get_token_data().await.ok()?; + + token_data.account_id.clone() + } + } + } } // Loads the available auth information from the auth.json or OPENAI_API_KEY environment variable. From 610addbc2e28159ebaf9b33e77778ba04de588ab Mon Sep 17 00:00:00 2001 From: Jeremy Rose <172423086+nornagon-openai@users.noreply.github.com> Date: Thu, 31 Jul 2025 17:00:48 -0700 Subject: [PATCH 024/100] do not dispatch key releases (#1771) when we enabled KKP in https://github.com/openai/codex/pull/1743, we started receiving keyup events, but didn't expect them anywhere in our code. for now, just don't dispatch them at all. --- codex-rs/tui/src/app.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index 3013cb6df0..86c042f813 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -15,6 +15,7 @@ use color_eyre::eyre::Result; use crossterm::SynchronizedUpdate; use crossterm::event::KeyCode; use crossterm::event::KeyEvent; +use crossterm::event::KeyEventKind; use ratatui::layout::Offset; use ratatui::prelude::Backend; use std::path::PathBuf; @@ -213,6 +214,7 @@ impl App<'_> { KeyEvent { code: KeyCode::Char('c'), modifiers: crossterm::event::KeyModifiers::CONTROL, + kind: KeyEventKind::Press, .. } => { match &mut self.app_state { @@ -227,6 +229,7 @@ impl App<'_> { KeyEvent { code: KeyCode::Char('d'), modifiers: crossterm::event::KeyModifiers::CONTROL, + kind: KeyEventKind::Press, .. } => { match &mut self.app_state { @@ -245,9 +248,15 @@ impl App<'_> { } } } - _ => { + KeyEvent { + kind: KeyEventKind::Press | KeyEventKind::Repeat, + .. + } => { self.dispatch_key_event(key_event); } + _ => { + // Ignore Release key events for now. + } }; } AppEvent::Paste(text) => { From ea69a1d72f11d0b1d24f9722dd3c31ff0fe69763 Mon Sep 17 00:00:00 2001 From: Jeremy Rose <172423086+nornagon-openai@users.noreply.github.com> Date: Thu, 31 Jul 2025 17:10:52 -0700 Subject: [PATCH 025/100] lighter approval modal (#1768) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The yellow hazard stripes were too scary :) This also has the added benefit of not rendering anything at the full width of the terminal, so resizing is a little easier to handle. Screenshot 2025-07-31 at 4 03 29 PM Screenshot 2025-07-31 at 4 05 03 PM --- codex-rs/tui/src/app.rs | 40 ++- codex-rs/tui/src/chatwidget.rs | 5 +- codex-rs/tui/src/user_approval_widget.rs | 306 +++++++++-------------- 3 files changed, 157 insertions(+), 194 deletions(-) diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index 86c042f813..7454a1ca11 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -10,7 +10,6 @@ use crate::tui; use codex_core::config::Config; use codex_core::protocol::Event; use codex_core::protocol::EventMsg; -use codex_core::protocol::ExecApprovalRequestEvent; use color_eyre::eyre::Result; use crossterm::SynchronizedUpdate; use crossterm::event::KeyCode; @@ -313,14 +312,41 @@ impl App<'_> { } #[cfg(debug_assertions)] SlashCommand::TestApproval => { + use std::collections::HashMap; + + use codex_core::protocol::ApplyPatchApprovalRequestEvent; + use codex_core::protocol::FileChange; + self.app_event_tx.send(AppEvent::CodexEvent(Event { id: "1".to_string(), - msg: EventMsg::ExecApprovalRequest(ExecApprovalRequestEvent { - call_id: "1".to_string(), - command: vec!["git".into(), "apply".into()], - cwd: self.config.cwd.clone(), - reason: Some("test".to_string()), - }), + // msg: EventMsg::ExecApprovalRequest(ExecApprovalRequestEvent { + // call_id: "1".to_string(), + // command: vec!["git".into(), "apply".into()], + // cwd: self.config.cwd.clone(), + // reason: Some("test".to_string()), + // }), + msg: EventMsg::ApplyPatchApprovalRequest( + ApplyPatchApprovalRequestEvent { + call_id: "1".to_string(), + changes: HashMap::from([ + ( + PathBuf::from("/tmp/test.txt"), + FileChange::Add { + content: "test".to_string(), + }, + ), + ( + PathBuf::from("/tmp/test2.txt"), + FileChange::Update { + unified_diff: "+test\n-test2".to_string(), + move_path: None, + }, + ), + ]), + reason: None, + grant_root: Some(PathBuf::from("/tmp")), + }, + ), })); } }, diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 50488f8b30..b0a6fb44e1 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -25,6 +25,7 @@ use codex_core::protocol::PatchApplyBeginEvent; use codex_core::protocol::TaskCompleteEvent; use codex_core::protocol::TokenUsage; use crossterm::event::KeyEvent; +use crossterm::event::KeyEventKind; use ratatui::buffer::Buffer; use ratatui::layout::Rect; use ratatui::widgets::Widget; @@ -157,7 +158,9 @@ impl ChatWidget<'_> { } pub(crate) fn handle_key_event(&mut self, key_event: KeyEvent) { - self.bottom_pane.clear_ctrl_c_quit_hint(); + if key_event.kind == KeyEventKind::Press { + self.bottom_pane.clear_ctrl_c_quit_hint(); + } match self.bottom_pane.handle_key_event(key_event) { InputResult::Submitted(text) => { diff --git a/codex-rs/tui/src/user_approval_widget.rs b/codex-rs/tui/src/user_approval_widget.rs index 855a7ea3db..91febde208 100644 --- a/codex-rs/tui/src/user_approval_widget.rs +++ b/codex-rs/tui/src/user_approval_widget.rs @@ -7,23 +7,24 @@ //! driven workflow – a fully‑fledged visual match is not required. use std::path::PathBuf; +use std::sync::LazyLock; use codex_core::protocol::Op; use codex_core::protocol::ReviewDecision; use crossterm::event::KeyCode; use crossterm::event::KeyEvent; +use crossterm::event::KeyEventKind; use ratatui::buffer::Buffer; use ratatui::layout::Rect; use ratatui::prelude::*; use ratatui::text::Line; -use ratatui::text::Span; -use ratatui::widgets::List; +use ratatui::widgets::Block; +use ratatui::widgets::BorderType; +use ratatui::widgets::Borders; use ratatui::widgets::Paragraph; use ratatui::widgets::Widget; use ratatui::widgets::WidgetRef; use ratatui::widgets::Wrap; -use tui_input::Input; -use tui_input::backend::crossterm::EventHandler; use crate::app_event::AppEvent; use crate::app_event_sender::AppEventSender; @@ -47,68 +48,62 @@ pub(crate) enum ApprovalRequest { /// Options displayed in the *select* mode. struct SelectOption { - label: &'static str, - decision: Option, - /// `true` when this option switches the widget to *input* mode. - enters_input_mode: bool, + label: Line<'static>, + description: &'static str, + key: KeyCode, + decision: ReviewDecision, } -// keep in same order as in the TS implementation -const SELECT_OPTIONS: &[SelectOption] = &[ - SelectOption { - label: "Yes (y)", - decision: Some(ReviewDecision::Approved), +static COMMAND_SELECT_OPTIONS: LazyLock> = LazyLock::new(|| { + vec![ + SelectOption { + label: Line::from(vec!["Y".underlined(), "es".into()]), + description: "Approve and run the command", + key: KeyCode::Char('y'), + decision: ReviewDecision::Approved, + }, + SelectOption { + label: Line::from(vec!["A".underlined(), "lways".into()]), + description: "Approve the command for the remainder of this session", + key: KeyCode::Char('a'), + decision: ReviewDecision::ApprovedForSession, + }, + SelectOption { + label: Line::from(vec!["N".underlined(), "o".into()]), + description: "Do not run the command", + key: KeyCode::Char('n'), + decision: ReviewDecision::Denied, + }, + ] +}); - enters_input_mode: false, - }, - SelectOption { - label: "Yes, always approve this exact command for this session (a)", - decision: Some(ReviewDecision::ApprovedForSession), - - enters_input_mode: false, - }, - SelectOption { - label: "Edit or give feedback (e)", - decision: None, - - enters_input_mode: true, - }, - SelectOption { - label: "No, and keep going (n)", - decision: Some(ReviewDecision::Denied), - - enters_input_mode: false, - }, - SelectOption { - label: "No, and stop for now (esc)", - decision: Some(ReviewDecision::Abort), - - enters_input_mode: false, - }, -]; - -/// Internal mode the widget is in – mirrors the TypeScript component. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum Mode { - Select, - Input, -} +static PATCH_SELECT_OPTIONS: LazyLock> = LazyLock::new(|| { + vec![ + SelectOption { + label: Line::from(vec!["Y".underlined(), "es".into()]), + description: "Approve and apply the changes", + key: KeyCode::Char('y'), + decision: ReviewDecision::Approved, + }, + SelectOption { + label: Line::from(vec!["N".underlined(), "o".into()]), + description: "Do not apply the changes", + key: KeyCode::Char('n'), + decision: ReviewDecision::Denied, + }, + ] +}); /// A modal prompting the user to approve or deny the pending request. pub(crate) struct UserApprovalWidget<'a> { approval_request: ApprovalRequest, app_event_tx: AppEventSender, confirmation_prompt: Paragraph<'a>, + select_options: &'a Vec, /// Currently selected index in *select* mode. selected_option: usize, - /// State for the optional input widget. - input: Input, - - /// Current mode. - mode: Mode, - /// Set to `true` once a decision has been sent – the parent view can then /// remove this widget from its queue. done: bool, @@ -116,7 +111,6 @@ pub(crate) struct UserApprovalWidget<'a> { impl UserApprovalWidget<'_> { pub(crate) fn new(approval_request: ApprovalRequest, app_event_tx: AppEventSender) -> Self { - let input = Input::default(); let confirmation_prompt = match &approval_request { ApprovalRequest::Exec { command, @@ -132,25 +126,20 @@ impl UserApprovalWidget<'_> { None => cwd.display().to_string(), }; let mut contents: Vec = vec![ - Line::from(vec![ - Span::from(cwd_str).dim(), - Span::from("$"), - Span::from(format!(" {cmd}")), - ]), + Line::from(vec!["codex".bold().magenta(), " wants to run:".into()]), + Line::from(vec![cwd_str.dim(), "$".into(), format!(" {cmd}").into()]), Line::from(""), ]; if let Some(reason) = reason { contents.push(Line::from(reason.clone().italic())); contents.push(Line::from("")); } - contents.extend(vec![Line::from("Allow command?"), Line::from("")]); Paragraph::new(contents).wrap(Wrap { trim: false }) } ApprovalRequest::ApplyPatch { reason, grant_root, .. } => { - let mut contents: Vec = - vec![Line::from("Apply patch".bold()), Line::from("")]; + let mut contents: Vec = vec![]; if let Some(r) = reason { contents.push(Line::from(r.clone().italic())); @@ -165,20 +154,19 @@ impl UserApprovalWidget<'_> { contents.push(Line::from("")); } - contents.push(Line::from("Allow changes?")); - contents.push(Line::from("")); - - Paragraph::new(contents) + Paragraph::new(contents).wrap(Wrap { trim: false }) } }; Self { + select_options: match &approval_request { + ApprovalRequest::Exec { .. } => &COMMAND_SELECT_OPTIONS, + ApprovalRequest::ApplyPatch { .. } => &PATCH_SELECT_OPTIONS, + }, approval_request, app_event_tx, confirmation_prompt, selected_option: 0, - input, - mode: Mode::Select, done: false, } } @@ -194,9 +182,8 @@ impl UserApprovalWidget<'_> { /// captures input while visible, we don’t need to report whether the event /// was consumed—callers can assume it always is. pub(crate) fn handle_key_event(&mut self, key: KeyEvent) { - match self.mode { - Mode::Select => self.handle_select_key(key), - Mode::Input => self.handle_input_key(key), + if key.kind == KeyEventKind::Press { + self.handle_select_key(key); } } @@ -208,58 +195,24 @@ impl UserApprovalWidget<'_> { fn handle_select_key(&mut self, key_event: KeyEvent) { match key_event.code { - KeyCode::Up => { - if self.selected_option == 0 { - self.selected_option = SELECT_OPTIONS.len() - 1; - } else { - self.selected_option -= 1; - } + KeyCode::Left => { + self.selected_option = (self.selected_option + self.select_options.len() - 1) + % self.select_options.len(); } - KeyCode::Down => { - self.selected_option = (self.selected_option + 1) % SELECT_OPTIONS.len(); - } - KeyCode::Char('y') => { - self.send_decision(ReviewDecision::Approved); - } - KeyCode::Char('a') => { - self.send_decision(ReviewDecision::ApprovedForSession); - } - KeyCode::Char('n') => { - self.send_decision(ReviewDecision::Denied); - } - KeyCode::Char('e') => { - self.mode = Mode::Input; + KeyCode::Right => { + self.selected_option = (self.selected_option + 1) % self.select_options.len(); } KeyCode::Enter => { - let opt = &SELECT_OPTIONS[self.selected_option]; - if opt.enters_input_mode { - self.mode = Mode::Input; - } else if let Some(decision) = opt.decision { - self.send_decision(decision); - } + let opt = &self.select_options[self.selected_option]; + self.send_decision(opt.decision); } KeyCode::Esc => { self.send_decision(ReviewDecision::Abort); } - _ => {} - } - } - - fn handle_input_key(&mut self, key_event: KeyEvent) { - // Handle special keys first. - match key_event.code { - KeyCode::Enter => { - let feedback = self.input.value().to_string(); - self.send_decision_with_feedback(ReviewDecision::Denied, feedback); - } - KeyCode::Esc => { - // Cancel input – treat as deny without feedback. - self.send_decision(ReviewDecision::Denied); - } - _ => { - // Feed into input widget for normal editing. - let ct_event = crossterm::event::Event::Key(key_event); - self.input.handle_event(&ct_event); + other => { + if let Some(opt) = self.select_options.iter().find(|opt| opt.key == other) { + self.send_decision(opt.decision); + } } } } @@ -312,87 +265,68 @@ impl UserApprovalWidget<'_> { } pub(crate) fn desired_height(&self, width: u16) -> u16 { - self.get_confirmation_prompt_height(width - 2) + SELECT_OPTIONS.len() as u16 + 2 + self.get_confirmation_prompt_height(width) + self.select_options.len() as u16 } } -const PLAIN: Style = Style::new(); -const BLUE_FG: Style = Style::new().fg(Color::LightCyan); - impl WidgetRef for &UserApprovalWidget<'_> { fn render_ref(&self, area: Rect, buf: &mut Buffer) { - // Take the area, wrap it in a block with a border, and divide up the - // remaining area into two chunks: one for the confirmation prompt and - // one for the response. - let inner = area.inner(Margin::new(0, 2)); - - // Determine how many rows we can allocate for the static confirmation - // prompt while *always* keeping enough space for the interactive - // response area (select list or input field). When the full prompt - // would exceed the available height we truncate it so the response - // options never get pushed out of view. This keeps the approval modal - // usable even when the overall bottom viewport is small. - - // Full height of the prompt (may be larger than the available area). - let full_prompt_height = self.get_confirmation_prompt_height(inner.width); - - // Minimum rows that must remain for the interactive section. - let min_response_rows = match self.mode { - Mode::Select => SELECT_OPTIONS.len() as u16, - // In input mode we need exactly two rows: one for the guidance - // prompt and one for the single-line input field. - Mode::Input => 2, - }; - - // Clamp prompt height so confirmation + response never exceed the - // available space. `saturating_sub` avoids underflow when the area is - // too small even for the minimal layout – in this unlikely case we - // fall back to zero-height prompt so at least the options are - // visible. - let prompt_height = full_prompt_height.min(inner.height.saturating_sub(min_response_rows)); - - let chunks = Layout::default() + let prompt_height = self.get_confirmation_prompt_height(area.width); + let [prompt_chunk, response_chunk] = Layout::default() .direction(Direction::Vertical) .constraints([Constraint::Length(prompt_height), Constraint::Min(0)]) - .split(inner); - let prompt_chunk = chunks[0]; - let response_chunk = chunks[1]; + .areas(area); - // Build the inner lines based on the mode. Collect them into a List of - // non-wrapping lines rather than a Paragraph for predictable layout. - let lines = match self.mode { - Mode::Select => SELECT_OPTIONS - .iter() - .enumerate() - .map(|(idx, opt)| { - let (prefix, style) = if idx == self.selected_option { - ("▶", BLUE_FG) - } else { - (" ", PLAIN) - }; - Line::styled(format!(" {prefix} {}", opt.label), style) - }) - .collect(), - Mode::Input => { - vec![ - Line::from("Give the model feedback on this command:"), - Line::from(self.input.value()), - ] - } + let lines: Vec = self + .select_options + .iter() + .enumerate() + .map(|(idx, opt)| { + let style = if idx == self.selected_option { + Style::new().bg(Color::Cyan).fg(Color::Black) + } else { + Style::new().bg(Color::DarkGray) + }; + opt.label.clone().alignment(Alignment::Center).style(style) + }) + .collect(); + + let [title_area, button_area, description_area] = Layout::vertical([ + Constraint::Length(1), + Constraint::Length(1), + Constraint::Min(0), + ]) + .areas(response_chunk.inner(Margin::new(1, 0))); + let title = match &self.approval_request { + ApprovalRequest::Exec { .. } => "Allow command?", + ApprovalRequest::ApplyPatch { .. } => "Apply changes?", }; - - let border = ("◢◤") - .repeat((area.width / 2).into()) - .fg(Color::LightYellow); - - border.render_ref(area, buf); - Paragraph::new(" Execution Request ".bold().black().on_light_yellow()) - .alignment(Alignment::Center) - .render_ref(area, buf); + Line::from(title).render(title_area, buf); self.confirmation_prompt.clone().render(prompt_chunk, buf); - List::new(lines).render_ref(response_chunk, buf); + let areas = Layout::horizontal( + lines + .iter() + .map(|l| Constraint::Length(l.width() as u16 + 2)), + ) + .spacing(1) + .split(button_area); + for (idx, area) in areas.iter().enumerate() { + let line = &lines[idx]; + line.render(*area, buf); + } - border.render_ref(Rect::new(0, area.y + area.height - 1, area.width, 1), buf); + Line::from(self.select_options[self.selected_option].description) + .style(Style::new().italic().fg(Color::DarkGray)) + .render(description_area.inner(Margin::new(1, 0)), buf); + + Block::bordered() + .border_type(BorderType::QuadrantOutside) + .border_style(Style::default().fg(Color::Cyan)) + .borders(Borders::LEFT) + .render_ref( + Rect::new(0, response_chunk.y, 1, response_chunk.height), + buf, + ); } } From d787434aa81c097f242b39c1101ec7f09afa8a72 Mon Sep 17 00:00:00 2001 From: Jeremy Rose <172423086+nornagon-openai@users.noreply.github.com> Date: Thu, 31 Jul 2025 17:13:36 -0700 Subject: [PATCH 026/100] fix: always send KeyEvent, we now check kind in the handler (#1772) https://github.com/openai/codex/pull/1754 and #1771 fixed the same thing in colliding ways. --- codex-rs/tui/src/app.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index 7454a1ca11..24b2301bb9 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -96,9 +96,7 @@ impl App<'_> { if let Ok(event) = crossterm::event::read() { match event { crossterm::event::Event::Key(key_event) => { - if key_event.kind == crossterm::event::KeyEventKind::Press { - app_event_tx.send(AppEvent::KeyEvent(key_event)); - } + app_event_tx.send(AppEvent::KeyEvent(key_event)); } crossterm::event::Event::Resize(_, _) => { app_event_tx.send(AppEvent::RequestRedraw); From 4aca3e46c84431b6afd883e00baf6189805e2ab1 Mon Sep 17 00:00:00 2001 From: Jeremy Rose <172423086+nornagon-openai@users.noreply.github.com> Date: Thu, 31 Jul 2025 17:15:26 -0700 Subject: [PATCH 027/100] insert history lines with redraw (#1769) This delays the call to insert_history_lines until a redraw is happening. Crucially, the new lines are inserted _after the viewport is resized_. This results in fewer stray blank lines below the viewport when modals (e.g. user approval) are closed. --- codex-rs/tui/src/app.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index 24b2301bb9..a03cc89983 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -17,6 +17,7 @@ use crossterm::event::KeyEvent; use crossterm::event::KeyEventKind; use ratatui::layout::Offset; use ratatui::prelude::Backend; +use ratatui::text::Line; use std::path::PathBuf; use std::sync::Arc; use std::sync::atomic::AtomicBool; @@ -55,6 +56,8 @@ pub(crate) struct App<'a> { /// True when a redraw has been scheduled but not yet executed. pending_redraw: Arc, + pending_history_lines: Vec>, + /// Stored parameters needed to instantiate the ChatWidget later, e.g., /// after dismissing the Git-repo warning. chat_args: Option, @@ -152,6 +155,7 @@ impl App<'_> { let file_search = FileSearchManager::new(config.cwd.clone(), app_event_tx.clone()); Self { app_event_tx, + pending_history_lines: Vec::new(), app_event_rx, app_state, config, @@ -197,7 +201,7 @@ impl App<'_> { while let Ok(event) = self.app_event_rx.recv() { match event { AppEvent::InsertHistory(lines) => { - crate::insert_history::insert_history_lines(terminal, lines); + self.pending_history_lines.extend(lines); self.app_event_tx.send(AppEvent::RequestRedraw); } AppEvent::RequestRedraw => { @@ -409,6 +413,13 @@ impl App<'_> { terminal.clear()?; terminal.set_viewport_area(area); } + if !self.pending_history_lines.is_empty() { + crate::insert_history::insert_history_lines( + terminal, + self.pending_history_lines.clone(), + ); + self.pending_history_lines.clear(); + } match &mut self.app_state { AppState::Chat { widget } => { terminal.draw(|frame| frame.render_widget_ref(&**widget, frame.area()))?; From 575590e4c2e432e8cf5392a5214da577621d4c8d Mon Sep 17 00:00:00 2001 From: easong-openai Date: Thu, 31 Jul 2025 17:30:44 -0700 Subject: [PATCH 028/100] Detect kitty terminals (#1748) We want to detect kitty terminals so we can preferentially upgrade their UX without degrading older terminals. --- codex-rs/tui/Cargo.toml | 2 ++ codex-rs/tui/src/app.rs | 12 +++++++ .../src/bottom_pane/approval_modal_view.rs | 1 + codex-rs/tui/src/bottom_pane/chat_composer.rs | 31 +++++++++++++------ codex-rs/tui/src/bottom_pane/mod.rs | 9 +++++- ...mposer__tests__backspace_after_pastes.snap | 2 +- ...tom_pane__chat_composer__tests__empty.snap | 2 +- ...tom_pane__chat_composer__tests__large.snap | 2 +- ...chat_composer__tests__multiple_pastes.snap | 2 +- ...tom_pane__chat_composer__tests__small.snap | 2 +- codex-rs/tui/src/chatwidget.rs | 2 ++ 11 files changed, 52 insertions(+), 15 deletions(-) diff --git a/codex-rs/tui/Cargo.toml b/codex-rs/tui/Cargo.toml index 63d287ca11..468f2f3be6 100644 --- a/codex-rs/tui/Cargo.toml +++ b/codex-rs/tui/Cargo.toml @@ -62,6 +62,8 @@ unicode-segmentation = "1.12.0" unicode-width = "0.1" uuid = "1" + + [dev-dependencies] insta = "1.43.1" pretty_assertions = "1" diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index a03cc89983..cc3118e860 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -15,6 +15,7 @@ use crossterm::SynchronizedUpdate; use crossterm::event::KeyCode; use crossterm::event::KeyEvent; use crossterm::event::KeyEventKind; +use crossterm::terminal::supports_keyboard_enhancement; use ratatui::layout::Offset; use ratatui::prelude::Backend; use ratatui::text::Line; @@ -61,6 +62,8 @@ pub(crate) struct App<'a> { /// Stored parameters needed to instantiate the ChatWidget later, e.g., /// after dismissing the Git-repo warning. chat_args: Option, + + enhanced_keys_supported: bool, } /// Aggregate parameters needed to create a `ChatWidget`, as creation may be @@ -70,6 +73,7 @@ struct ChatWidgetArgs { config: Config, initial_prompt: Option, initial_images: Vec, + enhanced_keys_supported: bool, } impl App<'_> { @@ -83,6 +87,8 @@ impl App<'_> { let app_event_tx = AppEventSender::new(app_event_tx); let pending_redraw = Arc::new(AtomicBool::new(false)); + let enhanced_keys_supported = supports_keyboard_enhancement().unwrap_or(false); + // Spawn a dedicated thread for reading the crossterm event loop and // re-publishing the events as AppEvents, as appropriate. { @@ -135,6 +141,7 @@ impl App<'_> { config: config.clone(), initial_prompt, initial_images, + enhanced_keys_supported, }), ) } else { @@ -143,6 +150,7 @@ impl App<'_> { app_event_tx.clone(), initial_prompt, initial_images, + enhanced_keys_supported, ); ( AppState::Chat { @@ -162,6 +170,7 @@ impl App<'_> { file_search, pending_redraw, chat_args, + enhanced_keys_supported, } } @@ -284,6 +293,7 @@ impl App<'_> { self.app_event_tx.clone(), None, Vec::new(), + self.enhanced_keys_supported, )); self.app_state = AppState::Chat { widget: new_widget }; self.app_event_tx.send(AppEvent::RequestRedraw); @@ -400,6 +410,7 @@ impl App<'_> { AppState::Chat { widget } => widget.desired_height(size.width), AppState::GitWarning { .. } => 10, }; + let mut area = terminal.viewport_area; area.height = desired_height.min(size.height); area.width = size.width; @@ -451,6 +462,7 @@ impl App<'_> { self.app_event_tx.clone(), args.initial_prompt, args.initial_images, + args.enhanced_keys_supported, )); self.app_state = AppState::Chat { widget }; self.app_event_tx.send(AppEvent::RequestRedraw); diff --git a/codex-rs/tui/src/bottom_pane/approval_modal_view.rs b/codex-rs/tui/src/bottom_pane/approval_modal_view.rs index 4cd952f9eb..8e9ff6d936 100644 --- a/codex-rs/tui/src/bottom_pane/approval_modal_view.rs +++ b/codex-rs/tui/src/bottom_pane/approval_modal_view.rs @@ -99,6 +99,7 @@ mod tests { let mut pane = BottomPane::new(super::super::BottomPaneParams { app_event_tx: AppEventSender::new(tx_raw2), has_input_focus: true, + enhanced_keys_supported: false, }); assert_eq!(CancellationEvent::Handled, view.on_ctrl_c(&mut pane)); assert!(view.queue.is_empty()); diff --git a/codex-rs/tui/src/bottom_pane/chat_composer.rs b/codex-rs/tui/src/bottom_pane/chat_composer.rs index 3bc573a003..7e187bec8b 100644 --- a/codex-rs/tui/src/bottom_pane/chat_composer.rs +++ b/codex-rs/tui/src/bottom_pane/chat_composer.rs @@ -41,6 +41,7 @@ pub(crate) struct ChatComposer<'a> { app_event_tx: AppEventSender, history: ChatComposerHistory, ctrl_c_quit_hint: bool, + use_shift_enter_hint: bool, dismissed_file_popup_token: Option, current_file_query: Option, pending_pastes: Vec<(String, String)>, @@ -54,17 +55,24 @@ enum ActivePopup { } impl ChatComposer<'_> { - pub fn new(has_input_focus: bool, app_event_tx: AppEventSender) -> Self { + pub fn new( + has_input_focus: bool, + app_event_tx: AppEventSender, + enhanced_keys_supported: bool, + ) -> Self { let mut textarea = TextArea::default(); textarea.set_placeholder_text(BASE_PLACEHOLDER_TEXT); textarea.set_cursor_line_style(ratatui::style::Style::default()); + let use_shift_enter_hint = enhanced_keys_supported; + let mut this = Self { textarea, active_popup: ActivePopup::None, app_event_tx, history: ChatComposerHistory::new(), ctrl_c_quit_hint: false, + use_shift_enter_hint, dismissed_file_popup_token: None, current_file_query: None, pending_pastes: Vec::new(), @@ -712,11 +720,16 @@ impl WidgetRef for &ChatComposer<'_> { Span::from(" to quit"), ] } else { + let newline_hint_key = if self.use_shift_enter_hint { + "Shift+⏎" + } else { + "Ctrl+J" + }; vec![ Span::from(" "), "⏎".set_style(key_hint_style), Span::from(" send "), - "Shift+⏎".set_style(key_hint_style), + newline_hint_key.set_style(key_hint_style), Span::from(" newline "), "Ctrl+C".set_style(key_hint_style), Span::from(" quit"), @@ -890,7 +903,7 @@ mod tests { let (tx, _rx) = std::sync::mpsc::channel(); let sender = AppEventSender::new(tx); - let mut composer = ChatComposer::new(true, sender); + let mut composer = ChatComposer::new(true, sender, false); let needs_redraw = composer.handle_paste("hello".to_string()); assert!(needs_redraw); @@ -913,7 +926,7 @@ mod tests { let (tx, _rx) = std::sync::mpsc::channel(); let sender = AppEventSender::new(tx); - let mut composer = ChatComposer::new(true, sender); + let mut composer = ChatComposer::new(true, sender, false); let large = "x".repeat(LARGE_PASTE_CHAR_THRESHOLD + 10); let needs_redraw = composer.handle_paste(large.clone()); @@ -942,7 +955,7 @@ mod tests { let large = "y".repeat(LARGE_PASTE_CHAR_THRESHOLD + 1); let (tx, _rx) = std::sync::mpsc::channel(); let sender = AppEventSender::new(tx); - let mut composer = ChatComposer::new(true, sender); + let mut composer = ChatComposer::new(true, sender, false); composer.handle_paste(large); assert_eq!(composer.pending_pastes.len(), 1); @@ -978,7 +991,7 @@ mod tests { for (name, input) in test_cases { // Create a fresh composer for each test case - let mut composer = ChatComposer::new(true, sender.clone()); + let mut composer = ChatComposer::new(true, sender.clone(), false); if let Some(text) = input { composer.handle_paste(text); @@ -1015,7 +1028,7 @@ mod tests { let (tx, _rx) = std::sync::mpsc::channel(); let sender = AppEventSender::new(tx); - let mut composer = ChatComposer::new(true, sender); + let mut composer = ChatComposer::new(true, sender, false); // Define test cases: (paste content, is_large) let test_cases = [ @@ -1088,7 +1101,7 @@ mod tests { let (tx, _rx) = std::sync::mpsc::channel(); let sender = AppEventSender::new(tx); - let mut composer = ChatComposer::new(true, sender); + let mut composer = ChatComposer::new(true, sender, false); // Define test cases: (content, is_large) let test_cases = [ @@ -1161,7 +1174,7 @@ mod tests { let (tx, _rx) = std::sync::mpsc::channel(); let sender = AppEventSender::new(tx); - let mut composer = ChatComposer::new(true, sender); + let mut composer = ChatComposer::new(true, sender, false); // Define test cases: (cursor_position_from_end, expected_pending_count) let test_cases = [ diff --git a/codex-rs/tui/src/bottom_pane/mod.rs b/codex-rs/tui/src/bottom_pane/mod.rs index 2710a3e997..281f0859a5 100644 --- a/codex-rs/tui/src/bottom_pane/mod.rs +++ b/codex-rs/tui/src/bottom_pane/mod.rs @@ -50,12 +50,18 @@ pub(crate) struct BottomPane<'a> { pub(crate) struct BottomPaneParams { pub(crate) app_event_tx: AppEventSender, pub(crate) has_input_focus: bool, + pub(crate) enhanced_keys_supported: bool, } impl BottomPane<'_> { pub fn new(params: BottomPaneParams) -> Self { + let enhanced_keys_supported = params.enhanced_keys_supported; Self { - composer: ChatComposer::new(params.has_input_focus, params.app_event_tx.clone()), + composer: ChatComposer::new( + params.has_input_focus, + params.app_event_tx.clone(), + enhanced_keys_supported, + ), active_view: None, app_event_tx: params.app_event_tx, has_input_focus: params.has_input_focus, @@ -298,6 +304,7 @@ mod tests { let mut pane = BottomPane::new(BottomPaneParams { app_event_tx: tx, has_input_focus: true, + enhanced_keys_supported: false, }); pane.push_approval_request(exec_request()); assert_eq!(CancellationEvent::Handled, pane.on_ctrl_c()); diff --git a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__backspace_after_pastes.snap b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__backspace_after_pastes.snap index 4f155dab30..178e2cc40f 100644 --- a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__backspace_after_pastes.snap +++ b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__backspace_after_pastes.snap @@ -11,4 +11,4 @@ expression: terminal.backend() "▌ " "▌ " "▌ " -" ⏎ send Shift+⏎ newline Ctrl+C quit " +" ⏎ send Ctrl+J newline Ctrl+C quit " diff --git a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__empty.snap b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__empty.snap index 4e8371f177..7a1a7f4ef3 100644 --- a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__empty.snap +++ b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__empty.snap @@ -11,4 +11,4 @@ expression: terminal.backend() "▌ " "▌ " "▌ " -" ⏎ send Shift+⏎ newline Ctrl+C quit " +" ⏎ send Ctrl+J newline Ctrl+C quit " diff --git a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__large.snap b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__large.snap index 80fea40d5f..d3f67c9219 100644 --- a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__large.snap +++ b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__large.snap @@ -11,4 +11,4 @@ expression: terminal.backend() "▌ " "▌ " "▌ " -" ⏎ send Shift+⏎ newline Ctrl+C quit " +" ⏎ send Ctrl+J newline Ctrl+C quit " diff --git a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__multiple_pastes.snap b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__multiple_pastes.snap index 26e8d26733..999e7b2e3f 100644 --- a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__multiple_pastes.snap +++ b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__multiple_pastes.snap @@ -11,4 +11,4 @@ expression: terminal.backend() "▌ " "▌ " "▌ " -" ⏎ send Shift+⏎ newline Ctrl+C quit " +" ⏎ send Ctrl+J newline Ctrl+C quit " diff --git a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__small.snap b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__small.snap index 0f1b9e6426..68d7711aab 100644 --- a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__small.snap +++ b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__chat_composer__tests__small.snap @@ -11,4 +11,4 @@ expression: terminal.backend() "▌ " "▌ " "▌ " -" ⏎ send Shift+⏎ newline Ctrl+C quit " +" ⏎ send Ctrl+J newline Ctrl+C quit " diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index b0a6fb44e1..4c415777a9 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -95,6 +95,7 @@ impl ChatWidget<'_> { app_event_tx: AppEventSender, initial_prompt: Option, initial_images: Vec, + enhanced_keys_supported: bool, ) -> Self { let (codex_op_tx, mut codex_op_rx) = unbounded_channel::(); @@ -140,6 +141,7 @@ impl ChatWidget<'_> { bottom_pane: BottomPane::new(BottomPaneParams { app_event_tx, has_input_focus: true, + enhanced_keys_supported, }), config, initial_user_message: create_initial_user_message( From d3aa5f46b78e48fc25b9d867740b7315a0c28cdd Mon Sep 17 00:00:00 2001 From: aibrahim-oai Date: Thu, 31 Jul 2025 19:30:03 -0700 Subject: [PATCH 029/100] MCP Protocol: Align tool-call response with CallToolResult [Stack 1/3] (#1750) # Summary - Align MCP server responses with mcp_types by emitting [CallToolResult, RequestId] instead of an object. Update send-message result to a tagged enum: Ok or Error { message }. # Why Protocol compliance with current MCP schema. # Tests - Updated assertions in mcp_protocol.rs for create/stream/send/list and error cases. This is the first PR in a stack. Stack: Final: #1686 Intermediate: #1751 First: #1750 --- codex-rs/mcp-server/src/mcp_protocol.rs | 92 ++++++++++++++++++++----- 1 file changed, 74 insertions(+), 18 deletions(-) diff --git a/codex-rs/mcp-server/src/mcp_protocol.rs b/codex-rs/mcp-server/src/mcp_protocol.rs index e507376c16..23304dc43d 100644 --- a/codex-rs/mcp-server/src/mcp_protocol.rs +++ b/codex-rs/mcp-server/src/mcp_protocol.rs @@ -7,7 +7,10 @@ use serde::Serialize; use strum_macros::Display; use uuid::Uuid; +use mcp_types::CallToolResult; +use mcp_types::ContentBlock; use mcp_types::RequestId; +use mcp_types::TextContent; #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(transparent)] @@ -118,10 +121,47 @@ pub struct ToolCallResponse { pub request_id: RequestId, #[serde(default, skip_serializing_if = "Option::is_none")] pub is_error: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] + #[serde(default, skip_serializing_if = "Option::is_none", flatten)] pub result: Option, } +impl From for CallToolResult { + fn from(val: ToolCallResponse) -> Self { + let ToolCallResponse { + request_id: _request_id, + is_error, + result, + } = val; + let (content, structured_content, is_error_out) = match result { + Some(res) => match serde_json::to_value(&res) { + Ok(v) => { + let content = vec![ContentBlock::TextContent(TextContent { + r#type: "text".to_string(), + text: v.to_string(), + annotations: None, + })]; + (content, Some(v), is_error) + } + Err(e) => { + let content = vec![ContentBlock::TextContent(TextContent { + r#type: "text".to_string(), + text: format!("Failed to serialize tool result: {e}"), + annotations: None, + })]; + (content, None, Some(true)) + } + }, + None => (vec![], None, is_error), + }; + + CallToolResult { + content, + is_error: is_error_out, + structured_content, + } + } +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(untagged)] pub enum ToolCallResponseResult { @@ -141,8 +181,10 @@ pub struct ConversationCreateResult { pub struct ConversationStreamResult {} #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct ConversationSendMessageResult { - pub success: bool, +#[serde(tag = "status", rename_all = "camelCase")] +pub enum ConversationSendMessageResult { + Ok, + Error { message: String }, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -455,10 +497,13 @@ mod tests { }, )), }; - let observed = to_val(&env); + let req_id = env.request_id.clone(); + let observed = to_val(&CallToolResult::from(env)); let expected = json!({ - "requestId": 1, - "result": { + "content": [ + { "type": "text", "text": "{\"conversation_id\":\"d0f6ecbe-84a2-41c1-b23d-b20473b25eab\",\"model\":\"o3\"}" } + ], + "structuredContent": { "conversation_id": "d0f6ecbe-84a2-41c1-b23d-b20473b25eab", "model": "o3" } @@ -467,6 +512,7 @@ mod tests { observed, expected, "response (ConversationCreate) must match" ); + assert_eq!(req_id, RequestId::Integer(1)); } #[test] @@ -478,15 +524,17 @@ mod tests { ConversationStreamResult {}, )), }; - let observed = to_val(&env); + let req_id = env.request_id.clone(); + let observed = to_val(&CallToolResult::from(env)); let expected = json!({ - "requestId": 2, - "result": {} + "content": [ { "type": "text", "text": "{}" } ], + "structuredContent": {} }); assert_eq!( observed, expected, "response (ConversationStream) must have empty object result" ); + assert_eq!(req_id, RequestId::Integer(2)); } #[test] @@ -495,18 +543,20 @@ mod tests { request_id: RequestId::Integer(3), is_error: None, result: Some(ToolCallResponseResult::ConversationSendMessage( - ConversationSendMessageResult { success: true }, + ConversationSendMessageResult::Ok, )), }; - let observed = to_val(&env); + let req_id = env.request_id.clone(); + let observed = to_val(&CallToolResult::from(env)); let expected = json!({ - "requestId": 3, - "result": { "success": true } + "content": [ { "type": "text", "text": "{\"status\":\"ok\"}" } ], + "structuredContent": { "status": "ok" } }); assert_eq!( observed, expected, "response (ConversationSendMessageAccepted) must match" ); + assert_eq!(req_id, RequestId::Integer(3)); } #[test] @@ -526,10 +576,13 @@ mod tests { }, )), }; - let observed = to_val(&env); + let req_id = env.request_id.clone(); + let observed = to_val(&CallToolResult::from(env)); let expected = json!({ - "requestId": 4, - "result": { + "content": [ + { "type": "text", "text": "{\"conversations\":[{\"conversation_id\":\"67e55044-10b1-426f-9247-bb680e5fe0c8\",\"title\":\"Refactor config loader\"}],\"next_cursor\":\"next123\"}" } + ], + "structuredContent": { "conversations": [ { "conversation_id": "67e55044-10b1-426f-9247-bb680e5fe0c8", @@ -543,6 +596,7 @@ mod tests { observed, expected, "response (ConversationsList with cursor) must match" ); + assert_eq!(req_id, RequestId::Integer(4)); } #[test] @@ -552,15 +606,17 @@ mod tests { is_error: Some(true), result: None, }; - let observed = to_val(&env); + let req_id = env.request_id.clone(); + let observed = to_val(&CallToolResult::from(env)); let expected = json!({ - "requestId": 4, + "content": [], "isError": true }); assert_eq!( observed, expected, "error response must omit `result` and include `isError`" ); + assert_eq!(req_id, RequestId::Integer(4)); } // ----- Notifications ----- From ad0295b89336d0264cd68d1389e977239e4751f0 Mon Sep 17 00:00:00 2001 From: aibrahim-oai Date: Thu, 31 Jul 2025 19:46:04 -0700 Subject: [PATCH 030/100] MCP server: route structured tool-call requests and expose mcp_protocol [Stack 2/3] (#1751) - Expose mcp_protocol from mcp-server for reuse in tests and callers. - In MessageProcessor, detect structured ToolCallRequestParams in tools/call and forward to a new handler. - Add handle_new_tool_calls scaffold (returns error for now). - Test helper: add send_send_user_message_tool_call to McpProcess to send ConversationSendMessage requests; This is the second PR in a stack. Stack: Final: #1686 Intermediate: #1751 First: #1750 --- codex-rs/Cargo.lock | 2 ++ codex-rs/mcp-server/src/lib.rs | 2 +- codex-rs/mcp-server/src/message_processor.rs | 23 ++++++++++++++++ codex-rs/mcp-server/tests/common/Cargo.toml | 2 ++ .../mcp-server/tests/common/mcp_process.rs | 26 +++++++++++++++++++ 5 files changed, 54 insertions(+), 1 deletion(-) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 87d59b21be..9123bd6374 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -2643,6 +2643,7 @@ version = "0.0.0" dependencies = [ "anyhow", "assert_cmd", + "codex-core", "codex-mcp-server", "mcp-types", "pretty_assertions", @@ -2650,6 +2651,7 @@ dependencies = [ "shlex", "tempfile", "tokio", + "uuid", "wiremock", ] diff --git a/codex-rs/mcp-server/src/lib.rs b/codex-rs/mcp-server/src/lib.rs index 0912fed118..ebef8ca903 100644 --- a/codex-rs/mcp-server/src/lib.rs +++ b/codex-rs/mcp-server/src/lib.rs @@ -19,7 +19,7 @@ mod codex_tool_config; mod codex_tool_runner; mod exec_approval; mod json_to_toml; -mod mcp_protocol; +pub mod mcp_protocol; mod message_processor; mod outgoing_message; mod patch_approval; diff --git a/codex-rs/mcp-server/src/message_processor.rs b/codex-rs/mcp-server/src/message_processor.rs index 7ba827d60b..a4013cc3a3 100644 --- a/codex-rs/mcp-server/src/message_processor.rs +++ b/codex-rs/mcp-server/src/message_processor.rs @@ -6,6 +6,7 @@ use crate::codex_tool_config::CodexToolCallParam; use crate::codex_tool_config::CodexToolCallReplyParam; use crate::codex_tool_config::create_tool_for_codex_tool_call_param; use crate::codex_tool_config::create_tool_for_codex_tool_call_reply_param; +use crate::mcp_protocol::ToolCallRequestParams; use crate::outgoing_message::OutgoingMessageSender; use codex_core::Codex; @@ -300,6 +301,14 @@ impl MessageProcessor { params: ::Params, ) { tracing::info!("tools/call -> params: {:?}", params); + // Serialize params into JSON and try to parse as new type + if let Ok(new_params) = + serde_json::to_value(¶ms).and_then(serde_json::from_value::) + { + // New tool call matched → forward + self.handle_new_tool_calls(id, new_params).await; + return; + } let CallToolRequestParams { name, arguments } = params; match name.as_str() { @@ -323,6 +332,20 @@ impl MessageProcessor { } } } + async fn handle_new_tool_calls(&self, request_id: RequestId, _params: ToolCallRequestParams) { + // TODO: implement the new tool calls + let result = CallToolResult { + content: vec![ContentBlock::TextContent(TextContent { + r#type: "text".to_string(), + text: "Unknown tool".to_string(), + annotations: None, + })], + is_error: Some(true), + structured_content: None, + }; + self.send_response::(request_id, result) + .await; + } async fn handle_tool_call_codex(&self, id: RequestId, arguments: Option) { let (initial_prompt, config): (String, CodexConfig) = match arguments { diff --git a/codex-rs/mcp-server/tests/common/Cargo.toml b/codex-rs/mcp-server/tests/common/Cargo.toml index 3aa246f154..420b97811d 100644 --- a/codex-rs/mcp-server/tests/common/Cargo.toml +++ b/codex-rs/mcp-server/tests/common/Cargo.toml @@ -10,6 +10,7 @@ path = "lib.rs" anyhow = "1" assert_cmd = "2" codex-mcp-server = { path = "../.." } +codex-core = { path = "../../../core" } mcp-types = { path = "../../../mcp-types" } pretty_assertions = "1.4.1" serde_json = "1" @@ -22,3 +23,4 @@ tokio = { version = "1", features = [ "rt-multi-thread", ] } wiremock = "0.6" +uuid = { version = "1", features = ["serde", "v4"] } \ No newline at end of file diff --git a/codex-rs/mcp-server/tests/common/mcp_process.rs b/codex-rs/mcp-server/tests/common/mcp_process.rs index 8138749c40..d7144af480 100644 --- a/codex-rs/mcp-server/tests/common/mcp_process.rs +++ b/codex-rs/mcp-server/tests/common/mcp_process.rs @@ -11,8 +11,13 @@ use tokio::process::ChildStdout; use anyhow::Context; use assert_cmd::prelude::*; +use codex_core::protocol::InputItem; use codex_mcp_server::CodexToolCallParam; use codex_mcp_server::CodexToolCallReplyParam; +use codex_mcp_server::mcp_protocol::ConversationId; +use codex_mcp_server::mcp_protocol::ConversationSendMessageArgs; +use codex_mcp_server::mcp_protocol::ToolCallRequestParams; + use mcp_types::CallToolRequestParams; use mcp_types::ClientCapabilities; use mcp_types::Implementation; @@ -29,6 +34,7 @@ use pretty_assertions::assert_eq; use serde_json::json; use std::process::Command as StdCommand; use tokio::process::Command; +use uuid::Uuid; pub struct McpProcess { next_request_id: AtomicI64, @@ -174,6 +180,26 @@ impl McpProcess { .await } + pub async fn send_user_message_tool_call( + &mut self, + message: &str, + session_id: &str, + ) -> anyhow::Result { + let params = ToolCallRequestParams::ConversationSendMessage(ConversationSendMessageArgs { + conversation_id: ConversationId(Uuid::parse_str(session_id)?), + content: vec![InputItem::Text { + text: message.to_string(), + }], + parent_message_id: None, + conversation_overrides: None, + }); + self.send_request( + mcp_types::CallToolRequest::METHOD, + Some(serde_json::to_value(params)?), + ) + .await + } + async fn send_request( &mut self, method: &str, From e2c994e32a31415e87070bef28ed698968d2e549 Mon Sep 17 00:00:00 2001 From: aibrahim-oai Date: Thu, 31 Jul 2025 21:34:32 -0700 Subject: [PATCH 031/100] Add /compact (#1527) - Add operation to summarize the context so far. - The operation runs a compact task that summarizes the context. - The operation clear the previous context to free the context window - The operation didn't use `run_task` to avoid corrupting the session - Add /compact in the tui https://github.com/user-attachments/assets/e06c24e5-dcfb-4806-934a-564d425a919c --- SUMMARY.md | 21 ++ codex-rs/core/src/codex.rs | 154 ++++++++++++- codex-rs/core/src/conversation_history.rs | 28 +++ codex-rs/core/src/protocol.rs | 4 + codex-rs/core/tests/compact.rs | 254 ++++++++++++++++++++++ codex-rs/tui/src/app.rs | 8 + codex-rs/tui/src/chatwidget.rs | 6 + codex-rs/tui/src/slash_command.rs | 2 + 8 files changed, 474 insertions(+), 3 deletions(-) create mode 100644 SUMMARY.md create mode 100644 codex-rs/core/tests/compact.rs diff --git a/SUMMARY.md b/SUMMARY.md new file mode 100644 index 0000000000..5cdecdbf4c --- /dev/null +++ b/SUMMARY.md @@ -0,0 +1,21 @@ +You are a summarization assistant. A conversation follows between a user and a coding-focused AI (Codex). Your task is to generate a clear summary capturing: + +• High-level objective or problem being solved +• Key instructions or design decisions given by the user +• Main code actions or behaviors from the AI +• Important variables, functions, modules, or outputs discussed +• Any unresolved questions or next steps + +Produce the summary in a structured format like: + +**Objective:** … + +**User instructions:** … (bulleted) + +**AI actions / code behavior:** … (bulleted) + +**Important entities:** … (e.g. function names, variables, files) + +**Open issues / next steps:** … (if any) + +**Summary (concise):** (one or two sentences) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 3dd1d513a2..5d48cd457f 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -442,6 +442,12 @@ impl Session { let _ = self.tx_event.send(event).await; } + /// Build the full turn input by concatenating the current conversation + /// history with additional items for this turn. + pub fn turn_input_with_history(&self, extra: Vec) -> Vec { + [self.state.lock().unwrap().history.contents(), extra].concat() + } + /// Returns the input if there was no task running to inject into pub fn inject_input(&self, input: Vec) -> Result<(), Vec> { let mut state = self.state.lock().unwrap(); @@ -564,6 +570,25 @@ impl AgentTask { handle, } } + fn compact( + sess: Arc, + sub_id: String, + input: Vec, + compact_instructions: String, + ) -> Self { + let handle = tokio::spawn(run_compact_task( + Arc::clone(&sess), + sub_id.clone(), + input, + compact_instructions, + )) + .abort_handle(); + Self { + sess, + sub_id, + handle, + } + } fn abort(self) { if !self.handle.is_finished() { @@ -884,6 +909,31 @@ async fn submission_loop( } }); } + Op::Compact => { + let sess = match sess.as_ref() { + Some(sess) => sess, + None => { + send_no_session_event(sub.id).await; + continue; + } + }; + + // Create a summarization request as user input + const SUMMARIZATION_PROMPT: &str = include_str!("../../../SUMMARY.md"); + + // Attempt to inject input into current task + if let Err(items) = sess.inject_input(vec![InputItem::Text { + text: "Start Summarization".to_string(), + }]) { + let task = AgentTask::compact( + sess.clone(), + sub.id, + items, + SUMMARIZATION_PROMPT.to_string(), + ); + sess.set_task(task); + } + } Op::Shutdown => { info!("Shutting down Codex instance"); @@ -945,7 +995,7 @@ async fn run_task(sess: Arc, sub_id: String, input: Vec) { return; } - let initial_input_for_turn = ResponseInputItem::from(input); + let initial_input_for_turn: ResponseInputItem = ResponseInputItem::from(input); sess.record_conversation_items(&[initial_input_for_turn.clone().into()]) .await; @@ -966,8 +1016,7 @@ async fn run_task(sess: Arc, sub_id: String, input: Vec) { // conversation history on each turn. The rollout file, however, should // only record the new items that originated in this turn so that it // represents an append-only log without duplicates. - let turn_input: Vec = - [sess.state.lock().unwrap().history.contents(), pending_input].concat(); + let turn_input: Vec = sess.turn_input_with_history(pending_input); let turn_input_messages: Vec = turn_input .iter() @@ -1293,6 +1342,88 @@ async fn try_run_turn( } } +async fn run_compact_task( + sess: Arc, + sub_id: String, + input: Vec, + compact_instructions: String, +) { + let start_event = Event { + id: sub_id.clone(), + msg: EventMsg::TaskStarted, + }; + if sess.tx_event.send(start_event).await.is_err() { + return; + } + + let initial_input_for_turn: ResponseInputItem = ResponseInputItem::from(input); + let turn_input: Vec = + sess.turn_input_with_history(vec![initial_input_for_turn.clone().into()]); + + let prompt = Prompt { + input: turn_input, + user_instructions: None, + store: !sess.disable_response_storage, + extra_tools: HashMap::new(), + base_instructions_override: Some(compact_instructions.clone()), + }; + + let max_retries = sess.client.get_provider().stream_max_retries(); + let mut retries = 0; + + loop { + let attempt_result = drain_to_completed(&sess, &prompt).await; + + match attempt_result { + Ok(()) => break, + Err(CodexErr::Interrupted) => return, + Err(e) => { + if retries < max_retries { + retries += 1; + let delay = backoff(retries); + sess.notify_background_event( + &sub_id, + format!( + "stream error: {e}; retrying {retries}/{max_retries} in {delay:?}…" + ), + ) + .await; + tokio::time::sleep(delay).await; + continue; + } else { + let event = Event { + id: sub_id.clone(), + msg: EventMsg::Error(ErrorEvent { + message: e.to_string(), + }), + }; + sess.send_event(event).await; + return; + } + } + } + } + + sess.remove_task(&sub_id); + let event = Event { + id: sub_id.clone(), + msg: EventMsg::AgentMessage(AgentMessageEvent { + message: "Compact task completed".to_string(), + }), + }; + sess.send_event(event).await; + let event = Event { + id: sub_id.clone(), + msg: EventMsg::TaskComplete(TaskCompleteEvent { + last_agent_message: None, + }), + }; + sess.send_event(event).await; + + let mut state = sess.state.lock().unwrap(); + state.history.keep_last_messages(1); +} + async fn handle_response_item( sess: &Session, sub_id: &str, @@ -1858,3 +1989,20 @@ fn get_last_assistant_message_from_turn(responses: &[ResponseItem]) -> Option CodexResult<()> { + let mut stream = sess.client.clone().stream(prompt).await?; + loop { + let maybe_event = stream.next().await; + let Some(event) = maybe_event else { + return Err(CodexErr::Stream( + "stream closed before response.completed".into(), + )); + }; + match event { + Ok(ResponseEvent::Completed { .. }) => return Ok(()), + Ok(_) => continue, + Err(e) => return Err(e), + } + } +} diff --git a/codex-rs/core/src/conversation_history.rs b/codex-rs/core/src/conversation_history.rs index 4cd989cbd9..f5254f339e 100644 --- a/codex-rs/core/src/conversation_history.rs +++ b/codex-rs/core/src/conversation_history.rs @@ -30,6 +30,34 @@ impl ConversationHistory { } } } + + pub(crate) fn keep_last_messages(&mut self, n: usize) { + if n == 0 { + self.items.clear(); + return; + } + + // Collect the last N message items (assistant/user), newest to oldest. + let mut kept: Vec = Vec::with_capacity(n); + for item in self.items.iter().rev() { + if let ResponseItem::Message { role, content, .. } = item { + kept.push(ResponseItem::Message { + // we need to remove the id or the model will complain that messages are sent without + // their reasonings + id: None, + role: role.clone(), + content: content.clone(), + }); + if kept.len() == n { + break; + } + } + } + + // Preserve chronological order (oldest to newest) within the kept slice. + kept.reverse(); + self.items = kept; + } } /// Anything that is not a system message or "reasoning" message is considered diff --git a/codex-rs/core/src/protocol.rs b/codex-rs/core/src/protocol.rs index bc922eb0e2..1b4832a273 100644 --- a/codex-rs/core/src/protocol.rs +++ b/codex-rs/core/src/protocol.rs @@ -121,6 +121,10 @@ pub enum Op { /// Request a single history entry identified by `log_id` + `offset`. GetHistoryEntryRequest { offset: usize, log_id: u64 }, + /// Request the agent to summarize the current conversation context. + /// The agent will use its existing context (either conversation history or previous response id) + /// to generate a summary which will be returned as an AgentMessage event. + Compact, /// Request to shut down codex instance. Shutdown, } diff --git a/codex-rs/core/tests/compact.rs b/codex-rs/core/tests/compact.rs new file mode 100644 index 0000000000..8967c86a8f --- /dev/null +++ b/codex-rs/core/tests/compact.rs @@ -0,0 +1,254 @@ +#![expect(clippy::unwrap_used)] + +use codex_core::Codex; +use codex_core::CodexSpawnOk; +use codex_core::ModelProviderInfo; +use codex_core::built_in_model_providers; +use codex_core::exec::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR; +use codex_core::protocol::EventMsg; +use codex_core::protocol::InputItem; +use codex_core::protocol::Op; +use codex_login::CodexAuth; +use core_test_support::load_default_config_for_test; +use core_test_support::wait_for_event; +use serde_json::Value; +use tempfile::TempDir; +use wiremock::Mock; +use wiremock::MockServer; +use wiremock::ResponseTemplate; +use wiremock::matchers::method; +use wiremock::matchers::path; + +use pretty_assertions::assert_eq; + +// --- Test helpers ----------------------------------------------------------- + +/// Build an SSE stream body from a list of JSON events. +fn sse(events: Vec) -> String { + use std::fmt::Write as _; + let mut out = String::new(); + for ev in events { + let kind = ev.get("type").and_then(|v| v.as_str()).unwrap(); + writeln!(&mut out, "event: {kind}").unwrap(); + if !ev.as_object().map(|o| o.len() == 1).unwrap_or(false) { + write!(&mut out, "data: {ev}\n\n").unwrap(); + } else { + out.push('\n'); + } + } + out +} + +/// Convenience: SSE event for a completed response with a specific id. +fn ev_completed(id: &str) -> Value { + serde_json::json!({ + "type": "response.completed", + "response": { + "id": id, + "usage": {"input_tokens":0,"input_tokens_details":null,"output_tokens":0,"output_tokens_details":null,"total_tokens":0} + } + }) +} + +/// Convenience: SSE event for a single assistant message output item. +fn ev_assistant_message(id: &str, text: &str) -> Value { + serde_json::json!({ + "type": "response.output_item.done", + "item": { + "type": "message", + "role": "assistant", + "id": id, + "content": [{"type": "output_text", "text": text}] + } + }) +} + +fn sse_response(body: String) -> ResponseTemplate { + ResponseTemplate::new(200) + .insert_header("content-type", "text/event-stream") + .set_body_raw(body, "text/event-stream") +} + +async fn mount_sse_once(server: &MockServer, matcher: M, body: String) +where + M: wiremock::Match + Send + Sync + 'static, +{ + Mock::given(method("POST")) + .and(path("/v1/responses")) + .and(matcher) + .respond_with(sse_response(body)) + .expect(1) + .mount(server) + .await; +} + +const FIRST_REPLY: &str = "FIRST_REPLY"; +const SUMMARY_TEXT: &str = "SUMMARY_ONLY_CONTEXT"; +const SUMMARIZE_TRIGGER: &str = "Start Summarization"; +const THIRD_USER_MSG: &str = "next turn"; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn summarize_context_three_requests_and_instructions() { + if std::env::var(CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR).is_ok() { + println!( + "Skipping test because it cannot execute when network is disabled in a Codex sandbox." + ); + return; + } + + // Set up a mock server that we can inspect after the run. + let server = MockServer::start().await; + + // SSE 1: assistant replies normally so it is recorded in history. + let sse1 = sse(vec![ + ev_assistant_message("m1", FIRST_REPLY), + ev_completed("r1"), + ]); + + // SSE 2: summarizer returns a summary message. + let sse2 = sse(vec![ + ev_assistant_message("m2", SUMMARY_TEXT), + ev_completed("r2"), + ]); + + // SSE 3: minimal completed; we only need to capture the request body. + let sse3 = sse(vec![ev_completed("r3")]); + + // Mount three expectations, one per request, matched by body content. + let first_matcher = |req: &wiremock::Request| { + let body = std::str::from_utf8(&req.body).unwrap_or(""); + body.contains("\"text\":\"hello world\"") + && !body.contains(&format!("\"text\":\"{SUMMARIZE_TRIGGER}\"")) + }; + mount_sse_once(&server, first_matcher, sse1).await; + + let second_matcher = |req: &wiremock::Request| { + let body = std::str::from_utf8(&req.body).unwrap_or(""); + body.contains(&format!("\"text\":\"{SUMMARIZE_TRIGGER}\"")) + }; + mount_sse_once(&server, second_matcher, sse2).await; + + let third_matcher = |req: &wiremock::Request| { + let body = std::str::from_utf8(&req.body).unwrap_or(""); + body.contains(&format!("\"text\":\"{THIRD_USER_MSG}\"")) + }; + mount_sse_once(&server, third_matcher, sse3).await; + + // Build config pointing to the mock server and spawn Codex. + let model_provider = ModelProviderInfo { + base_url: Some(format!("{}/v1", server.uri())), + ..built_in_model_providers()["openai"].clone() + }; + let home = TempDir::new().unwrap(); + let mut config = load_default_config_for_test(&home); + config.model_provider = model_provider; + let ctrl_c = std::sync::Arc::new(tokio::sync::Notify::new()); + let CodexSpawnOk { codex, .. } = Codex::spawn( + config, + Some(CodexAuth::from_api_key("dummy".to_string())), + ctrl_c.clone(), + ) + .await + .unwrap(); + + // 1) Normal user input – should hit server once. + codex + .submit(Op::UserInput { + items: vec![InputItem::Text { + text: "hello world".into(), + }], + }) + .await + .unwrap(); + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TaskComplete(_))).await; + + // 2) Summarize – second hit with summarization instructions. + codex.submit(Op::Compact).await.unwrap(); + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TaskComplete(_))).await; + + // 3) Next user input – third hit; history should include only the summary. + codex + .submit(Op::UserInput { + items: vec![InputItem::Text { + text: THIRD_USER_MSG.into(), + }], + }) + .await + .unwrap(); + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TaskComplete(_))).await; + + // Inspect the three captured requests. + let requests = server.received_requests().await.unwrap(); + assert_eq!(requests.len(), 3, "expected exactly three requests"); + + let req1 = &requests[0]; + let req2 = &requests[1]; + let req3 = &requests[2]; + + let body1 = req1.body_json::().unwrap(); + let body2 = req2.body_json::().unwrap(); + let body3 = req3.body_json::().unwrap(); + + // System instructions should change for the summarization turn. + let instr1 = body1.get("instructions").and_then(|v| v.as_str()).unwrap(); + let instr2 = body2.get("instructions").and_then(|v| v.as_str()).unwrap(); + assert_ne!( + instr1, instr2, + "summarization should override base instructions" + ); + assert!( + instr2.contains("You are a summarization assistant"), + "summarization instructions not applied" + ); + + // The summarization request should include the injected user input marker. + let input2 = body2.get("input").and_then(|v| v.as_array()).unwrap(); + // The last item is the user message created from the injected input. + let last2 = input2.last().unwrap(); + assert_eq!(last2.get("type").unwrap().as_str().unwrap(), "message"); + assert_eq!(last2.get("role").unwrap().as_str().unwrap(), "user"); + let text2 = last2["content"][0]["text"].as_str().unwrap(); + assert!(text2.contains(SUMMARIZE_TRIGGER)); + + // Third request must contain only the summary from step 2 as prior history plus new user msg. + let input3 = body3.get("input").and_then(|v| v.as_array()).unwrap(); + println!("third request body: {body3}"); + assert!( + input3.len() >= 2, + "expected summary + new user message in third request" + ); + + // Collect all (role, text) message tuples. + let mut messages: Vec<(String, String)> = Vec::new(); + for item in input3 { + if item["type"].as_str() == Some("message") { + let role = item["role"].as_str().unwrap_or_default().to_string(); + let text = item["content"][0]["text"] + .as_str() + .unwrap_or_default() + .to_string(); + messages.push((role, text)); + } + } + + // Exactly one assistant message should remain after compaction and the new user message is present. + let assistant_count = messages.iter().filter(|(r, _)| r == "assistant").count(); + assert_eq!( + assistant_count, 1, + "exactly one assistant message should remain after compaction" + ); + assert!( + messages + .iter() + .any(|(r, t)| r == "user" && t == THIRD_USER_MSG), + "third request should include the new user message" + ); + assert!( + !messages.iter().any(|(_, t)| t.contains("hello world")), + "third request should not include the original user input" + ); + assert!( + !messages.iter().any(|(_, t)| t.contains(SUMMARIZE_TRIGGER)), + "third request should not include the summarize trigger" + ); +} diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index cc3118e860..d4c2f3c12d 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -10,6 +10,8 @@ use crate::tui; use codex_core::config::Config; use codex_core::protocol::Event; use codex_core::protocol::EventMsg; +use codex_core::protocol::ExecApprovalRequestEvent; +use codex_core::protocol::Op; use color_eyre::eyre::Result; use crossterm::SynchronizedUpdate; use crossterm::event::KeyCode; @@ -298,6 +300,12 @@ impl App<'_> { self.app_state = AppState::Chat { widget: new_widget }; self.app_event_tx.send(AppEvent::RequestRedraw); } + SlashCommand::Compact => { + if let AppState::Chat { widget } = &mut self.app_state { + widget.clear_token_usage(); + self.app_event_tx.send(AppEvent::CodexOp(Op::Compact)); + } + } SlashCommand::Quit => { break; } diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 4c415777a9..c777af5264 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -502,6 +502,12 @@ impl ChatWidget<'_> { pub(crate) fn token_usage(&self) -> &TokenUsage { &self.token_usage } + + pub(crate) fn clear_token_usage(&mut self) { + self.token_usage = TokenUsage::default(); + self.bottom_pane + .set_token_usage(self.token_usage.clone(), self.config.model_context_window); + } } impl WidgetRef for &ChatWidget<'_> { diff --git a/codex-rs/tui/src/slash_command.rs b/codex-rs/tui/src/slash_command.rs index 7df1bcbdec..d82a16608f 100644 --- a/codex-rs/tui/src/slash_command.rs +++ b/codex-rs/tui/src/slash_command.rs @@ -13,6 +13,7 @@ pub enum SlashCommand { // DO NOT ALPHA-SORT! Enum order is presentation order in the popup, so // more frequently used commands should be listed first. New, + Compact, Diff, Quit, #[cfg(debug_assertions)] @@ -24,6 +25,7 @@ impl SlashCommand { pub fn description(self) -> &'static str { match self { SlashCommand::New => "Start a new chat.", + SlashCommand::Compact => "Compact the chat history.", SlashCommand::Quit => "Exit the application.", SlashCommand::Diff => { "Show git diff of the working directory (including untracked files)" From b67c485d84eba0d087203a850762da0cfe3361f9 Mon Sep 17 00:00:00 2001 From: aibrahim-oai Date: Fri, 1 Aug 2025 09:17:13 -0700 Subject: [PATCH 032/100] ci fix (#1782) --- codex-rs/core/tests/compact.rs | 2 +- codex-rs/tui/src/app.rs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/codex-rs/core/tests/compact.rs b/codex-rs/core/tests/compact.rs index 8967c86a8f..bf47feed1f 100644 --- a/codex-rs/core/tests/compact.rs +++ b/codex-rs/core/tests/compact.rs @@ -4,10 +4,10 @@ use codex_core::Codex; use codex_core::CodexSpawnOk; use codex_core::ModelProviderInfo; use codex_core::built_in_model_providers; -use codex_core::exec::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR; use codex_core::protocol::EventMsg; use codex_core::protocol::InputItem; use codex_core::protocol::Op; +use codex_core::spawn::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR; use codex_login::CodexAuth; use core_test_support::load_default_config_for_test; use core_test_support::wait_for_event; diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index d4c2f3c12d..efaec7f19a 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -10,7 +10,6 @@ use crate::tui; use codex_core::config::Config; use codex_core::protocol::Event; use codex_core::protocol::EventMsg; -use codex_core::protocol::ExecApprovalRequestEvent; use codex_core::protocol::Op; use color_eyre::eyre::Result; use crossterm::SynchronizedUpdate; From 88ea215c80e4fd33dc802b31cd20f8894c60dd6b Mon Sep 17 00:00:00 2001 From: pakrym-oai Date: Fri, 1 Aug 2025 09:55:23 -0700 Subject: [PATCH 033/100] Add a custom originator setting (#1781) --- codex-rs/core/src/client.rs | 9 +++- codex-rs/core/src/config.rs | 10 +++++ codex-rs/core/src/model_provider_info.rs | 16 ++----- codex-rs/core/tests/client.rs | 57 +++++++++++++++++++++++- 4 files changed, 76 insertions(+), 16 deletions(-) diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index 0f7f51c2fc..b9ea6b13f4 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -207,7 +207,14 @@ impl ModelClient { } } - let req_builder = self.provider.apply_http_headers(req_builder); + req_builder = self.provider.apply_http_headers(req_builder); + + let originator = self + .config + .internal_originator + .as_deref() + .unwrap_or("codex_cli_rs"); + req_builder = req_builder.header("originator", originator); let res = req_builder.send().await; if let Ok(resp) = &res { diff --git a/codex-rs/core/src/config.rs b/codex-rs/core/src/config.rs index a65ec09674..1b8da89419 100644 --- a/codex-rs/core/src/config.rs +++ b/codex-rs/core/src/config.rs @@ -146,6 +146,9 @@ pub struct Config { /// Include an experimental plan tool that the model can use to update its current plan and status of each step. pub include_plan_tool: bool, + + /// The value for the `originator` header included with Responses API requests. + pub internal_originator: Option, } impl Config { @@ -336,6 +339,9 @@ pub struct ConfigToml { /// Experimental path to a file whose contents replace the built-in BASE_INSTRUCTIONS. pub experimental_instructions_file: Option, + + /// The value for the `originator` header included with Responses API requests. + pub internal_originator: Option, } impl ConfigToml { @@ -529,6 +535,7 @@ impl Config { experimental_resume, include_plan_tool: include_plan_tool.unwrap_or(false), + internal_originator: cfg.internal_originator, }; Ok(config) } @@ -887,6 +894,7 @@ disable_response_storage = true experimental_resume: None, base_instructions: None, include_plan_tool: false, + internal_originator: None, }, o3_profile_config ); @@ -936,6 +944,7 @@ disable_response_storage = true experimental_resume: None, base_instructions: None, include_plan_tool: false, + internal_originator: None, }; assert_eq!(expected_gpt3_profile_config, gpt3_profile_config); @@ -1000,6 +1009,7 @@ disable_response_storage = true experimental_resume: None, base_instructions: None, include_plan_tool: false, + internal_originator: None, }; assert_eq!(expected_zdr_profile_config, zdr_profile_config); diff --git a/codex-rs/core/src/model_provider_info.rs b/codex-rs/core/src/model_provider_info.rs index 4640f53ad7..2936637779 100644 --- a/codex-rs/core/src/model_provider_info.rs +++ b/codex-rs/core/src/model_provider_info.rs @@ -12,10 +12,6 @@ use std::env::VarError; use std::time::Duration; use crate::error::EnvVarError; - -/// Value for the `OpenAI-Originator` header that is sent with requests to -/// OpenAI. -const OPENAI_ORIGINATOR_HEADER: &str = "codex_cli_rs"; const DEFAULT_STREAM_IDLE_TIMEOUT_MS: u64 = 300_000; const DEFAULT_STREAM_MAX_RETRIES: u64 = 10; const DEFAULT_REQUEST_MAX_RETRIES: u64 = 4; @@ -229,15 +225,9 @@ pub fn built_in_model_providers() -> HashMap { wire_api: WireApi::Responses, query_params: None, http_headers: Some( - [ - ( - "originator".to_string(), - OPENAI_ORIGINATOR_HEADER.to_string(), - ), - ("version".to_string(), env!("CARGO_PKG_VERSION").to_string()), - ] - .into_iter() - .collect(), + [("version".to_string(), env!("CARGO_PKG_VERSION").to_string())] + .into_iter() + .collect(), ), env_http_headers: Some( [ diff --git a/codex-rs/core/tests/client.rs b/codex-rs/core/tests/client.rs index bfa29657d5..a22a94388b 100644 --- a/codex-rs/core/tests/client.rs +++ b/codex-rs/core/tests/client.rs @@ -95,8 +95,8 @@ async fn includes_session_id_and_model_headers_in_request() { // get request from the server let request = &server.received_requests().await.unwrap()[0]; let request_session_id = request.headers.get("session_id").unwrap(); - let request_originator = request.headers.get("originator").unwrap(); let request_authorization = request.headers.get("authorization").unwrap(); + let request_originator = request.headers.get("originator").unwrap(); assert!(current_session_id.is_some()); assert_eq!( @@ -170,6 +170,59 @@ async fn includes_base_instructions_override_in_request() { ); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn originator_config_override_is_used() { + #![allow(clippy::unwrap_used)] + + // Mock server + let server = MockServer::start().await; + + let first = ResponseTemplate::new(200) + .insert_header("content-type", "text/event-stream") + .set_body_raw(sse_completed("resp1"), "text/event-stream"); + + Mock::given(method("POST")) + .and(path("/v1/responses")) + .respond_with(first) + .expect(1) + .mount(&server) + .await; + + let model_provider = ModelProviderInfo { + base_url: Some(format!("{}/v1", server.uri())), + ..built_in_model_providers()["openai"].clone() + }; + + let codex_home = TempDir::new().unwrap(); + let mut config = load_default_config_for_test(&codex_home); + config.model_provider = model_provider; + config.internal_originator = Some("my_override".to_string()); + + let ctrl_c = std::sync::Arc::new(tokio::sync::Notify::new()); + let CodexSpawnOk { codex, .. } = Codex::spawn( + config, + Some(CodexAuth::from_api_key("Test API Key".to_string())), + ctrl_c.clone(), + ) + .await + .unwrap(); + + codex + .submit(Op::UserInput { + items: vec![InputItem::Text { + text: "hello".into(), + }], + }) + .await + .unwrap(); + + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TaskComplete(_))).await; + + let request = &server.received_requests().await.unwrap()[0]; + let request_originator = request.headers.get("originator").unwrap(); + assert_eq!(request_originator.to_str().unwrap(), "my_override"); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn chatgpt_auth_sends_correct_request() { #![allow(clippy::unwrap_used)] @@ -235,8 +288,8 @@ async fn chatgpt_auth_sends_correct_request() { // get request from the server let request = &server.received_requests().await.unwrap()[0]; let request_session_id = request.headers.get("session_id").unwrap(); - let request_originator = request.headers.get("originator").unwrap(); let request_authorization = request.headers.get("authorization").unwrap(); + let request_originator = request.headers.get("originator").unwrap(); let request_chatgpt_account_id = request.headers.get("chatgpt-account-id").unwrap(); let request_body = request.body_json::().unwrap(); From f918198bbbfa95e20c629eed758cbd53ad38ab97 Mon Sep 17 00:00:00 2001 From: aibrahim-oai Date: Fri, 1 Aug 2025 10:04:12 -0700 Subject: [PATCH 034/100] Introduce a new function to just send user message [Stack 3/3] (#1686) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - MCP server: add send-user-message tool to send user input to a running Codex session - Added an integration tests for the happy and sad paths Changes: • Add tool definition and schema. • Expose tool in capabilities. • Route and handle tool requests with validation. • Tests for success, bad UUID, and missing session. follow‑ups • Listen path not implemented yet; the tool is present but marked “don’t use yet” in code comments. • Session run flag reset: clear running_session_id_set appropriately after turn completion/errors. This is the third PR in a stack. Stack: Final: #1686 Intermediate: #1751 First: #1750 --- codex-rs/mcp-server/src/lib.rs | 3 +- codex-rs/mcp-server/src/mcp_protocol.rs | 36 ++-- codex-rs/mcp-server/src/message_processor.rs | 63 +++++-- codex-rs/mcp-server/src/tool_handlers/mod.rs | 1 + .../src/tool_handlers/send_message.rs | 124 +++++++++++++ codex-rs/mcp-server/tests/send_message.rs | 163 ++++++++++++++++++ 6 files changed, 358 insertions(+), 32 deletions(-) create mode 100644 codex-rs/mcp-server/src/tool_handlers/mod.rs create mode 100644 codex-rs/mcp-server/src/tool_handlers/send_message.rs create mode 100644 codex-rs/mcp-server/tests/send_message.rs diff --git a/codex-rs/mcp-server/src/lib.rs b/codex-rs/mcp-server/src/lib.rs index ebef8ca903..6b3c0ddbe6 100644 --- a/codex-rs/mcp-server/src/lib.rs +++ b/codex-rs/mcp-server/src/lib.rs @@ -20,9 +20,10 @@ mod codex_tool_runner; mod exec_approval; mod json_to_toml; pub mod mcp_protocol; -mod message_processor; +pub(crate) mod message_processor; mod outgoing_message; mod patch_approval; +pub(crate) mod tool_handlers; use crate::message_processor::MessageProcessor; use crate::outgoing_message::OutgoingMessage; diff --git a/codex-rs/mcp-server/src/mcp_protocol.rs b/codex-rs/mcp-server/src/mcp_protocol.rs index 23304dc43d..287890bf5e 100644 --- a/codex-rs/mcp-server/src/mcp_protocol.rs +++ b/codex-rs/mcp-server/src/mcp_protocol.rs @@ -132,32 +132,32 @@ impl From for CallToolResult { is_error, result, } = val; - let (content, structured_content, is_error_out) = match result { + match result { Some(res) => match serde_json::to_value(&res) { - Ok(v) => { - let content = vec![ContentBlock::TextContent(TextContent { + Ok(v) => CallToolResult { + content: vec![ContentBlock::TextContent(TextContent { r#type: "text".to_string(), text: v.to_string(), annotations: None, - })]; - (content, Some(v), is_error) - } - Err(e) => { - let content = vec![ContentBlock::TextContent(TextContent { + })], + is_error, + structured_content: Some(v), + }, + Err(e) => CallToolResult { + content: vec![ContentBlock::TextContent(TextContent { r#type: "text".to_string(), text: format!("Failed to serialize tool result: {e}"), annotations: None, - })]; - (content, None, Some(true)) - } + })], + is_error: Some(true), + structured_content: None, + }, + }, + None => CallToolResult { + content: vec![], + is_error, + structured_content: None, }, - None => (vec![], None, is_error), - }; - - CallToolResult { - content, - is_error: is_error_out, - structured_content, } } } diff --git a/codex-rs/mcp-server/src/message_processor.rs b/codex-rs/mcp-server/src/message_processor.rs index a4013cc3a3..14e9bb3887 100644 --- a/codex-rs/mcp-server/src/message_processor.rs +++ b/codex-rs/mcp-server/src/message_processor.rs @@ -1,4 +1,5 @@ use std::collections::HashMap; +use std::collections::HashSet; use std::path::PathBuf; use std::sync::Arc; @@ -7,11 +8,15 @@ use crate::codex_tool_config::CodexToolCallReplyParam; use crate::codex_tool_config::create_tool_for_codex_tool_call_param; use crate::codex_tool_config::create_tool_for_codex_tool_call_reply_param; use crate::mcp_protocol::ToolCallRequestParams; +use crate::mcp_protocol::ToolCallResponse; +use crate::mcp_protocol::ToolCallResponseResult; use crate::outgoing_message::OutgoingMessageSender; +use crate::tool_handlers::send_message::handle_send_message; use codex_core::Codex; use codex_core::config::Config as CodexConfig; use codex_core::protocol::Submission; +use mcp_types::CallToolRequest; use mcp_types::CallToolRequestParams; use mcp_types::CallToolResult; use mcp_types::ClientRequest; @@ -38,6 +43,7 @@ pub(crate) struct MessageProcessor { codex_linux_sandbox_exe: Option, session_map: Arc>>>, running_requests_id_to_codex_uuid: Arc>>, + running_session_ids: Arc>>, } impl MessageProcessor { @@ -53,9 +59,18 @@ impl MessageProcessor { codex_linux_sandbox_exe, session_map: Arc::new(Mutex::new(HashMap::new())), running_requests_id_to_codex_uuid: Arc::new(Mutex::new(HashMap::new())), + running_session_ids: Arc::new(Mutex::new(HashSet::new())), } } + pub(crate) fn session_map(&self) -> Arc>>> { + self.session_map.clone() + } + + pub(crate) fn running_session_ids(&self) -> Arc>> { + self.running_session_ids.clone() + } + pub(crate) async fn process_request(&mut self, request: JSONRPCRequest) { // Hold on to the ID so we can respond. let request_id = request.id.clone(); @@ -332,19 +347,25 @@ impl MessageProcessor { } } } - async fn handle_new_tool_calls(&self, request_id: RequestId, _params: ToolCallRequestParams) { - // TODO: implement the new tool calls - let result = CallToolResult { - content: vec![ContentBlock::TextContent(TextContent { - r#type: "text".to_string(), - text: "Unknown tool".to_string(), - annotations: None, - })], - is_error: Some(true), - structured_content: None, - }; - self.send_response::(request_id, result) - .await; + async fn handle_new_tool_calls(&self, request_id: RequestId, params: ToolCallRequestParams) { + match params { + ToolCallRequestParams::ConversationSendMessage(args) => { + handle_send_message(self, request_id, args).await; + } + _ => { + let result = CallToolResult { + content: vec![ContentBlock::TextContent(TextContent { + r#type: "text".to_string(), + text: "Unknown tool".to_string(), + annotations: None, + })], + is_error: Some(true), + structured_content: None, + }; + self.send_response::(request_id, result) + .await; + } + } } async fn handle_tool_call_codex(&self, id: RequestId, arguments: Option) { @@ -654,4 +675,20 @@ impl MessageProcessor { ) { tracing::info!("notifications/message -> params: {:?}", params); } + + pub(crate) async fn send_response_with_optional_error( + &self, + id: RequestId, + message: Option, + error: Option, + ) { + let response = ToolCallResponse { + request_id: id.clone(), + is_error: error, + result: message, + }; + let result: CallToolResult = response.into(); + self.send_response::(id.clone(), result) + .await; + } } diff --git a/codex-rs/mcp-server/src/tool_handlers/mod.rs b/codex-rs/mcp-server/src/tool_handlers/mod.rs new file mode 100644 index 0000000000..1907ec64e0 --- /dev/null +++ b/codex-rs/mcp-server/src/tool_handlers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod send_message; diff --git a/codex-rs/mcp-server/src/tool_handlers/send_message.rs b/codex-rs/mcp-server/src/tool_handlers/send_message.rs new file mode 100644 index 0000000000..894176bef6 --- /dev/null +++ b/codex-rs/mcp-server/src/tool_handlers/send_message.rs @@ -0,0 +1,124 @@ +use std::collections::HashMap; +use std::sync::Arc; + +use codex_core::Codex; +use codex_core::protocol::Op; +use codex_core::protocol::Submission; +use mcp_types::RequestId; +use tokio::sync::Mutex; +use uuid::Uuid; + +use crate::mcp_protocol::ConversationSendMessageArgs; +use crate::mcp_protocol::ConversationSendMessageResult; +use crate::mcp_protocol::ToolCallResponseResult; +use crate::message_processor::MessageProcessor; + +pub(crate) async fn handle_send_message( + message_processor: &MessageProcessor, + id: RequestId, + arguments: ConversationSendMessageArgs, +) { + let ConversationSendMessageArgs { + conversation_id, + content: items, + parent_message_id: _, + conversation_overrides: _, + } = arguments; + + if items.is_empty() { + message_processor + .send_response_with_optional_error( + id, + Some(ToolCallResponseResult::ConversationSendMessage( + ConversationSendMessageResult::Error { + message: "No content items provided".to_string(), + }, + )), + Some(true), + ) + .await; + return; + } + + let session_id = conversation_id.0; + let Some(codex) = get_session(session_id, message_processor.session_map()).await else { + message_processor + .send_response_with_optional_error( + id, + Some(ToolCallResponseResult::ConversationSendMessage( + ConversationSendMessageResult::Error { + message: "Session does not exist".to_string(), + }, + )), + Some(true), + ) + .await; + return; + }; + + let running = { + let running_sessions = message_processor.running_session_ids(); + let mut running_sessions = running_sessions.lock().await; + !running_sessions.insert(session_id) + }; + + if running { + message_processor + .send_response_with_optional_error( + id, + Some(ToolCallResponseResult::ConversationSendMessage( + ConversationSendMessageResult::Error { + message: "Session is already running".to_string(), + }, + )), + Some(true), + ) + .await; + return; + } + + let request_id_string = match &id { + RequestId::String(s) => s.clone(), + RequestId::Integer(i) => i.to_string(), + }; + + let submit_res = codex + .submit_with_id(Submission { + id: request_id_string, + op: Op::UserInput { items }, + }) + .await; + + if let Err(e) = submit_res { + message_processor + .send_response_with_optional_error( + id, + Some(ToolCallResponseResult::ConversationSendMessage( + ConversationSendMessageResult::Error { + message: format!("Failed to submit user input: {e}"), + }, + )), + Some(true), + ) + .await; + return; + } + + message_processor + .send_response_with_optional_error( + id, + Some(ToolCallResponseResult::ConversationSendMessage( + ConversationSendMessageResult::Ok, + )), + Some(false), + ) + .await; +} + +pub(crate) async fn get_session( + session_id: Uuid, + session_map: Arc>>>, +) -> Option> { + let guard = session_map.lock().await; + guard.get(&session_id).cloned() +} diff --git a/codex-rs/mcp-server/tests/send_message.rs b/codex-rs/mcp-server/tests/send_message.rs new file mode 100644 index 0000000000..fd4b210b0b --- /dev/null +++ b/codex-rs/mcp-server/tests/send_message.rs @@ -0,0 +1,163 @@ +#![allow(clippy::expect_used)] + +use std::path::Path; +use std::thread::sleep; +use std::time::Duration; + +use codex_mcp_server::CodexToolCallParam; +use mcp_test_support::McpProcess; +use mcp_test_support::create_final_assistant_message_sse_response; +use mcp_test_support::create_mock_chat_completions_server; +use mcp_types::JSONRPC_VERSION; +use mcp_types::JSONRPCResponse; +use mcp_types::RequestId; +use pretty_assertions::assert_eq; +use serde_json::json; +use tempfile::TempDir; +use tokio::time::timeout; + +const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_send_message_success() { + // Spin up a mock completions server that immediately ends the Codex turn. + // Two Codex turns hit the mock model (session start + send-user-message). Provide two SSE responses. + let responses = vec![ + create_final_assistant_message_sse_response("Done").expect("build mock assistant message"), + create_final_assistant_message_sse_response("Done").expect("build mock assistant message"), + ]; + let server = create_mock_chat_completions_server(responses).await; + + // Create a temporary Codex home with config pointing at the mock server. + let codex_home = TempDir::new().expect("create temp dir"); + create_config_toml(codex_home.path(), &server.uri()).expect("write config.toml"); + + // Start MCP server process and initialize. + let mut mcp_process = McpProcess::new(codex_home.path()) + .await + .expect("spawn mcp process"); + timeout(DEFAULT_READ_TIMEOUT, mcp_process.initialize()) + .await + .expect("init timed out") + .expect("init failed"); + + // Kick off a Codex session so we have a valid session_id. + let codex_request_id = mcp_process + .send_codex_tool_call(CodexToolCallParam { + prompt: "Start a session".to_string(), + ..Default::default() + }) + .await + .expect("send codex tool call"); + + // Wait for the session_configured event to get the session_id. + let session_id = mcp_process + .read_stream_until_configured_response_message() + .await + .expect("read session_configured"); + + // The original codex call will finish quickly given our mock; consume its response. + timeout( + DEFAULT_READ_TIMEOUT, + mcp_process.read_stream_until_response_message(RequestId::Integer(codex_request_id)), + ) + .await + .expect("codex response timeout") + .expect("codex response error"); + + // Now exercise the send-user-message tool. + let send_msg_request_id = mcp_process + .send_user_message_tool_call("Hello again", &session_id) + .await + .expect("send send-message tool call"); + + let response: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp_process.read_stream_until_response_message(RequestId::Integer(send_msg_request_id)), + ) + .await + .expect("send-user-message response timeout") + .expect("send-user-message response error"); + + assert_eq!( + JSONRPCResponse { + jsonrpc: JSONRPC_VERSION.into(), + id: RequestId::Integer(send_msg_request_id), + result: json!({ + "content": [ + { + "text": "{\"status\":\"ok\"}", + "type": "text", + } + ], + "isError": false, + "structuredContent": { + "status": "ok" + } + }), + }, + response + ); + // wait for the server to hear the user message + sleep(Duration::from_secs(1)); + + // Ensure the server and tempdir live until end of test + drop(server); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_send_message_session_not_found() { + // Start MCP without creating a Codex session + let codex_home = TempDir::new().expect("tempdir"); + let mut mcp = McpProcess::new(codex_home.path()).await.expect("spawn"); + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()) + .await + .expect("timeout") + .expect("init"); + + let unknown = uuid::Uuid::new_v4().to_string(); + let req_id = mcp + .send_user_message_tool_call("ping", &unknown) + .await + .expect("send tool"); + + let resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(req_id)), + ) + .await + .expect("timeout") + .expect("resp"); + + let result = resp.result.clone(); + let content = result["content"][0]["text"].as_str().unwrap_or(""); + assert!(content.contains("Session does not exist")); + assert_eq!(result["isError"], json!(true)); +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +fn create_config_toml(codex_home: &Path, server_uri: &str) -> std::io::Result<()> { + let config_toml = codex_home.join("config.toml"); + std::fs::write( + config_toml, + format!( + r#" +model = "mock-model" +approval_policy = "never" +sandbox_mode = "danger-full-access" + +model_provider = "mock_provider" + +[model_providers.mock_provider] +name = "Mock provider for test" +base_url = "{server_uri}/v1" +wire_api = "chat" +request_max_retries = 0 +stream_max_retries = 0 +"# + ), + ) +} From 8360c6a3ec83d0aa5d208b7a7eeb619c4ce27cad Mon Sep 17 00:00:00 2001 From: Jeremy Rose <172423086+nornagon-openai@users.noreply.github.com> Date: Fri, 1 Aug 2025 10:37:43 -0700 Subject: [PATCH 035/100] fix insert_history modifier handling (#1774) This fixes a bug in insert_history_lines where writing `Line::From(vec!["A".bold(), "B".into()])` would write "B" as bold, because "B" didn't explicitly subtract bold. --- codex-rs/tui/src/history_cell.rs | 27 ++++++---------- codex-rs/tui/src/insert_history.rs | 50 +++++++++++++++++++++++++----- 2 files changed, 52 insertions(+), 25 deletions(-) diff --git a/codex-rs/tui/src/history_cell.rs b/codex-rs/tui/src/history_cell.rs index 0e9f25332d..c2aafdd522 100644 --- a/codex-rs/tui/src/history_cell.rs +++ b/codex-rs/tui/src/history_cell.rs @@ -545,24 +545,17 @@ impl HistoryCell { } else { for (idx, PlanItemArg { step, status }) in plan.into_iter().enumerate() { let num = idx + 1; - let (icon, style): (&str, Style) = match status { - StepStatus::Completed => ("✓", Style::default().fg(Color::Green)), - StepStatus::InProgress => ( - "▶", - Style::default() - .fg(Color::Yellow) - .add_modifier(Modifier::BOLD), - ), - StepStatus::Pending => ("○", Style::default().fg(Color::Gray)), + let icon_span: Span = match status { + StepStatus::Completed => Span::from("✓").fg(Color::Green), + StepStatus::InProgress => Span::from("▶").fg(Color::Yellow).bold(), + StepStatus::Pending => Span::from("○").fg(Color::Gray), }; - let prefix = vec![ - Span::raw(format!("{num:>2}. [")), - Span::styled(icon.to_string(), style), - Span::raw("] "), - ]; - let mut spans = prefix; - spans.push(Span::raw(step)); - lines.push(Line::from(spans)); + lines.push(Line::from(vec![ + format!("{num:>2}. [").into(), + icon_span, + "] ".into(), + step.into(), + ])); } } diff --git a/codex-rs/tui/src/insert_history.rs b/codex-rs/tui/src/insert_history.rs index efd08a71c2..87d88b7f03 100644 --- a/codex-rs/tui/src/insert_history.rs +++ b/codex-rs/tui/src/insert_history.rs @@ -216,18 +216,18 @@ where { let mut fg = Color::Reset; let mut bg = Color::Reset; - let mut modifier = Modifier::empty(); + let mut last_modifier = Modifier::empty(); for span in content { - let mut next_modifier = modifier; - next_modifier.insert(span.style.add_modifier); - next_modifier.remove(span.style.sub_modifier); - if next_modifier != modifier { + let mut modifier = Modifier::empty(); + modifier.insert(span.style.add_modifier); + modifier.remove(span.style.sub_modifier); + if modifier != last_modifier { let diff = ModifierDiff { - from: modifier, - to: next_modifier, + from: last_modifier, + to: modifier, }; diff.queue(&mut writer)?; - modifier = next_modifier; + last_modifier = modifier; } let next_fg = span.style.fg.unwrap_or(Color::Reset); let next_bg = span.style.bg.unwrap_or(Color::Reset); @@ -250,3 +250,37 @@ where SetAttribute(crossterm::style::Attribute::Reset), ) } + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used)] + use super::*; + + #[test] + fn writes_bold_then_regular_spans() { + use ratatui::style::Stylize; + + let spans = ["A".bold(), "B".into()]; + + let mut actual: Vec = Vec::new(); + write_spans(&mut actual, spans.iter()).unwrap(); + + let mut expected: Vec = Vec::new(); + queue!( + expected, + SetAttribute(crossterm::style::Attribute::Bold), + Print("A"), + SetAttribute(crossterm::style::Attribute::NormalIntensity), + Print("B"), + SetForegroundColor(CColor::Reset), + SetBackgroundColor(CColor::Reset), + SetAttribute(crossterm::style::Attribute::Reset), + ) + .unwrap(); + + assert_eq!( + String::from_utf8(actual).unwrap(), + String::from_utf8(expected).unwrap() + ); + } +} From bc7beddaa2d9fab9cea376c4a6d8a26bfac3b349 Mon Sep 17 00:00:00 2001 From: aibrahim-oai Date: Fri, 1 Aug 2025 13:04:34 -0700 Subject: [PATCH 036/100] feat: stream exec stdout events (#1786) ## Summary - stream command stdout as `ExecCommandStdout` events - forward streamed stdout to clients and ignore in human output processor - adjust call sites for new streaming API --- codex-rs/Cargo.lock | 10 ++ codex-rs/core/Cargo.toml | 1 + codex-rs/core/src/codex.rs | 11 ++ codex-rs/core/src/exec.rs | 53 ++++++- codex-rs/core/src/protocol.rs | 25 ++++ codex-rs/core/src/shell.rs | 1 + codex-rs/core/tests/exec_stream_events.rs | 135 ++++++++++++++++++ .../src/event_processor_with_human_output.rs | 2 + codex-rs/linux-sandbox/tests/landlock.rs | 2 + codex-rs/mcp-server/src/codex_tool_runner.rs | 2 + codex-rs/tui/src/chatwidget.rs | 1 + 11 files changed, 238 insertions(+), 5 deletions(-) create mode 100644 codex-rs/core/tests/exec_stream_events.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 9123bd6374..460a440e63 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -695,6 +695,7 @@ dependencies = [ "reqwest", "seccompiler", "serde", + "serde_bytes", "serde_json", "sha1", "shlex", @@ -3951,6 +3952,15 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_bytes" +version = "0.11.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8437fd221bde2d4ca316d61b90e337e9e702b3820b87d63caa9ba6c02bd06d96" +dependencies = [ + "serde", +] + [[package]] name = "serde_derive" version = "1.0.219" diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index ecc904cd5e..db3fd4f834 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -31,6 +31,7 @@ rand = "0.9" reqwest = { version = "0.12", features = ["json", "stream"] } serde = { version = "1", features = ["derive"] } serde_json = "1" +serde_bytes = "0.11" sha1 = "0.10.6" shlex = "1.3.0" strum_macros = "0.27.2" diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 5d48cd457f..6e21642da1 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -48,6 +48,7 @@ use crate::error::SandboxErr; use crate::exec::ExecParams; use crate::exec::ExecToolCallOutput; use crate::exec::SandboxType; +use crate::exec::StdoutStream; use crate::exec::process_exec_tool_call; use crate::exec_env::create_env; use crate::mcp_connection_manager::McpConnectionManager; @@ -1759,6 +1760,11 @@ async fn handle_container_exec_with_params( sess.ctrl_c.clone(), &sess.sandbox_policy, &sess.codex_linux_sandbox_exe, + Some(StdoutStream { + sub_id: sub_id.clone(), + call_id: call_id.clone(), + tx_event: sess.tx_event.clone(), + }), ) .await; @@ -1879,6 +1885,11 @@ async fn handle_sandbox_error( sess.ctrl_c.clone(), &sess.sandbox_policy, &sess.codex_linux_sandbox_exe, + Some(StdoutStream { + sub_id: sub_id.clone(), + call_id: call_id.clone(), + tx_event: sess.tx_event.clone(), + }), ) .await; diff --git a/codex-rs/core/src/exec.rs b/codex-rs/core/src/exec.rs index 06416e6768..a5a88ee9c0 100644 --- a/codex-rs/core/src/exec.rs +++ b/codex-rs/core/src/exec.rs @@ -10,6 +10,7 @@ use std::sync::Arc; use std::time::Duration; use std::time::Instant; +use async_channel::Sender; use tokio::io::AsyncRead; use tokio::io::AsyncReadExt; use tokio::io::BufReader; @@ -19,10 +20,15 @@ use tokio::sync::Notify; use crate::error::CodexErr; use crate::error::Result; use crate::error::SandboxErr; +use crate::protocol::Event; +use crate::protocol::EventMsg; +use crate::protocol::ExecCommandStderrDeltaEvent; +use crate::protocol::ExecCommandStdoutDeltaEvent; use crate::protocol::SandboxPolicy; use crate::seatbelt::spawn_command_under_seatbelt; use crate::spawn::StdioPolicy; use crate::spawn::spawn_child_async; +use serde_bytes::ByteBuf; // Maximum we send for each stream, which is either: // - 10KiB OR @@ -56,18 +62,26 @@ pub enum SandboxType { LinuxSeccomp, } +#[derive(Clone)] +pub struct StdoutStream { + pub sub_id: String, + pub call_id: String, + pub tx_event: Sender, +} + pub async fn process_exec_tool_call( params: ExecParams, sandbox_type: SandboxType, ctrl_c: Arc, sandbox_policy: &SandboxPolicy, codex_linux_sandbox_exe: &Option, + stdout_stream: Option, ) -> Result { let start = Instant::now(); let raw_output_result: std::result::Result = match sandbox_type { - SandboxType::None => exec(params, sandbox_policy, ctrl_c).await, + SandboxType::None => exec(params, sandbox_policy, ctrl_c, stdout_stream.clone()).await, SandboxType::MacosSeatbelt => { let ExecParams { command, @@ -83,7 +97,7 @@ pub async fn process_exec_tool_call( env, ) .await?; - consume_truncated_output(child, ctrl_c, timeout_ms).await + consume_truncated_output(child, ctrl_c, timeout_ms, stdout_stream.clone()).await } SandboxType::LinuxSeccomp => { let ExecParams { @@ -106,7 +120,7 @@ pub async fn process_exec_tool_call( ) .await?; - consume_truncated_output(child, ctrl_c, timeout_ms).await + consume_truncated_output(child, ctrl_c, timeout_ms, stdout_stream).await } }; let duration = start.elapsed(); @@ -233,6 +247,7 @@ async fn exec( }: ExecParams, sandbox_policy: &SandboxPolicy, ctrl_c: Arc, + stdout_stream: Option, ) -> Result { let (program, args) = command.split_first().ok_or_else(|| { CodexErr::Io(io::Error::new( @@ -251,7 +266,7 @@ async fn exec( env, ) .await?; - consume_truncated_output(child, ctrl_c, timeout_ms).await + consume_truncated_output(child, ctrl_c, timeout_ms, stdout_stream).await } /// Consumes the output of a child process, truncating it so it is suitable for @@ -260,6 +275,7 @@ pub(crate) async fn consume_truncated_output( mut child: Child, ctrl_c: Arc, timeout_ms: Option, + stdout_stream: Option, ) -> Result { // Both stdout and stderr were configured with `Stdio::piped()` // above, therefore `take()` should normally return `Some`. If it doesn't @@ -280,11 +296,15 @@ pub(crate) async fn consume_truncated_output( BufReader::new(stdout_reader), MAX_STREAM_OUTPUT, MAX_STREAM_OUTPUT_LINES, + stdout_stream.clone(), + false, )); let stderr_handle = tokio::spawn(read_capped( BufReader::new(stderr_reader), MAX_STREAM_OUTPUT, MAX_STREAM_OUTPUT_LINES, + stdout_stream.clone(), + true, )); let interrupted = ctrl_c.notified(); @@ -318,10 +338,12 @@ pub(crate) async fn consume_truncated_output( }) } -async fn read_capped( +async fn read_capped( mut reader: R, max_output: usize, max_lines: usize, + stream: Option, + is_stderr: bool, ) -> io::Result> { let mut buf = Vec::with_capacity(max_output.min(8 * 1024)); let mut tmp = [0u8; 8192]; @@ -335,6 +357,27 @@ async fn read_capped( break; } + if let Some(stream) = &stream { + let chunk = tmp[..n].to_vec(); + let msg = if is_stderr { + EventMsg::ExecCommandStderrDelta(ExecCommandStderrDeltaEvent { + call_id: stream.call_id.clone(), + chunk: ByteBuf::from(chunk), + }) + } else { + EventMsg::ExecCommandStdoutDelta(ExecCommandStdoutDeltaEvent { + call_id: stream.call_id.clone(), + chunk: ByteBuf::from(chunk), + }) + }; + let event = Event { + id: stream.sub_id.clone(), + msg, + }; + #[allow(clippy::let_unit_value)] + let _ = stream.tx_event.send(event).await; + } + // Copy into the buffer only while we still have byte and line budget. if remaining_bytes > 0 && remaining_lines > 0 { let mut copy_len = 0; diff --git a/codex-rs/core/src/protocol.rs b/codex-rs/core/src/protocol.rs index 1b4832a273..8d26eda98e 100644 --- a/codex-rs/core/src/protocol.rs +++ b/codex-rs/core/src/protocol.rs @@ -13,6 +13,7 @@ use std::time::Duration; use mcp_types::CallToolResult; use serde::Deserialize; use serde::Serialize; +use serde_bytes::ByteBuf; use strum_macros::Display; use uuid::Uuid; @@ -323,6 +324,12 @@ pub enum EventMsg { /// Notification that the server is about to execute a command. ExecCommandBegin(ExecCommandBeginEvent), + /// Incremental chunk of stdout from a running command. + ExecCommandStdoutDelta(ExecCommandStdoutDeltaEvent), + + /// Incremental chunk of stderr from a running command. + ExecCommandStderrDelta(ExecCommandStderrDeltaEvent), + ExecCommandEnd(ExecCommandEndEvent), ExecApprovalRequest(ExecApprovalRequestEvent), @@ -476,6 +483,24 @@ pub struct ExecCommandEndEvent { pub exit_code: i32, } +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ExecCommandStdoutDeltaEvent { + /// Identifier for the ExecCommandBegin that produced this chunk. + pub call_id: String, + /// Raw stdout bytes (may not be valid UTF-8). + #[serde(with = "serde_bytes")] + pub chunk: ByteBuf, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ExecCommandStderrDeltaEvent { + /// Identifier for the ExecCommandBegin that produced this chunk. + pub call_id: String, + /// Raw stderr bytes (may not be valid UTF-8). + #[serde(with = "serde_bytes")] + pub chunk: ByteBuf, +} + #[derive(Debug, Clone, Deserialize, Serialize)] pub struct ExecApprovalRequestEvent { /// Identifier for the associated exec call, if available. diff --git a/codex-rs/core/src/shell.rs b/codex-rs/core/src/shell.rs index 98addffce2..1e895a3701 100644 --- a/codex-rs/core/src/shell.rs +++ b/codex-rs/core/src/shell.rs @@ -220,6 +220,7 @@ mod tests { Arc::new(Notify::new()), &SandboxPolicy::DangerFullAccess, &None, + None, ) .await .unwrap(); diff --git a/codex-rs/core/tests/exec_stream_events.rs b/codex-rs/core/tests/exec_stream_events.rs new file mode 100644 index 0000000000..059b69a064 --- /dev/null +++ b/codex-rs/core/tests/exec_stream_events.rs @@ -0,0 +1,135 @@ +#![cfg(unix)] + +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::Arc; + +use async_channel::Receiver; +use codex_core::exec::ExecParams; +use codex_core::exec::SandboxType; +use codex_core::exec::StdoutStream; +use codex_core::exec::process_exec_tool_call; +use codex_core::protocol::Event; +use codex_core::protocol::EventMsg; +use codex_core::protocol::ExecCommandStderrDeltaEvent; +use codex_core::protocol::ExecCommandStdoutDeltaEvent; +use codex_core::protocol::SandboxPolicy; +use tokio::sync::Notify; + +fn collect_stdout_events(rx: Receiver) -> Vec { + let mut out = Vec::new(); + while let Ok(ev) = rx.try_recv() { + if let EventMsg::ExecCommandStdoutDelta(ExecCommandStdoutDeltaEvent { chunk, .. }) = ev.msg + { + out.extend_from_slice(&chunk); + } + } + out +} + +#[tokio::test] +async fn test_exec_stdout_stream_events_echo() { + let (tx, rx) = async_channel::unbounded::(); + + let stdout_stream = StdoutStream { + sub_id: "test-sub".to_string(), + call_id: "call-1".to_string(), + tx_event: tx, + }; + + let cmd = vec![ + "/bin/sh".to_string(), + "-c".to_string(), + // Use printf for predictable behavior across shells + "printf 'hello-world\n'".to_string(), + ]; + + let params = ExecParams { + command: cmd, + cwd: std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")), + timeout_ms: Some(5_000), + env: HashMap::new(), + }; + + let ctrl_c = Arc::new(Notify::new()); + let policy = SandboxPolicy::new_read_only_policy(); + + let result = process_exec_tool_call( + params, + SandboxType::None, + ctrl_c, + &policy, + &None, + Some(stdout_stream), + ) + .await; + + let result = match result { + Ok(r) => r, + Err(e) => panic!("process_exec_tool_call failed: {e}"), + }; + + assert_eq!(result.exit_code, 0); + assert_eq!(result.stdout, "hello-world\n"); + + let streamed = collect_stdout_events(rx); + // We should have received at least the same contents (possibly in one chunk) + assert_eq!(String::from_utf8_lossy(&streamed), "hello-world\n"); +} + +#[tokio::test] +async fn test_exec_stderr_stream_events_echo() { + let (tx, rx) = async_channel::unbounded::(); + + let stdout_stream = StdoutStream { + sub_id: "test-sub".to_string(), + call_id: "call-2".to_string(), + tx_event: tx, + }; + + let cmd = vec![ + "/bin/sh".to_string(), + "-c".to_string(), + // Write to stderr explicitly + "printf 'oops\n' 1>&2".to_string(), + ]; + + let params = ExecParams { + command: cmd, + cwd: std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")), + timeout_ms: Some(5_000), + env: HashMap::new(), + }; + + let ctrl_c = Arc::new(Notify::new()); + let policy = SandboxPolicy::new_read_only_policy(); + + let result = process_exec_tool_call( + params, + SandboxType::None, + ctrl_c, + &policy, + &None, + Some(stdout_stream), + ) + .await; + + let result = match result { + Ok(r) => r, + Err(e) => panic!("process_exec_tool_call failed: {e}"), + }; + + assert_eq!(result.exit_code, 0); + assert_eq!(result.stdout, ""); + assert_eq!(result.stderr, "oops\n"); + + // Collect only stderr delta events + let mut err = Vec::new(); + while let Ok(ev) = rx.try_recv() { + if let EventMsg::ExecCommandStderrDelta(ExecCommandStderrDeltaEvent { chunk, .. }) = ev.msg + { + err.extend_from_slice(&chunk); + } + } + assert_eq!(String::from_utf8_lossy(&err), "oops\n"); +} diff --git a/codex-rs/exec/src/event_processor_with_human_output.rs b/codex-rs/exec/src/event_processor_with_human_output.rs index 54604d538b..05643c6282 100644 --- a/codex-rs/exec/src/event_processor_with_human_output.rs +++ b/codex-rs/exec/src/event_processor_with_human_output.rs @@ -239,6 +239,8 @@ impl EventProcessor for EventProcessorWithHumanOutput { cwd.to_string_lossy(), ); } + EventMsg::ExecCommandStdoutDelta(_) => {} + EventMsg::ExecCommandStderrDelta(_) => {} EventMsg::ExecCommandEnd(ExecCommandEndEvent { call_id, stdout, diff --git a/codex-rs/linux-sandbox/tests/landlock.rs b/codex-rs/linux-sandbox/tests/landlock.rs index 7eacda46c1..c16e9227df 100644 --- a/codex-rs/linux-sandbox/tests/landlock.rs +++ b/codex-rs/linux-sandbox/tests/landlock.rs @@ -59,6 +59,7 @@ async fn run_cmd(cmd: &[&str], writable_roots: &[PathBuf], timeout_ms: u64) { ctrl_c, &sandbox_policy, &codex_linux_sandbox_exe, + None, ) .await .unwrap(); @@ -149,6 +150,7 @@ async fn assert_network_blocked(cmd: &[&str]) { ctrl_c, &sandbox_policy, &codex_linux_sandbox_exe, + None, ) .await; diff --git a/codex-rs/mcp-server/src/codex_tool_runner.rs b/codex-rs/mcp-server/src/codex_tool_runner.rs index f25659b284..94d47f7497 100644 --- a/codex-rs/mcp-server/src/codex_tool_runner.rs +++ b/codex-rs/mcp-server/src/codex_tool_runner.rs @@ -258,6 +258,8 @@ async fn run_codex_tool_session_inner( | EventMsg::McpToolCallBegin(_) | EventMsg::McpToolCallEnd(_) | EventMsg::ExecCommandBegin(_) + | EventMsg::ExecCommandStdoutDelta(_) + | EventMsg::ExecCommandStderrDelta(_) | EventMsg::ExecCommandEnd(_) | EventMsg::BackgroundEvent(_) | EventMsg::PatchApplyBegin(_) diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index c777af5264..d6454c82a7 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -374,6 +374,7 @@ impl ChatWidget<'_> { ); self.add_to_history(HistoryCell::new_active_exec_command(command)); } + EventMsg::ExecCommandStdoutDelta(_) => {} EventMsg::PatchApplyBegin(PatchApplyBeginEvent { call_id: _, auto_approved, From f20de21cb608feef4fe295f434bf22498821bdcd Mon Sep 17 00:00:00 2001 From: aibrahim-oai Date: Fri, 1 Aug 2025 14:00:19 -0700 Subject: [PATCH 037/100] collabse `stdout` and `stderr` delta events into one (#1787) --- codex-rs/core/src/exec.rs | 24 +++++++++---------- codex-rs/core/src/protocol.rs | 23 ++++++++---------- codex-rs/core/tests/exec_stream_events.rs | 16 +++++++++---- .../src/event_processor_with_human_output.rs | 3 +-- codex-rs/mcp-server/src/codex_tool_runner.rs | 3 +-- codex-rs/tui/src/chatwidget.rs | 2 +- 6 files changed, 36 insertions(+), 35 deletions(-) diff --git a/codex-rs/core/src/exec.rs b/codex-rs/core/src/exec.rs index a5a88ee9c0..5301f0220d 100644 --- a/codex-rs/core/src/exec.rs +++ b/codex-rs/core/src/exec.rs @@ -22,8 +22,8 @@ use crate::error::Result; use crate::error::SandboxErr; use crate::protocol::Event; use crate::protocol::EventMsg; -use crate::protocol::ExecCommandStderrDeltaEvent; -use crate::protocol::ExecCommandStdoutDeltaEvent; +use crate::protocol::ExecCommandOutputDeltaEvent; +use crate::protocol::ExecOutputStream; use crate::protocol::SandboxPolicy; use crate::seatbelt::spawn_command_under_seatbelt; use crate::spawn::StdioPolicy; @@ -359,17 +359,15 @@ async fn read_capped( if let Some(stream) = &stream { let chunk = tmp[..n].to_vec(); - let msg = if is_stderr { - EventMsg::ExecCommandStderrDelta(ExecCommandStderrDeltaEvent { - call_id: stream.call_id.clone(), - chunk: ByteBuf::from(chunk), - }) - } else { - EventMsg::ExecCommandStdoutDelta(ExecCommandStdoutDeltaEvent { - call_id: stream.call_id.clone(), - chunk: ByteBuf::from(chunk), - }) - }; + let msg = EventMsg::ExecCommandOutputDelta(ExecCommandOutputDeltaEvent { + call_id: stream.call_id.clone(), + stream: if is_stderr { + ExecOutputStream::Stderr + } else { + ExecOutputStream::Stdout + }, + chunk: ByteBuf::from(chunk), + }); let event = Event { id: stream.sub_id.clone(), msg, diff --git a/codex-rs/core/src/protocol.rs b/codex-rs/core/src/protocol.rs index 8d26eda98e..3be0c92ceb 100644 --- a/codex-rs/core/src/protocol.rs +++ b/codex-rs/core/src/protocol.rs @@ -324,11 +324,8 @@ pub enum EventMsg { /// Notification that the server is about to execute a command. ExecCommandBegin(ExecCommandBeginEvent), - /// Incremental chunk of stdout from a running command. - ExecCommandStdoutDelta(ExecCommandStdoutDeltaEvent), - - /// Incremental chunk of stderr from a running command. - ExecCommandStderrDelta(ExecCommandStderrDeltaEvent), + /// Incremental chunk of output from a running command. + ExecCommandOutputDelta(ExecCommandOutputDeltaEvent), ExecCommandEnd(ExecCommandEndEvent), @@ -484,19 +481,19 @@ pub struct ExecCommandEndEvent { } #[derive(Debug, Clone, Deserialize, Serialize)] -pub struct ExecCommandStdoutDeltaEvent { - /// Identifier for the ExecCommandBegin that produced this chunk. - pub call_id: String, - /// Raw stdout bytes (may not be valid UTF-8). - #[serde(with = "serde_bytes")] - pub chunk: ByteBuf, +#[serde(rename_all = "snake_case")] +pub enum ExecOutputStream { + Stdout, + Stderr, } #[derive(Debug, Clone, Deserialize, Serialize)] -pub struct ExecCommandStderrDeltaEvent { +pub struct ExecCommandOutputDeltaEvent { /// Identifier for the ExecCommandBegin that produced this chunk. pub call_id: String, - /// Raw stderr bytes (may not be valid UTF-8). + /// Which stream produced this chunk. + pub stream: ExecOutputStream, + /// Raw bytes from the stream (may not be valid UTF-8). #[serde(with = "serde_bytes")] pub chunk: ByteBuf, } diff --git a/codex-rs/core/tests/exec_stream_events.rs b/codex-rs/core/tests/exec_stream_events.rs index 059b69a064..50f6888f73 100644 --- a/codex-rs/core/tests/exec_stream_events.rs +++ b/codex-rs/core/tests/exec_stream_events.rs @@ -11,15 +11,19 @@ use codex_core::exec::StdoutStream; use codex_core::exec::process_exec_tool_call; use codex_core::protocol::Event; use codex_core::protocol::EventMsg; -use codex_core::protocol::ExecCommandStderrDeltaEvent; -use codex_core::protocol::ExecCommandStdoutDeltaEvent; +use codex_core::protocol::ExecCommandOutputDeltaEvent; +use codex_core::protocol::ExecOutputStream; use codex_core::protocol::SandboxPolicy; use tokio::sync::Notify; fn collect_stdout_events(rx: Receiver) -> Vec { let mut out = Vec::new(); while let Ok(ev) = rx.try_recv() { - if let EventMsg::ExecCommandStdoutDelta(ExecCommandStdoutDeltaEvent { chunk, .. }) = ev.msg + if let EventMsg::ExecCommandOutputDelta(ExecCommandOutputDeltaEvent { + stream: ExecOutputStream::Stdout, + chunk, + .. + }) = ev.msg { out.extend_from_slice(&chunk); } @@ -126,7 +130,11 @@ async fn test_exec_stderr_stream_events_echo() { // Collect only stderr delta events let mut err = Vec::new(); while let Ok(ev) = rx.try_recv() { - if let EventMsg::ExecCommandStderrDelta(ExecCommandStderrDeltaEvent { chunk, .. }) = ev.msg + if let EventMsg::ExecCommandOutputDelta(ExecCommandOutputDeltaEvent { + stream: ExecOutputStream::Stderr, + chunk, + .. + }) = ev.msg { err.extend_from_slice(&chunk); } diff --git a/codex-rs/exec/src/event_processor_with_human_output.rs b/codex-rs/exec/src/event_processor_with_human_output.rs index 05643c6282..7393ab72a0 100644 --- a/codex-rs/exec/src/event_processor_with_human_output.rs +++ b/codex-rs/exec/src/event_processor_with_human_output.rs @@ -239,8 +239,7 @@ impl EventProcessor for EventProcessorWithHumanOutput { cwd.to_string_lossy(), ); } - EventMsg::ExecCommandStdoutDelta(_) => {} - EventMsg::ExecCommandStderrDelta(_) => {} + EventMsg::ExecCommandOutputDelta(_) => {} EventMsg::ExecCommandEnd(ExecCommandEndEvent { call_id, stdout, diff --git a/codex-rs/mcp-server/src/codex_tool_runner.rs b/codex-rs/mcp-server/src/codex_tool_runner.rs index 94d47f7497..d489ffe076 100644 --- a/codex-rs/mcp-server/src/codex_tool_runner.rs +++ b/codex-rs/mcp-server/src/codex_tool_runner.rs @@ -258,8 +258,7 @@ async fn run_codex_tool_session_inner( | EventMsg::McpToolCallBegin(_) | EventMsg::McpToolCallEnd(_) | EventMsg::ExecCommandBegin(_) - | EventMsg::ExecCommandStdoutDelta(_) - | EventMsg::ExecCommandStderrDelta(_) + | EventMsg::ExecCommandOutputDelta(_) | EventMsg::ExecCommandEnd(_) | EventMsg::BackgroundEvent(_) | EventMsg::PatchApplyBegin(_) diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index d6454c82a7..374a4b5688 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -374,7 +374,7 @@ impl ChatWidget<'_> { ); self.add_to_history(HistoryCell::new_active_exec_command(command)); } - EventMsg::ExecCommandStdoutDelta(_) => {} + EventMsg::ExecCommandOutputDelta(_) => {} EventMsg::PatchApplyBegin(PatchApplyBeginEvent { call_id: _, auto_approved, From 92f3566d78fa95b90ccafbb0083e0e6a599f87c5 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Fri, 1 Aug 2025 14:15:55 -0700 Subject: [PATCH 038/100] chore: introduce SandboxPolicy::WorkspaceWrite::include_default_writable_roots (#1785) Without this change, it is challenging to create integration tests to verify that the folders not included in `writable_roots` in `SandboxPolicy::WorkspaceWrite` are read-only because, by default, `get_writable_roots_with_cwd()` includes `TMPDIR`, which is where most integrationt tests do their work. This introduces a `use_exact_writable_roots` option to disable the default includes returned by `get_writable_roots_with_cwd()`. --- [//]: # (BEGIN SAPLING FOOTER) Stack created with [Sapling](https://sapling-scm.com). Best reviewed with [ReviewStack](https://reviewstack.dev/openai/codex/pull/1785). * #1765 * __->__ #1785 --- codex-rs/common/src/sandbox_summary.rs | 4 ++++ codex-rs/core/src/config.rs | 2 ++ codex-rs/core/src/protocol.rs | 21 ++++++++++++++++++++- codex-rs/linux-sandbox/tests/landlock.rs | 1 + 4 files changed, 27 insertions(+), 1 deletion(-) diff --git a/codex-rs/common/src/sandbox_summary.rs b/codex-rs/common/src/sandbox_summary.rs index 3d33d92836..e0e309a9d9 100644 --- a/codex-rs/common/src/sandbox_summary.rs +++ b/codex-rs/common/src/sandbox_summary.rs @@ -7,6 +7,7 @@ pub fn summarize_sandbox_policy(sandbox_policy: &SandboxPolicy) -> String { SandboxPolicy::WorkspaceWrite { writable_roots, network_access, + include_default_writable_roots, } => { let mut summary = "workspace-write".to_string(); if !writable_roots.is_empty() { @@ -19,6 +20,9 @@ pub fn summarize_sandbox_policy(sandbox_policy: &SandboxPolicy) -> String { .join(", ") )); } + if !*include_default_writable_roots { + summary.push_str(" (exact writable roots)"); + } if *network_access { summary.push_str(" (network access enabled)"); } diff --git a/codex-rs/core/src/config.rs b/codex-rs/core/src/config.rs index 1b8da89419..87629d821a 100644 --- a/codex-rs/core/src/config.rs +++ b/codex-rs/core/src/config.rs @@ -356,6 +356,7 @@ impl ConfigToml { Some(s) => SandboxPolicy::WorkspaceWrite { writable_roots: s.writable_roots.clone(), network_access: s.network_access, + include_default_writable_roots: true, }, None => SandboxPolicy::new_workspace_write_policy(), }, @@ -727,6 +728,7 @@ writable_roots = [ SandboxPolicy::WorkspaceWrite { writable_roots: vec![PathBuf::from("/tmp")], network_access: false, + include_default_writable_roots: true, }, sandbox_workspace_write_cfg.derive_sandbox_policy(sandbox_mode_override) ); diff --git a/codex-rs/core/src/protocol.rs b/codex-rs/core/src/protocol.rs index 3be0c92ceb..af65f4d3a8 100644 --- a/codex-rs/core/src/protocol.rs +++ b/codex-rs/core/src/protocol.rs @@ -180,9 +180,19 @@ pub enum SandboxPolicy { /// default. #[serde(default)] network_access: bool, + + /// When set to `true`, will include defaults like the current working + /// directory and TMPDIR (on macOS). When `false`, only `writable_roots` + /// are used. (Mainly used for testing.) + #[serde(default = "default_true")] + include_default_writable_roots: bool, }, } +fn default_true() -> bool { + true +} + impl FromStr for SandboxPolicy { type Err = serde_json::Error; @@ -204,6 +214,7 @@ impl SandboxPolicy { SandboxPolicy::WorkspaceWrite { writable_roots: vec![], network_access: false, + include_default_writable_roots: true, } } @@ -235,7 +246,15 @@ impl SandboxPolicy { match self { SandboxPolicy::DangerFullAccess => Vec::new(), SandboxPolicy::ReadOnly => Vec::new(), - SandboxPolicy::WorkspaceWrite { writable_roots, .. } => { + SandboxPolicy::WorkspaceWrite { + writable_roots, + include_default_writable_roots, + .. + } => { + if !*include_default_writable_roots { + return writable_roots.clone(); + } + let mut roots = writable_roots.clone(); roots.push(cwd.to_path_buf()); diff --git a/codex-rs/linux-sandbox/tests/landlock.rs b/codex-rs/linux-sandbox/tests/landlock.rs index c16e9227df..1375a4c686 100644 --- a/codex-rs/linux-sandbox/tests/landlock.rs +++ b/codex-rs/linux-sandbox/tests/landlock.rs @@ -49,6 +49,7 @@ async fn run_cmd(cmd: &[&str], writable_roots: &[PathBuf], timeout_ms: u64) { let sandbox_policy = SandboxPolicy::WorkspaceWrite { writable_roots: writable_roots.to_vec(), network_access: false, + include_default_writable_roots: true, }; let sandbox_program = env!("CARGO_BIN_EXE_codex-linux-sandbox"); let codex_linux_sandbox_exe = Some(PathBuf::from(sandbox_program)); From fe62f859a68ade8e3b63bf626472a0adbda867e3 Mon Sep 17 00:00:00 2001 From: aibrahim-oai Date: Fri, 1 Aug 2025 15:13:53 -0700 Subject: [PATCH 039/100] Add Error variant to ConversationCreateResult [Stack 1/2] (#1784) Switch ConversationCreateResult from a struct to a tagged enum (Ok | Error) Stack: Top: #1783 Bottom: #1784 --- codex-rs/mcp-server/src/mcp_protocol.rs | 44 ++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/codex-rs/mcp-server/src/mcp_protocol.rs b/codex-rs/mcp-server/src/mcp_protocol.rs index 287890bf5e..2f8858a37b 100644 --- a/codex-rs/mcp-server/src/mcp_protocol.rs +++ b/codex-rs/mcp-server/src/mcp_protocol.rs @@ -172,15 +172,22 @@ pub enum ToolCallResponseResult { } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct ConversationCreateResult { - pub conversation_id: ConversationId, - pub model: String, +#[serde(untagged)] +pub enum ConversationCreateResult { + Ok { + conversation_id: ConversationId, + model: String, + }, + Error { + message: String, + }, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct ConversationStreamResult {} #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +// TODO: remove this status because we have is_error field in the response. #[serde(tag = "status", rename_all = "camelCase")] pub enum ConversationSendMessageResult { Ok, @@ -491,7 +498,7 @@ mod tests { request_id: RequestId::Integer(1), is_error: None, result: Some(ToolCallResponseResult::ConversationCreate( - ConversationCreateResult { + ConversationCreateResult::Ok { conversation_id: ConversationId(uuid!("d0f6ecbe-84a2-41c1-b23d-b20473b25eab")), model: "o3".into(), }, @@ -515,6 +522,35 @@ mod tests { assert_eq!(req_id, RequestId::Integer(1)); } + #[test] + fn response_error_conversation_create_full_schema() { + let env = ToolCallResponse { + request_id: RequestId::Integer(2), + is_error: Some(true), + result: Some(ToolCallResponseResult::ConversationCreate( + ConversationCreateResult::Error { + message: "Failed to initialize session".into(), + }, + )), + }; + let req_id = env.request_id.clone(); + let observed = to_val(&CallToolResult::from(env)); + let expected = json!({ + "content": [ + { "type": "text", "text": "{\"message\":\"Failed to initialize session\"}" } + ], + "isError": true, + "structuredContent": { + "message": "Failed to initialize session" + } + }); + assert_eq!( + observed, expected, + "error response (ConversationCreate) must match" + ); + assert_eq!(req_id, RequestId::Integer(2)); + } + #[test] fn response_success_conversation_stream_empty_result_object() { let env = ToolCallResponse { From 97ab8fb6101dc5203104117023453862320e91ca Mon Sep 17 00:00:00 2001 From: aibrahim-oai Date: Fri, 1 Aug 2025 15:18:36 -0700 Subject: [PATCH 040/100] MCP: add conversation.create tool [Stack 2/2] (#1783) Introduce conversation.create handler (handle_create_conversation) and wire it in MessageProcessor. Stack: Top: #1783 Bottom: #1784 --------- Co-authored-by: Gabriel Peal --- codex-rs/mcp-server/src/conversation_loop.rs | 121 +++++++++++++ codex-rs/mcp-server/src/lib.rs | 1 + codex-rs/mcp-server/src/message_processor.rs | 8 + .../src/tool_handlers/create_conversation.rs | 160 ++++++++++++++++++ codex-rs/mcp-server/src/tool_handlers/mod.rs | 1 + .../mcp-server/tests/common/mcp_process.rs | 36 ++++ .../mcp-server/tests/create_conversation.rs | 128 ++++++++++++++ 7 files changed, 455 insertions(+) create mode 100644 codex-rs/mcp-server/src/conversation_loop.rs create mode 100644 codex-rs/mcp-server/src/tool_handlers/create_conversation.rs create mode 100644 codex-rs/mcp-server/tests/create_conversation.rs diff --git a/codex-rs/mcp-server/src/conversation_loop.rs b/codex-rs/mcp-server/src/conversation_loop.rs new file mode 100644 index 0000000000..534275181a --- /dev/null +++ b/codex-rs/mcp-server/src/conversation_loop.rs @@ -0,0 +1,121 @@ +use std::sync::Arc; + +use crate::exec_approval::handle_exec_approval_request; +use crate::outgoing_message::OutgoingMessageSender; +use crate::outgoing_message::OutgoingNotificationMeta; +use crate::patch_approval::handle_patch_approval_request; +use codex_core::Codex; +use codex_core::protocol::AgentMessageEvent; +use codex_core::protocol::ApplyPatchApprovalRequestEvent; +use codex_core::protocol::EventMsg; +use codex_core::protocol::ExecApprovalRequestEvent; +use mcp_types::RequestId; +use tracing::error; + +pub async fn run_conversation_loop( + codex: Arc, + outgoing: Arc, + request_id: RequestId, +) { + let request_id_str = match &request_id { + RequestId::String(s) => s.clone(), + RequestId::Integer(n) => n.to_string(), + }; + + // Stream events until the task needs to pause for user interaction or + // completes. + loop { + match codex.next_event().await { + Ok(event) => { + outgoing + .send_event_as_notification( + &event, + Some(OutgoingNotificationMeta::new(Some(request_id.clone()))), + ) + .await; + + match event.msg { + EventMsg::ExecApprovalRequest(ExecApprovalRequestEvent { + command, + cwd, + call_id, + reason: _, + }) => { + handle_exec_approval_request( + command, + cwd, + outgoing.clone(), + codex.clone(), + request_id.clone(), + request_id_str.clone(), + event.id.clone(), + call_id, + ) + .await; + continue; + } + EventMsg::Error(_) => { + error!("Codex runtime error"); + } + EventMsg::ApplyPatchApprovalRequest(ApplyPatchApprovalRequestEvent { + call_id, + reason, + grant_root, + changes, + }) => { + handle_patch_approval_request( + call_id, + reason, + grant_root, + changes, + outgoing.clone(), + codex.clone(), + request_id.clone(), + request_id_str.clone(), + event.id.clone(), + ) + .await; + continue; + } + EventMsg::TaskComplete(_) => {} + EventMsg::SessionConfigured(_) => { + tracing::error!("unexpected SessionConfigured event"); + } + EventMsg::AgentMessageDelta(_) => { + // TODO: think how we want to support this in the MCP + } + EventMsg::AgentReasoningDelta(_) => { + // TODO: think how we want to support this in the MCP + } + EventMsg::AgentMessage(AgentMessageEvent { .. }) => { + // TODO: think how we want to support this in the MCP + } + EventMsg::TaskStarted + | EventMsg::TokenCount(_) + | EventMsg::AgentReasoning(_) + | EventMsg::McpToolCallBegin(_) + | EventMsg::McpToolCallEnd(_) + | EventMsg::ExecCommandBegin(_) + | EventMsg::ExecCommandEnd(_) + | EventMsg::BackgroundEvent(_) + | EventMsg::ExecCommandOutputDelta(_) + | EventMsg::PatchApplyBegin(_) + | EventMsg::PatchApplyEnd(_) + | EventMsg::GetHistoryEntryResponse(_) + | EventMsg::PlanUpdate(_) + | EventMsg::ShutdownComplete => { + // For now, we do not do anything extra for these + // events. Note that + // send(codex_event_to_notification(&event)) above has + // already dispatched these events as notifications, + // though we may want to do give different treatment to + // individual events in the future. + } + } + } + Err(e) => { + error!("Codex runtime error: {e}"); + } + } + } +} diff --git a/codex-rs/mcp-server/src/lib.rs b/codex-rs/mcp-server/src/lib.rs index 6b3c0ddbe6..f6d0838efc 100644 --- a/codex-rs/mcp-server/src/lib.rs +++ b/codex-rs/mcp-server/src/lib.rs @@ -17,6 +17,7 @@ use tracing_subscriber::EnvFilter; mod codex_tool_config; mod codex_tool_runner; +mod conversation_loop; mod exec_approval; mod json_to_toml; pub mod mcp_protocol; diff --git a/codex-rs/mcp-server/src/message_processor.rs b/codex-rs/mcp-server/src/message_processor.rs index 14e9bb3887..2f99cda723 100644 --- a/codex-rs/mcp-server/src/message_processor.rs +++ b/codex-rs/mcp-server/src/message_processor.rs @@ -11,6 +11,7 @@ use crate::mcp_protocol::ToolCallRequestParams; use crate::mcp_protocol::ToolCallResponse; use crate::mcp_protocol::ToolCallResponseResult; use crate::outgoing_message::OutgoingMessageSender; +use crate::tool_handlers::create_conversation::handle_create_conversation; use crate::tool_handlers::send_message::handle_send_message; use codex_core::Codex; @@ -67,6 +68,10 @@ impl MessageProcessor { self.session_map.clone() } + pub(crate) fn outgoing(&self) -> Arc { + self.outgoing.clone() + } + pub(crate) fn running_session_ids(&self) -> Arc>> { self.running_session_ids.clone() } @@ -349,6 +354,9 @@ impl MessageProcessor { } async fn handle_new_tool_calls(&self, request_id: RequestId, params: ToolCallRequestParams) { match params { + ToolCallRequestParams::ConversationCreate(args) => { + handle_create_conversation(self, request_id, args).await; + } ToolCallRequestParams::ConversationSendMessage(args) => { handle_send_message(self, request_id, args).await; } diff --git a/codex-rs/mcp-server/src/tool_handlers/create_conversation.rs b/codex-rs/mcp-server/src/tool_handlers/create_conversation.rs new file mode 100644 index 0000000000..28a8965115 --- /dev/null +++ b/codex-rs/mcp-server/src/tool_handlers/create_conversation.rs @@ -0,0 +1,160 @@ +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::Arc; + +use codex_core::Codex; +use codex_core::codex_wrapper::init_codex; +use codex_core::config::Config as CodexConfig; +use codex_core::config::ConfigOverrides; +use codex_core::protocol::EventMsg; +use codex_core::protocol::SessionConfiguredEvent; +use mcp_types::RequestId; +use tokio::sync::Mutex; +use uuid::Uuid; + +use crate::conversation_loop::run_conversation_loop; +use crate::json_to_toml::json_to_toml; +use crate::mcp_protocol::ConversationCreateArgs; +use crate::mcp_protocol::ConversationCreateResult; +use crate::mcp_protocol::ConversationId; +use crate::mcp_protocol::ToolCallResponseResult; +use crate::message_processor::MessageProcessor; + +pub(crate) async fn handle_create_conversation( + message_processor: &MessageProcessor, + id: RequestId, + args: ConversationCreateArgs, +) { + // Build ConfigOverrides from args + let ConversationCreateArgs { + prompt: _, // not used here; creation only establishes the session + model, + cwd, + approval_policy, + sandbox, + config, + profile, + base_instructions, + } = args; + + // Convert config overrides JSON into CLI-style TOML overrides + let cli_overrides: Vec<(String, toml::Value)> = match config { + Some(v) => match v.as_object() { + Some(map) => map + .into_iter() + .map(|(k, v)| (k.clone(), json_to_toml(v.clone()))) + .collect(), + None => Vec::new(), + }, + None => Vec::new(), + }; + + let overrides = ConfigOverrides { + model: Some(model.clone()), + cwd: Some(PathBuf::from(cwd)), + approval_policy, + sandbox_mode: sandbox, + model_provider: None, + config_profile: profile, + codex_linux_sandbox_exe: None, + base_instructions, + include_plan_tool: None, + }; + + let cfg: CodexConfig = match CodexConfig::load_with_cli_overrides(cli_overrides, overrides) { + Ok(cfg) => cfg, + Err(e) => { + message_processor + .send_response_with_optional_error( + id, + Some(ToolCallResponseResult::ConversationCreate( + ConversationCreateResult::Error { + message: format!("Failed to load config: {e}"), + }, + )), + Some(true), + ) + .await; + return; + } + }; + + // Initialize Codex session + let codex_conversation = match init_codex(cfg).await { + Ok(conv) => conv, + Err(e) => { + message_processor + .send_response_with_optional_error( + id, + Some(ToolCallResponseResult::ConversationCreate( + ConversationCreateResult::Error { + message: format!("Failed to initialize session: {e}"), + }, + )), + Some(true), + ) + .await; + return; + } + }; + + // Expect SessionConfigured; if not, return error. + let EventMsg::SessionConfigured(SessionConfiguredEvent { model, .. }) = + &codex_conversation.session_configured.msg + else { + message_processor + .send_response_with_optional_error( + id, + Some(ToolCallResponseResult::ConversationCreate( + ConversationCreateResult::Error { + message: "Expected SessionConfigured event".to_string(), + }, + )), + Some(true), + ) + .await; + return; + }; + + let effective_model = model.clone(); + + let session_id = codex_conversation.session_id; + let codex_arc = Arc::new(codex_conversation.codex); + + // Store session for future calls + insert_session( + session_id, + codex_arc.clone(), + message_processor.session_map(), + ) + .await; + // Run the conversation loop in the background so this request can return immediately. + let outgoing = message_processor.outgoing(); + let spawn_id = id.clone(); + tokio::spawn(async move { + run_conversation_loop(codex_arc.clone(), outgoing, spawn_id).await; + }); + + // Reply with the new conversation id and effective model + message_processor + .send_response_with_optional_error( + id, + Some(ToolCallResponseResult::ConversationCreate( + ConversationCreateResult::Ok { + conversation_id: ConversationId(session_id), + model: effective_model, + }, + )), + Some(false), + ) + .await; +} + +async fn insert_session( + session_id: Uuid, + codex: Arc, + session_map: Arc>>>, +) { + let mut guard = session_map.lock().await; + guard.insert(session_id, codex); +} diff --git a/codex-rs/mcp-server/src/tool_handlers/mod.rs b/codex-rs/mcp-server/src/tool_handlers/mod.rs index 1907ec64e0..5863bdc288 100644 --- a/codex-rs/mcp-server/src/tool_handlers/mod.rs +++ b/codex-rs/mcp-server/src/tool_handlers/mod.rs @@ -1 +1,2 @@ +pub(crate) mod create_conversation; pub(crate) mod send_message; diff --git a/codex-rs/mcp-server/tests/common/mcp_process.rs b/codex-rs/mcp-server/tests/common/mcp_process.rs index d7144af480..83cf085cd4 100644 --- a/codex-rs/mcp-server/tests/common/mcp_process.rs +++ b/codex-rs/mcp-server/tests/common/mcp_process.rs @@ -14,6 +14,7 @@ use assert_cmd::prelude::*; use codex_core::protocol::InputItem; use codex_mcp_server::CodexToolCallParam; use codex_mcp_server::CodexToolCallReplyParam; +use codex_mcp_server::mcp_protocol::ConversationCreateArgs; use codex_mcp_server::mcp_protocol::ConversationId; use codex_mcp_server::mcp_protocol::ConversationSendMessageArgs; use codex_mcp_server::mcp_protocol::ToolCallRequestParams; @@ -200,6 +201,41 @@ impl McpProcess { .await } + pub async fn send_conversation_create_tool_call( + &mut self, + prompt: &str, + model: &str, + cwd: &str, + ) -> anyhow::Result { + let params = ToolCallRequestParams::ConversationCreate(ConversationCreateArgs { + prompt: prompt.to_string(), + model: model.to_string(), + cwd: cwd.to_string(), + approval_policy: None, + sandbox: None, + config: None, + profile: None, + base_instructions: None, + }); + self.send_request( + mcp_types::CallToolRequest::METHOD, + Some(serde_json::to_value(params)?), + ) + .await + } + + pub async fn send_conversation_create_with_args( + &mut self, + args: ConversationCreateArgs, + ) -> anyhow::Result { + let params = ToolCallRequestParams::ConversationCreate(args); + self.send_request( + mcp_types::CallToolRequest::METHOD, + Some(serde_json::to_value(params)?), + ) + .await + } + async fn send_request( &mut self, method: &str, diff --git a/codex-rs/mcp-server/tests/create_conversation.rs b/codex-rs/mcp-server/tests/create_conversation.rs new file mode 100644 index 0000000000..41052ea7b3 --- /dev/null +++ b/codex-rs/mcp-server/tests/create_conversation.rs @@ -0,0 +1,128 @@ +#![allow(clippy::expect_used, clippy::unwrap_used)] + +use std::path::Path; + +use mcp_test_support::McpProcess; +use mcp_test_support::create_final_assistant_message_sse_response; +use mcp_test_support::create_mock_chat_completions_server; +use mcp_types::JSONRPCResponse; +use mcp_types::RequestId; +use pretty_assertions::assert_eq; +use serde_json::json; +use tempfile::TempDir; +use tokio::time::timeout; + +const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_conversation_create_and_send_message_ok() { + // Mock server – we won't strictly rely on it, but provide one to satisfy any model wiring. + let responses = vec![ + create_final_assistant_message_sse_response("Done").expect("build mock assistant message"), + ]; + let server = create_mock_chat_completions_server(responses).await; + + // Temporary Codex home with config pointing at the mock server. + let codex_home = TempDir::new().expect("create temp dir"); + create_config_toml(codex_home.path(), &server.uri()).expect("write config.toml"); + + // Start MCP server process and initialize. + let mut mcp = McpProcess::new(codex_home.path()) + .await + .expect("spawn mcp process"); + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()) + .await + .expect("init timeout") + .expect("init failed"); + + // Create a conversation via the new tool. + let req_id = mcp + .send_conversation_create_tool_call("", "o3", "/repo") + .await + .expect("send conversationCreate"); + + let resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(req_id)), + ) + .await + .expect("create response timeout") + .expect("create response error"); + + // Structured content must include status=ok, a UUID conversation_id and the model we passed. + let sc = &resp.result["structuredContent"]; + let conv_id = sc["conversation_id"].as_str().expect("uuid string"); + assert!(!conv_id.is_empty()); + assert_eq!(sc["model"], json!("o3")); + + // Now send a message to the created conversation and expect an OK result. + let send_id = mcp + .send_user_message_tool_call("Hello", conv_id) + .await + .expect("send message"); + + let send_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(send_id)), + ) + .await + .expect("send response timeout") + .expect("send response error"); + assert_eq!( + send_resp.result["structuredContent"], + json!({ "status": "ok" }) + ); + + // avoid race condition by waiting for the mock server to receive the chat.completions request + let deadline = std::time::Instant::now() + DEFAULT_READ_TIMEOUT; + loop { + let requests = server.received_requests().await.unwrap_or_default(); + if !requests.is_empty() { + break; + } + if std::time::Instant::now() >= deadline { + panic!("mock server did not receive the chat.completions request in time"); + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + + // Verify the outbound request body matches expectations for Chat Completions. + let request = &server.received_requests().await.unwrap()[0]; + let body = request + .body_json::() + .expect("parse request body as JSON"); + assert_eq!(body["model"], json!("o3")); + assert!(body["stream"].as_bool().unwrap_or(false)); + let messages = body["messages"] + .as_array() + .expect("messages should be array"); + let last = messages.last().expect("at least one message"); + assert_eq!(last["role"], json!("user")); + assert_eq!(last["content"], json!("Hello")); + + drop(server); +} + +// Helper to create a config.toml pointing at the mock model server. +fn create_config_toml(codex_home: &Path, server_uri: &str) -> std::io::Result<()> { + let config_toml = codex_home.join("config.toml"); + std::fs::write( + config_toml, + format!( + r#" +model = "mock-model" +approval_policy = "never" +sandbox_mode = "danger-full-access" + +model_provider = "mock_provider" + +[model_providers.mock_provider] +name = "Mock provider for test" +base_url = "{server_uri}/v1" +wire_api = "chat" +request_max_retries = 0 +stream_max_retries = 0 +"# + ), + ) +} From 80555d4ff25b8d7af34be207436c786de4fbaf71 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Fri, 1 Aug 2025 16:11:24 -0700 Subject: [PATCH 041/100] feat: make .git read-only within a writable root when using Seatbelt (#1765) To make `--full-auto` safer, this PR updates the Seatbelt policy so that a `SandboxPolicy` with a `writable_root` that contains a `.git/` _directory_ will make `.git/` _read-only_ (though as a follow-up, we should also consider the case where `.git` is a _file_ with a `gitdir: /path/to/actual/repo/.git` entry that should also be protected). The two major changes in this PR: - Updating `SandboxPolicy::get_writable_roots_with_cwd()` to return a `Vec` instead of a `Vec` where a `WritableRoot` can specify a list of read-only subpaths. - Updating `create_seatbelt_command_args()` to honor the read-only subpaths in `WritableRoot`. The logic to update the policy is a fairly straightforward update to `create_seatbelt_command_args()`, but perhaps the more interesting part of this PR is the introduction of an integration test in `tests/sandbox.rs`. Leveraging the new API in #1785, we test `SandboxPolicy` under various conditions, including ones where `$TMPDIR` is not readable, which is critical for verifying the new behavior. To ensure that Codex can run its own tests, e.g.: ``` just codex debug seatbelt --full-auto -- cargo test if_git_repo_is_writable_root_then_dot_git_folder_is_read_only ``` I had to introduce the use of `CODEX_SANDBOX=sandbox`, which is comparable to how `CODEX_SANDBOX_NETWORK_DISABLED=1` was already being used. Adding a comparable change for Landlock will be done in a subsequent PR. --- AGENTS.md | 4 +- codex-rs/config.md | 2 + codex-rs/core/src/protocol.rs | 56 ++++-- codex-rs/core/src/seatbelt.rs | 238 +++++++++++++++++++++++-- codex-rs/core/src/spawn.rs | 5 + codex-rs/core/tests/sandbox.rs | 195 ++++++++++++++++++++ codex-rs/linux-sandbox/src/landlock.rs | 6 +- 7 files changed, 478 insertions(+), 28 deletions(-) create mode 100644 codex-rs/core/tests/sandbox.rs diff --git a/AGENTS.md b/AGENTS.md index 27af48ae60..5c3f659c35 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,7 +2,9 @@ In the codex-rs folder where the rust code lives: -- Never add or modify any code related to `CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR`. You operate in a sandbox where `CODEX_SANDBOX_NETWORK_DISABLED=1` will be set whenever you use the `shell` tool. Any existing code that uses `CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR` was authored with this fact in mind. It is often used to early exit out of tests that the author knew you would not be able to run given your sandbox limitations. +- Never add or modify any code related to `CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR` or `CODEX_SANDBOX_ENV_VAR`. + - You operate in a sandbox where `CODEX_SANDBOX_NETWORK_DISABLED=1` will be set whenever you use the `shell` tool. Any existing code that uses `CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR` was authored with this fact in mind. It is often used to early exit out of tests that the author knew you would not be able to run given your sandbox limitations. + - Similarly, when you spawn a process using Seatbelt (`/usr/bin/sandbox-exec`), `CODEX_SANDBOX=seatbelt` will be set on the child process. Integration tests that want to run Seatbelt themselves cannot be run under Seatbelt, so checks for `CODEX_SANDBOX=seatbelt` are also often used to early exit out of tests, as appropriate. Before creating a pull request with changes to `codex-rs`, run `just fmt` (in `codex-rs` directory) to format the code and `just fix` (in `codex-rs` directory) to fix any linter issues in the code, ensure the test suite passes by running `cargo test --all-features` in the `codex-rs` directory. diff --git a/codex-rs/config.md b/codex-rs/config.md index 1a407a239b..c7dfe42a75 100644 --- a/codex-rs/config.md +++ b/codex-rs/config.md @@ -259,6 +259,8 @@ disk, but attempts to write a file or access the network will be blocked. A more relaxed policy is `workspace-write`. When specified, the current working directory for the Codex task will be writable (as well as `$TMPDIR` on macOS). Note that the CLI defaults to using the directory where it was spawned as `cwd`, though this can be overridden using `--cwd/-C`. +On macOS (and soon Linux), all writable roots (including `cwd`) that contain a `.git/` folder _as an immediate child_ will configure the `.git/` folder to be read-only while the rest of the Git repository will be writable. This means that commands like `git commit` will fail, by default (as it entails writing to `.git/`), and will require Codex to ask for permission. + ```toml # same as `--sandbox workspace-write` sandbox_mode = "workspace-write" diff --git a/codex-rs/core/src/protocol.rs b/codex-rs/core/src/protocol.rs index af65f4d3a8..1bfeee56ab 100644 --- a/codex-rs/core/src/protocol.rs +++ b/codex-rs/core/src/protocol.rs @@ -189,6 +189,16 @@ pub enum SandboxPolicy { }, } +/// A writable root path accompanied by a list of subpaths that should remain +/// read‑only even when the root is writable. This is primarily used to ensure +/// top‑level VCS metadata directories (e.g. `.git`) under a writable root are +/// not modified by the agent. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct WritableRoot { + pub root: PathBuf, + pub read_only_subpaths: Vec, +} + fn default_true() -> bool { true } @@ -240,9 +250,10 @@ impl SandboxPolicy { } } - /// Returns the list of writable roots that should be passed down to the - /// Landlock rules installer, tailored to the current working directory. - pub fn get_writable_roots_with_cwd(&self, cwd: &Path) -> Vec { + /// Returns the list of writable roots (tailored to the current working + /// directory) together with subpaths that should remain read‑only under + /// each writable root. + pub fn get_writable_roots_with_cwd(&self, cwd: &Path) -> Vec { match self { SandboxPolicy::DangerFullAccess => Vec::new(), SandboxPolicy::ReadOnly => Vec::new(), @@ -251,24 +262,39 @@ impl SandboxPolicy { include_default_writable_roots, .. } => { - if !*include_default_writable_roots { - return writable_roots.clone(); - } + // Start from explicitly configured writable roots. + let mut roots: Vec = writable_roots.clone(); - let mut roots = writable_roots.clone(); - roots.push(cwd.to_path_buf()); + // Optionally include defaults (cwd and TMPDIR on macOS). + if *include_default_writable_roots { + roots.push(cwd.to_path_buf()); - // Also include the per-user tmp dir on macOS. - // Note this is added dynamically rather than storing it in - // writable_roots because writable_roots contains only static - // values deserialized from the config file. - if cfg!(target_os = "macos") { - if let Some(tmpdir) = std::env::var_os("TMPDIR") { - roots.push(PathBuf::from(tmpdir)); + // Also include the per-user tmp dir on macOS. + // Note this is added dynamically rather than storing it in + // `writable_roots` because `writable_roots` contains only static + // values deserialized from the config file. + if cfg!(target_os = "macos") { + if let Some(tmpdir) = std::env::var_os("TMPDIR") { + roots.push(PathBuf::from(tmpdir)); + } } } + // For each root, compute subpaths that should remain read-only. roots + .into_iter() + .map(|writable_root| { + let mut subpaths = Vec::new(); + let top_level_git = writable_root.join(".git"); + if top_level_git.is_dir() { + subpaths.push(top_level_git); + } + WritableRoot { + root: writable_root, + read_only_subpaths: subpaths, + } + }) + .collect() } } } diff --git a/codex-rs/core/src/seatbelt.rs b/codex-rs/core/src/seatbelt.rs index be2acb1bdc..0364840b1a 100644 --- a/codex-rs/core/src/seatbelt.rs +++ b/codex-rs/core/src/seatbelt.rs @@ -4,6 +4,7 @@ use std::path::PathBuf; use tokio::process::Child; use crate::protocol::SandboxPolicy; +use crate::spawn::CODEX_SANDBOX_ENV_VAR; use crate::spawn::StdioPolicy; use crate::spawn::spawn_child_async; @@ -20,10 +21,11 @@ pub async fn spawn_command_under_seatbelt( sandbox_policy: &SandboxPolicy, cwd: PathBuf, stdio_policy: StdioPolicy, - env: HashMap, + mut env: HashMap, ) -> std::io::Result { let args = create_seatbelt_command_args(command, sandbox_policy, &cwd); let arg0 = None; + env.insert(CODEX_SANDBOX_ENV_VAR.to_string(), "seatbelt".to_string()); spawn_child_async( PathBuf::from(MACOS_PATH_TO_SEATBELT_EXECUTABLE), args, @@ -50,16 +52,38 @@ fn create_seatbelt_command_args( ) } else { let writable_roots = sandbox_policy.get_writable_roots_with_cwd(cwd); - let (writable_folder_policies, cli_args): (Vec, Vec) = writable_roots - .iter() - .enumerate() - .map(|(index, root)| { - let param_name = format!("WRITABLE_ROOT_{index}"); - let policy: String = format!("(subpath (param \"{param_name}\"))"); - let cli_arg = format!("-D{param_name}={}", root.to_string_lossy()); - (policy, cli_arg) - }) - .unzip(); + + let mut writable_folder_policies: Vec = Vec::new(); + let mut cli_args: Vec = Vec::new(); + + for (index, wr) in writable_roots.iter().enumerate() { + // Canonicalize to avoid mismatches like /var vs /private/var on macOS. + let canonical_root = wr.root.canonicalize().unwrap_or_else(|_| wr.root.clone()); + let root_param = format!("WRITABLE_ROOT_{index}"); + cli_args.push(format!( + "-D{root_param}={}", + canonical_root.to_string_lossy() + )); + + if wr.read_only_subpaths.is_empty() { + writable_folder_policies.push(format!("(subpath (param \"{root_param}\"))")); + } else { + // Add parameters for each read-only subpath and generate + // the `(require-not ...)` clauses. + let mut require_parts: Vec = Vec::new(); + require_parts.push(format!("(subpath (param \"{root_param}\"))")); + for (subpath_index, ro) in wr.read_only_subpaths.iter().enumerate() { + let canonical_ro = ro.canonicalize().unwrap_or_else(|_| ro.clone()); + let ro_param = format!("WRITABLE_ROOT_{index}_RO_{subpath_index}"); + cli_args.push(format!("-D{ro_param}={}", canonical_ro.to_string_lossy())); + require_parts + .push(format!("(require-not (subpath (param \"{ro_param}\")))")); + } + let policy_component = format!("(require-all {} )", require_parts.join(" ")); + writable_folder_policies.push(policy_component); + } + } + if writable_folder_policies.is_empty() { ("".to_string(), Vec::::new()) } else { @@ -88,9 +112,201 @@ fn create_seatbelt_command_args( let full_policy = format!( "{MACOS_SEATBELT_BASE_POLICY}\n{file_read_policy}\n{file_write_policy}\n{network_policy}" ); + let mut seatbelt_args: Vec = vec!["-p".to_string(), full_policy]; seatbelt_args.extend(extra_cli_args); seatbelt_args.push("--".to_string()); seatbelt_args.extend(command); seatbelt_args } + +#[cfg(test)] +mod tests { + #![expect(clippy::expect_used)] + use super::MACOS_SEATBELT_BASE_POLICY; + use super::create_seatbelt_command_args; + use crate::protocol::SandboxPolicy; + use pretty_assertions::assert_eq; + use std::fs; + use std::path::Path; + use std::path::PathBuf; + use tempfile::TempDir; + + #[test] + fn create_seatbelt_args_with_read_only_git_subpath() { + // Create a temporary workspace with two writable roots: one containing + // a top-level .git directory and one without it. + let tmp = TempDir::new().expect("tempdir"); + let PopulatedTmp { + root_with_git, + root_without_git, + root_with_git_canon, + root_with_git_git_canon, + root_without_git_canon, + } = populate_tmpdir(tmp.path()); + + // Build a policy that only includes the two test roots as writable and + // does not automatically include defaults like cwd or TMPDIR. + let policy = SandboxPolicy::WorkspaceWrite { + writable_roots: vec![root_with_git.clone(), root_without_git.clone()], + network_access: false, + include_default_writable_roots: false, + }; + + let args = create_seatbelt_command_args( + vec!["/bin/echo".to_string(), "hello".to_string()], + &policy, + tmp.path(), + ); + + // Build the expected policy text using a raw string for readability. + // Note that the policy includes: + // - the base policy, + // - read-only access to the filesystem, + // - write access to WRITABLE_ROOT_0 (but not its .git) and WRITABLE_ROOT_1. + let expected_policy = format!( + r#"{MACOS_SEATBELT_BASE_POLICY} +; allow read-only file operations +(allow file-read*) +(allow file-write* +(require-all (subpath (param "WRITABLE_ROOT_0")) (require-not (subpath (param "WRITABLE_ROOT_0_RO_0"))) ) (subpath (param "WRITABLE_ROOT_1")) +) +"#, + ); + + let expected_args = vec![ + "-p".to_string(), + expected_policy, + format!( + "-DWRITABLE_ROOT_0={}", + root_with_git_canon.to_string_lossy() + ), + format!( + "-DWRITABLE_ROOT_0_RO_0={}", + root_with_git_git_canon.to_string_lossy() + ), + format!( + "-DWRITABLE_ROOT_1={}", + root_without_git_canon.to_string_lossy() + ), + "--".to_string(), + "/bin/echo".to_string(), + "hello".to_string(), + ]; + + assert_eq!(args, expected_args); + } + + #[test] + fn create_seatbelt_args_for_cwd_as_git_repo() { + // Create a temporary workspace with two writable roots: one containing + // a top-level .git directory and one without it. + let tmp = TempDir::new().expect("tempdir"); + let PopulatedTmp { + root_with_git, + root_with_git_canon, + root_with_git_git_canon, + .. + } = populate_tmpdir(tmp.path()); + + // Build a policy that does not specify any writable_roots, but does + // use the default ones (cwd and TMPDIR) and verifies the `.git` check + // is done properly for cwd. + let policy = SandboxPolicy::WorkspaceWrite { + writable_roots: vec![], + network_access: false, + include_default_writable_roots: true, + }; + + let args = create_seatbelt_command_args( + vec!["/bin/echo".to_string(), "hello".to_string()], + &policy, + root_with_git.as_path(), + ); + + let tmpdir_env_var = if cfg!(target_os = "macos") { + std::env::var("TMPDIR") + .ok() + .map(PathBuf::from) + .and_then(|p| p.canonicalize().ok()) + .map(|p| p.to_string_lossy().to_string()) + } else { + None + }; + let tempdir_policy_entry = if tmpdir_env_var.is_some() { + " (subpath (param \"WRITABLE_ROOT_1\"))" + } else { + "" + }; + + // Build the expected policy text using a raw string for readability. + // Note that the policy includes: + // - the base policy, + // - read-only access to the filesystem, + // - write access to WRITABLE_ROOT_0 (but not its .git) and WRITABLE_ROOT_1. + let expected_policy = format!( + r#"{MACOS_SEATBELT_BASE_POLICY} +; allow read-only file operations +(allow file-read*) +(allow file-write* +(require-all (subpath (param "WRITABLE_ROOT_0")) (require-not (subpath (param "WRITABLE_ROOT_0_RO_0"))) ){tempdir_policy_entry} +) +"#, + ); + + let mut expected_args = vec![ + "-p".to_string(), + expected_policy, + format!( + "-DWRITABLE_ROOT_0={}", + root_with_git_canon.to_string_lossy() + ), + format!( + "-DWRITABLE_ROOT_0_RO_0={}", + root_with_git_git_canon.to_string_lossy() + ), + ]; + + if let Some(p) = tmpdir_env_var { + expected_args.push(format!("-DWRITABLE_ROOT_1={p}")); + } + + expected_args.extend(vec![ + "--".to_string(), + "/bin/echo".to_string(), + "hello".to_string(), + ]); + + assert_eq!(args, expected_args); + } + + struct PopulatedTmp { + root_with_git: PathBuf, + root_without_git: PathBuf, + root_with_git_canon: PathBuf, + root_with_git_git_canon: PathBuf, + root_without_git_canon: PathBuf, + } + + fn populate_tmpdir(tmp: &Path) -> PopulatedTmp { + let root_with_git = tmp.join("with_git"); + let root_without_git = tmp.join("no_git"); + fs::create_dir_all(&root_with_git).expect("create with_git"); + fs::create_dir_all(&root_without_git).expect("create no_git"); + fs::create_dir_all(root_with_git.join(".git")).expect("create .git"); + + // Ensure we have canonical paths for -D parameter matching. + let root_with_git_canon = root_with_git.canonicalize().expect("canonicalize with_git"); + let root_with_git_git_canon = root_with_git_canon.join(".git"); + let root_without_git_canon = root_without_git + .canonicalize() + .expect("canonicalize no_git"); + PopulatedTmp { + root_with_git, + root_without_git, + root_with_git_canon, + root_with_git_git_canon, + root_without_git_canon, + } + } +} diff --git a/codex-rs/core/src/spawn.rs b/codex-rs/core/src/spawn.rs index 9fde26539b..1c82df3180 100644 --- a/codex-rs/core/src/spawn.rs +++ b/codex-rs/core/src/spawn.rs @@ -17,6 +17,11 @@ use crate::protocol::SandboxPolicy; /// attributes, so this may change in the future. pub const CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR: &str = "CODEX_SANDBOX_NETWORK_DISABLED"; +/// Should be set when the process is spawned under a sandbox. Currently, the +/// value is "seatbelt" for macOS, but it may change in the future to +/// accommodate sandboxing configuration and other sandboxing mechanisms. +pub const CODEX_SANDBOX_ENV_VAR: &str = "CODEX_SANDBOX"; + #[derive(Debug, Clone, Copy)] pub enum StdioPolicy { RedirectForShellTool, diff --git a/codex-rs/core/tests/sandbox.rs b/codex-rs/core/tests/sandbox.rs new file mode 100644 index 0000000000..e85156bf05 --- /dev/null +++ b/codex-rs/core/tests/sandbox.rs @@ -0,0 +1,195 @@ +#![cfg(target_os = "macos")] +#![expect(clippy::expect_used)] + +use std::collections::HashMap; +use std::path::Path; +use std::path::PathBuf; + +use codex_core::protocol::SandboxPolicy; +use codex_core::seatbelt::spawn_command_under_seatbelt; +use codex_core::spawn::CODEX_SANDBOX_ENV_VAR; +use codex_core::spawn::StdioPolicy; +use tempfile::TempDir; + +struct TestScenario { + repo_parent: PathBuf, + file_outside_repo: PathBuf, + repo_root: PathBuf, + file_in_repo_root: PathBuf, + file_in_dot_git_dir: PathBuf, +} + +struct TestExpectations { + file_outside_repo_is_writable: bool, + file_in_repo_root_is_writable: bool, + file_in_dot_git_dir_is_writable: bool, +} + +impl TestScenario { + async fn run_test(&self, policy: &SandboxPolicy, expectations: TestExpectations) { + if std::env::var(CODEX_SANDBOX_ENV_VAR) == Ok("seatbelt".to_string()) { + eprintln!("{CODEX_SANDBOX_ENV_VAR} is set to 'seatbelt', skipping test."); + return; + } + + assert_eq!( + touch(&self.file_outside_repo, policy).await, + expectations.file_outside_repo_is_writable + ); + assert_eq!( + self.file_outside_repo.exists(), + expectations.file_outside_repo_is_writable + ); + + assert_eq!( + touch(&self.file_in_repo_root, policy).await, + expectations.file_in_repo_root_is_writable + ); + assert_eq!( + self.file_in_repo_root.exists(), + expectations.file_in_repo_root_is_writable + ); + + assert_eq!( + touch(&self.file_in_dot_git_dir, policy).await, + expectations.file_in_dot_git_dir_is_writable + ); + assert_eq!( + self.file_in_dot_git_dir.exists(), + expectations.file_in_dot_git_dir_is_writable + ); + } +} + +/// If the user has added a workspace root that is not a Git repo root, then +/// the user has to specify `--skip-git-repo-check` or go through some +/// interstitial that indicates they are taking on some risk because Git +/// cannot be used to backup their work before the agent begins. +/// +/// Because the user has agreed to this risk, we do not try find all .git +/// folders in the workspace and block them (though we could change our +/// position on this in the future). +#[tokio::test] +async fn if_parent_of_repo_is_writable_then_dot_git_folder_is_writable() { + let tmp = TempDir::new().expect("should be able to create temp dir"); + let test_scenario = create_test_scenario(&tmp); + let policy = SandboxPolicy::WorkspaceWrite { + writable_roots: vec![test_scenario.repo_parent.clone()], + network_access: false, + include_default_writable_roots: false, + }; + + test_scenario + .run_test( + &policy, + TestExpectations { + file_outside_repo_is_writable: true, + file_in_repo_root_is_writable: true, + file_in_dot_git_dir_is_writable: true, + }, + ) + .await; +} + +/// When the writable root is the root of a Git repository (as evidenced by the +/// presence of a .git folder), then the .git folder should be read-only if +/// the policy is `WorkspaceWrite`. +#[tokio::test] +async fn if_git_repo_is_writable_root_then_dot_git_folder_is_read_only() { + let tmp = TempDir::new().expect("should be able to create temp dir"); + let test_scenario = create_test_scenario(&tmp); + let policy = SandboxPolicy::WorkspaceWrite { + writable_roots: vec![test_scenario.repo_root.clone()], + network_access: false, + include_default_writable_roots: false, + }; + + test_scenario + .run_test( + &policy, + TestExpectations { + file_outside_repo_is_writable: false, + file_in_repo_root_is_writable: true, + file_in_dot_git_dir_is_writable: false, + }, + ) + .await; +} + +/// Under DangerFullAccess, all writes should be permitted anywhere on disk, +/// including inside the .git folder. +#[tokio::test] +async fn danger_full_access_allows_all_writes() { + let tmp = TempDir::new().expect("should be able to create temp dir"); + let test_scenario = create_test_scenario(&tmp); + let policy = SandboxPolicy::DangerFullAccess; + + test_scenario + .run_test( + &policy, + TestExpectations { + file_outside_repo_is_writable: true, + file_in_repo_root_is_writable: true, + file_in_dot_git_dir_is_writable: true, + }, + ) + .await; +} + +/// Under ReadOnly, writes should not be permitted anywhere on disk. +#[tokio::test] +async fn read_only_forbids_all_writes() { + let tmp = TempDir::new().expect("should be able to create temp dir"); + let test_scenario = create_test_scenario(&tmp); + let policy = SandboxPolicy::ReadOnly; + + test_scenario + .run_test( + &policy, + TestExpectations { + file_outside_repo_is_writable: false, + file_in_repo_root_is_writable: false, + file_in_dot_git_dir_is_writable: false, + }, + ) + .await; +} + +fn create_test_scenario(tmp: &TempDir) -> TestScenario { + let repo_parent = tmp.path().to_path_buf(); + let repo_root = repo_parent.join("repo"); + let dot_git_dir = repo_root.join(".git"); + + std::fs::create_dir(&repo_root).expect("should be able to create repo root"); + std::fs::create_dir(&dot_git_dir).expect("should be able to create .git dir"); + + TestScenario { + file_outside_repo: repo_parent.join("outside.txt"), + repo_parent, + file_in_repo_root: repo_root.join("repo_file.txt"), + repo_root, + file_in_dot_git_dir: dot_git_dir.join("dot_git_file.txt"), + } +} + +/// Note that `path` must be absolute. +async fn touch(path: &Path, policy: &SandboxPolicy) -> bool { + assert!(path.is_absolute(), "Path must be absolute: {path:?}"); + let mut child = spawn_command_under_seatbelt( + vec![ + "/usr/bin/touch".to_string(), + path.to_string_lossy().to_string(), + ], + policy, + std::env::current_dir().expect("should be able to get current dir"), + StdioPolicy::RedirectForShellTool, + HashMap::new(), + ) + .await + .expect("should be able to spawn command under seatbelt"); + child + .wait() + .await + .expect("should be able to wait for child process") + .success() +} diff --git a/codex-rs/linux-sandbox/src/landlock.rs b/codex-rs/linux-sandbox/src/landlock.rs index 326e2cb487..e13e3c8b75 100644 --- a/codex-rs/linux-sandbox/src/landlock.rs +++ b/codex-rs/linux-sandbox/src/landlock.rs @@ -36,7 +36,11 @@ pub(crate) fn apply_sandbox_policy_to_current_thread( } if !sandbox_policy.has_full_disk_write_access() { - let writable_roots = sandbox_policy.get_writable_roots_with_cwd(cwd); + let writable_roots = sandbox_policy + .get_writable_roots_with_cwd(cwd) + .into_iter() + .map(|writable_root| writable_root.root) + .collect(); install_filesystem_landlock_rules_on_current_thread(writable_roots)?; } From 929ba50adce36c3b3e5cf0af42e38347a57bb09b Mon Sep 17 00:00:00 2001 From: pakrym-oai Date: Fri, 1 Aug 2025 16:30:15 -0700 Subject: [PATCH 042/100] Update succesfull login page look (#1789) --- codex-rs/login/src/login_with_chatgpt.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/codex-rs/login/src/login_with_chatgpt.py b/codex-rs/login/src/login_with_chatgpt.py index 2dbf5be58a..4c07feeba0 100644 --- a/codex-rs/login/src/login_with_chatgpt.py +++ b/codex-rs/login/src/login_with_chatgpt.py @@ -686,6 +686,7 @@ LOGIN_SUCCESS_HTML = """ justify-content: center; position: relative; background: white; + font-family: system-ui, -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, 'Open Sans', 'Helvetica Neue', sans-serif; } .inner-container { @@ -703,6 +704,7 @@ LOGIN_SUCCESS_HTML = """ align-items: center; gap: 20px; display: flex; + margin-top: 15vh; } .svg-wrapper { position: relative; @@ -710,9 +712,9 @@ LOGIN_SUCCESS_HTML = """ .title { text-align: center; color: var(--text-primary, #0D0D0D); - font-size: 28px; + font-size: 32px; font-weight: 400; - line-height: 36.40px; + line-height: 40px; word-wrap: break-word; } .setup-box { @@ -785,16 +787,26 @@ LOGIN_SUCCESS_HTML = """ word-wrap: break-word; text-decoration: none; } + .logo { + display: flex; + align-items: center; + justify-content: center; + width: 4rem; + height: 4rem; + border-radius: 16px; + border: .5px solid rgba(0, 0, 0, 0.1); + box-shadow: rgba(0, 0, 0, 0.1) 0px 4px 16px 0px; + box-sizing: border-box; + background-color: rgb(255, 255, 255); + }
-
- - - +
Signed in to Codex CLI
From 7e0f506da208d6ee9c289cc28b13532226d5abdb Mon Sep 17 00:00:00 2001 From: Jeremy Rose <172423086+nornagon-openai@users.noreply.github.com> Date: Fri, 1 Aug 2025 17:31:38 -0700 Subject: [PATCH 043/100] check for updates (#1764) 1. Ping https://api.github.com/repos/openai/codex/releases/latest (at most once every 20 hrs) 2. Store the result in ~/.codex/version.jsonl 3. If CARGO_PKG_VERSION < latest_version, print a message at boot. --------- Co-authored-by: easong-openai --- codex-cli/bin/codex.js | 1 + codex-rs/Cargo.lock | 3 + codex-rs/tui/Cargo.toml | 3 + codex-rs/tui/src/lib.rs | 37 ++++++++++ codex-rs/tui/src/updates.rs | 137 ++++++++++++++++++++++++++++++++++++ 5 files changed, 181 insertions(+) create mode 100644 codex-rs/tui/src/updates.rs diff --git a/codex-cli/bin/codex.js b/codex-cli/bin/codex.js index ae1fb9593c..df06dd36a7 100755 --- a/codex-cli/bin/codex.js +++ b/codex-cli/bin/codex.js @@ -83,6 +83,7 @@ if (wantsNative && process.platform !== 'win32') { const child = spawn(binaryPath, process.argv.slice(2), { stdio: "inherit", + env: { ...process.env, CODEX_MANAGED_BY_NPM: "1" }, }); child.on("error", (err) => { diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 460a440e63..d71553cf4a 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -843,6 +843,7 @@ version = "0.0.0" dependencies = [ "anyhow", "base64 0.22.1", + "chrono", "clap", "codex-ansi-escape", "codex-arg0", @@ -861,6 +862,8 @@ dependencies = [ "ratatui", "ratatui-image", "regex-lite", + "reqwest", + "serde", "serde_json", "shlex", "strum 0.27.2", diff --git a/codex-rs/tui/Cargo.toml b/codex-rs/tui/Cargo.toml index 468f2f3be6..09b537c6c3 100644 --- a/codex-rs/tui/Cargo.toml +++ b/codex-rs/tui/Cargo.toml @@ -17,6 +17,7 @@ workspace = true [dependencies] anyhow = "1" base64 = "0.22.1" +chrono = { version = "0.4", features = ["serde"] } clap = { version = "4", features = ["derive"] } codex-ansi-escape = { path = "../ansi-escape" } codex-arg0 = { path = "../arg0" } @@ -41,6 +42,8 @@ ratatui = { version = "0.29.0", features = [ ] } ratatui-image = "8.0.0" regex-lite = "0.1" +reqwest = { version = "0.12", features = ["json"] } +serde = { version = "1", features = ["derive"] } serde_json = { version = "1", features = ["preserve_order"] } shlex = "1.3.0" strum = "0.27.2" diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index f0a0e9d833..0ec9be6153 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -41,6 +41,11 @@ mod text_formatting; mod tui; mod user_approval_widget; +#[cfg(not(debug_assertions))] +mod updates; +#[cfg(not(debug_assertions))] +use color_eyre::owo_colors::OwoColorize; + pub use cli::Cli; pub async fn run_main( @@ -139,6 +144,38 @@ pub async fn run_main( .with(tui_layer) .try_init(); + #[allow(clippy::print_stderr)] + #[cfg(not(debug_assertions))] + if let Some(latest_version) = updates::get_upgrade_version(&config) { + let current_version = env!("CARGO_PKG_VERSION"); + let exe = std::env::current_exe()?; + let managed_by_npm = std::env::var_os("CODEX_MANAGED_BY_NPM").is_some(); + + eprintln!( + "{} {current_version} -> {latest_version}.", + "✨⬆️ Update available!".bold().cyan() + ); + + if managed_by_npm { + let npm_cmd = "npm install -g @openai/codex@latest"; + eprintln!("Run {} to update.", npm_cmd.cyan().on_black()); + } else if cfg!(target_os = "macos") + && (exe.starts_with("/opt/homebrew") || exe.starts_with("/usr/local")) + { + let brew_cmd = "brew upgrade codex"; + eprintln!("Run {} to update.", brew_cmd.cyan().on_black()); + } else { + eprintln!( + "See {} for the latest releases and installation options.", + "https://github.com/openai/codex/releases/latest" + .cyan() + .on_black() + ); + } + + eprintln!(""); + } + let show_login_screen = should_show_login_screen(&config); if show_login_screen { std::io::stdout() diff --git a/codex-rs/tui/src/updates.rs b/codex-rs/tui/src/updates.rs new file mode 100644 index 0000000000..c7f7afd2a5 --- /dev/null +++ b/codex-rs/tui/src/updates.rs @@ -0,0 +1,137 @@ +#![cfg(any(not(debug_assertions), test))] + +use chrono::DateTime; +use chrono::Duration; +use chrono::Utc; +use serde::Deserialize; +use serde::Serialize; +use std::path::Path; +use std::path::PathBuf; + +use codex_core::config::Config; + +pub fn get_upgrade_version(config: &Config) -> Option { + let version_file = version_filepath(config); + let info = read_version_info(&version_file).ok(); + + if match &info { + None => true, + Some(info) => info.last_checked_at < Utc::now() - Duration::hours(20), + } { + // Refresh the cached latest version in the background so TUI startup + // isn’t blocked by a network call. The UI reads the previously cached + // value (if any) for this run; the next run shows the banner if needed. + tokio::spawn(async move { + check_for_update(&version_file) + .await + .inspect_err(|e| tracing::error!("Failed to update version: {e}")) + }); + } + + info.and_then(|info| { + let current_version = env!("CARGO_PKG_VERSION"); + if is_newer(&info.latest_version, current_version).unwrap_or(false) { + Some(info.latest_version) + } else { + None + } + }) +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +struct VersionInfo { + latest_version: String, + // ISO-8601 timestamp (RFC3339) + last_checked_at: DateTime, +} + +#[derive(Deserialize, Debug, Clone)] +struct ReleaseInfo { + tag_name: String, +} + +const VERSION_FILENAME: &str = "version.json"; +const LATEST_RELEASE_URL: &str = "https://api.github.com/repos/openai/codex/releases/latest"; + +fn version_filepath(config: &Config) -> PathBuf { + config.codex_home.join(VERSION_FILENAME) +} + +fn read_version_info(version_file: &Path) -> anyhow::Result { + let contents = std::fs::read_to_string(version_file)?; + Ok(serde_json::from_str(&contents)?) +} + +async fn check_for_update(version_file: &Path) -> anyhow::Result<()> { + let ReleaseInfo { + tag_name: latest_tag_name, + } = reqwest::Client::new() + .get(LATEST_RELEASE_URL) + .header( + "User-Agent", + format!( + "codex/{} (+https://github.com/openai/codex)", + env!("CARGO_PKG_VERSION") + ), + ) + .send() + .await? + .error_for_status()? + .json::() + .await?; + + let info = VersionInfo { + latest_version: latest_tag_name + .strip_prefix("rust-v") + .ok_or_else(|| anyhow::anyhow!("Failed to parse latest tag name '{latest_tag_name}'"))? + .into(), + last_checked_at: Utc::now(), + }; + + let json_line = format!("{}\n", serde_json::to_string(&info)?); + if let Some(parent) = version_file.parent() { + tokio::fs::create_dir_all(parent).await?; + } + tokio::fs::write(version_file, json_line).await?; + Ok(()) +} + +fn is_newer(latest: &str, current: &str) -> Option { + match (parse_version(latest), parse_version(current)) { + (Some(l), Some(c)) => Some(l > c), + _ => None, + } +} + +fn parse_version(v: &str) -> Option<(u64, u64, u64)> { + let mut iter = v.trim().split('.'); + let maj = iter.next()?.parse::().ok()?; + let min = iter.next()?.parse::().ok()?; + let pat = iter.next()?.parse::().ok()?; + Some((maj, min, pat)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn prerelease_version_is_not_considered_newer() { + assert_eq!(is_newer("0.11.0-beta.1", "0.11.0"), None); + assert_eq!(is_newer("1.0.0-rc.1", "1.0.0"), None); + } + + #[test] + fn plain_semver_comparisons_work() { + assert_eq!(is_newer("0.11.1", "0.11.0"), Some(true)); + assert_eq!(is_newer("0.11.0", "0.11.1"), Some(false)); + assert_eq!(is_newer("1.0.0", "0.9.9"), Some(true)); + assert_eq!(is_newer("0.9.9", "1.0.0"), Some(false)); + } + + #[test] + fn whitespace_is_ignored() { + assert_eq!(parse_version(" 1.2.3 \n"), Some((1, 2, 3))); + assert_eq!(is_newer(" 1.2.3 ", "1.2.2"), Some(true)); + } +} From 81bb1c9e264095708a01f6326bf8d527a6b2d47b Mon Sep 17 00:00:00 2001 From: aibrahim-oai Date: Sat, 2 Aug 2025 12:05:06 -0700 Subject: [PATCH 044/100] Fix compact (#1798) We are not recording the summary in the history. --- codex-rs/core/src/codex.rs | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 6e21642da1..e759acb4a2 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -1373,7 +1373,7 @@ async fn run_compact_task( let mut retries = 0; loop { - let attempt_result = drain_to_completed(&sess, &prompt).await; + let attempt_result = drain_to_completed(&sess, &sub_id, &prompt).await; match attempt_result { Ok(()) => break, @@ -2001,7 +2001,7 @@ fn get_last_assistant_message_from_turn(responses: &[ResponseItem]) -> Option CodexResult<()> { +async fn drain_to_completed(sess: &Session, sub_id: &str, prompt: &Prompt) -> CodexResult<()> { let mut stream = sess.client.clone().stream(prompt).await?; loop { let maybe_event = stream.next().await; @@ -2011,7 +2011,32 @@ async fn drain_to_completed(sess: &Session, prompt: &Prompt) -> CodexResult<()> )); }; match event { - Ok(ResponseEvent::Completed { .. }) => return Ok(()), + Ok(ResponseEvent::OutputItemDone(item)) => { + // Record only to in-memory conversation history; avoid state snapshot. + let mut state = sess.state.lock().unwrap(); + state.history.record_items(std::slice::from_ref(&item)); + } + Ok(ResponseEvent::Completed { + response_id: _, + token_usage, + }) => { + let token_usage = match token_usage { + Some(usage) => usage, + None => { + return Err(CodexErr::Stream( + "token_usage was None in ResponseEvent::Completed".into(), + )); + } + }; + sess.tx_event + .send(Event { + id: sub_id.to_string(), + msg: EventMsg::TokenCount(token_usage), + }) + .await + .ok(); + return Ok(()); + } Ok(_) => continue, Err(e) => return Err(e), } From 75eecb656e227f5b02924cabd4a98ed798f294d8 Mon Sep 17 00:00:00 2001 From: David Z Hao Date: Sun, 3 Aug 2025 06:59:26 -0700 Subject: [PATCH 045/100] Fix MacOS multiprocessing by relaxing sandbox (#1808) The following test script fails in the codex sandbox: ``` import multiprocessing from multiprocessing import Lock, Process def f(lock): with lock: print("Lock acquired in child process") if __name__ == '__main__': lock = Lock() p = Process(target=f, args=(lock,)) p.start() p.join() ``` with ``` Traceback (most recent call last): File "/Users/david.hao/code/codex/codex-rs/cli/test.py", line 9, in lock = Lock() ^^^^^^ File "/Users/david.hao/.local/share/uv/python/cpython-3.12.9-macos-aarch64-none/lib/python3.12/multiprocessing/context.py", line 68, in Lock return Lock(ctx=self.get_context()) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/Users/david.hao/.local/share/uv/python/cpython-3.12.9-macos-aarch64-none/lib/python3.12/multiprocessing/synchronize.py", line 169, in __init__ SemLock.__init__(self, SEMAPHORE, 1, 1, ctx=ctx) File "/Users/david.hao/.local/share/uv/python/cpython-3.12.9-macos-aarch64-none/lib/python3.12/multiprocessing/synchronize.py", line 57, in __init__ sl = self._semlock = _multiprocessing.SemLock( ^^^^^^^^^^^^^^^^^^^^^^^^^ PermissionError: [Errno 1] Operation not permitted ``` After reading, adding this line to the sandbox configs fixes things - MacOS multiprocessing appears to use sem_lock(), which opens an IPC which is considered a disk write even though no file is created. I interrogated ChatGPT about whether it's okay to loosen, and my impression after reading is that it is, although would appreciate a close look Breadcrumb: You can run `cargo run -- debug seatbelt --full-auto ` to test the sandbox --- codex-cli/src/utils/agent/sandbox/macos-seatbelt.ts | 6 +++++- codex-rs/core/src/seatbelt_base_policy.sbpl | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/codex-cli/src/utils/agent/sandbox/macos-seatbelt.ts b/codex-cli/src/utils/agent/sandbox/macos-seatbelt.ts index 290e6b5630..162fe908c4 100644 --- a/codex-cli/src/utils/agent/sandbox/macos-seatbelt.ts +++ b/codex-cli/src/utils/agent/sandbox/macos-seatbelt.ts @@ -147,4 +147,8 @@ const READ_ONLY_SEATBELT_POLICY = ` (sysctl-name "kern.version") (sysctl-name "sysctl.proc_cputype") (sysctl-name-prefix "hw.perflevel") -)`.trim(); +) + +; Added on top of Chrome profile +; Needed for python multiprocessing on MacOS for the SemLock +(allow ipc-posix-sem)`.trim(); diff --git a/codex-rs/core/src/seatbelt_base_policy.sbpl b/codex-rs/core/src/seatbelt_base_policy.sbpl index c9664651c2..b250494836 100644 --- a/codex-rs/core/src/seatbelt_base_policy.sbpl +++ b/codex-rs/core/src/seatbelt_base_policy.sbpl @@ -65,3 +65,7 @@ (sysctl-name "sysctl.proc_cputype") (sysctl-name-prefix "hw.perflevel") ) + +; Added on top of Chrome profile +; Needed for python multiprocessing on MacOS for the SemLock +(allow ipc-posix-sem) From 4c9f7b6bcc4a135162e792b9adb2b245f0c4976d Mon Sep 17 00:00:00 2001 From: Gabriel Peal Date: Sun, 3 Aug 2025 07:19:12 -0700 Subject: [PATCH 046/100] Fix flaky test_shell_command_approval_triggers_elicitation test (#1802) This doesn't flake very often but this should fix it. --- codex-rs/mcp-server/src/exec_approval.rs | 2 +- codex-rs/mcp-server/tests/codex_tool.rs | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/codex-rs/mcp-server/src/exec_approval.rs b/codex-rs/mcp-server/src/exec_approval.rs index f073214bf5..54abfb35bd 100644 --- a/codex-rs/mcp-server/src/exec_approval.rs +++ b/codex-rs/mcp-server/src/exec_approval.rs @@ -18,7 +18,7 @@ use crate::codex_tool_runner::INVALID_PARAMS_ERROR_CODE; /// Conforms to [`mcp_types::ElicitRequestParams`] so that it can be used as the /// `params` field of an [`ElicitRequest`]. -#[derive(Debug, Serialize)] +#[derive(Debug, Deserialize, Serialize)] pub struct ExecApprovalElicitRequestParams { // These fields are required so that `params` // conforms to ElicitRequestParams. diff --git a/codex-rs/mcp-server/tests/codex_tool.rs b/codex-rs/mcp-server/tests/codex_tool.rs index 7d59ff202e..fc992a8cd1 100644 --- a/codex-rs/mcp-server/tests/codex_tool.rs +++ b/codex-rs/mcp-server/tests/codex_tool.rs @@ -89,14 +89,18 @@ async fn shell_command_approval_triggers_elicitation() -> anyhow::Result<()> { // This is the first request from the server, so the id should be 0 given // how things are currently implemented. let elicitation_request_id = RequestId::Integer(0); + let params = serde_json::from_value::( + elicitation_request + .params + .clone() + .ok_or_else(|| anyhow::anyhow!("elicitation_request.params must be set"))?, + )?; let expected_elicitation_request = create_expected_elicitation_request( elicitation_request_id.clone(), shell_command.clone(), workdir_for_shell_function_call.path(), codex_request_id.to_string(), - // Internal Codex id: empirically it is 1, but this is - // admittedly an internal detail that could change. - "1".to_string(), + params.codex_event_id.clone(), )?; assert_eq!(expected_elicitation_request, elicitation_request); From d62b703a211f6e06fc81b3f96671f431e1e59252 Mon Sep 17 00:00:00 2001 From: Jeremy Rose <172423086+nornagon-openai@users.noreply.github.com> Date: Sun, 3 Aug 2025 11:31:35 -0700 Subject: [PATCH 047/100] custom textarea (#1794) This replaces tui-textarea with a custom textarea component. Key differences: 1. wrapped lines 2. better unicode handling 3. uses the native terminal cursor This should perhaps be spun out into its own separate crate at some point, but for now it's convenient to have it in-tree. --- codex-rs/Cargo.lock | 39 +- codex-rs/tui/Cargo.toml | 4 +- codex-rs/tui/src/app.rs | 13 +- codex-rs/tui/src/bottom_pane/chat_composer.rs | 637 ++++---- .../src/bottom_pane/chat_composer_history.rs | 91 +- codex-rs/tui/src/bottom_pane/mod.rs | 17 +- codex-rs/tui/src/bottom_pane/textarea.rs | 1294 +++++++++++++++++ codex-rs/tui/src/chatwidget.rs | 4 + 8 files changed, 1690 insertions(+), 409 deletions(-) create mode 100644 codex-rs/tui/src/bottom_pane/textarea.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index d71553cf4a..e1a0e162dc 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -859,6 +859,7 @@ dependencies = [ "mcp-types", "path-clean", "pretty_assertions", + "rand 0.8.5", "ratatui", "ratatui-image", "regex-lite", @@ -868,13 +869,13 @@ dependencies = [ "shlex", "strum 0.27.2", "strum_macros 0.27.2", + "textwrap 0.16.2", "tokio", "tracing", "tracing-appender", "tracing-subscriber", "tui-input", "tui-markdown", - "tui-textarea", "unicode-segmentation", "unicode-width 0.1.14", "uuid", @@ -4173,6 +4174,12 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +[[package]] +name = "smawk" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c" + [[package]] name = "socket2" version = "0.5.10" @@ -4235,7 +4242,7 @@ dependencies = [ "starlark_syntax", "static_assertions", "strsim 0.10.0", - "textwrap", + "textwrap 0.11.0", "thiserror 1.0.69", ] @@ -4524,6 +4531,17 @@ dependencies = [ "unicode-width 0.1.14", ] +[[package]] +name = "textwrap" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" +dependencies = [ + "smawk", + "unicode-linebreak", + "unicode-width 0.2.0", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -4988,17 +5006,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "tui-textarea" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a5318dd619ed73c52a9417ad19046724effc1287fb75cdcc4eca1d6ac1acbae" -dependencies = [ - "crossterm", - "ratatui", - "unicode-width 0.2.0", -] - [[package]] name = "typenum" version = "1.18.0" @@ -5017,6 +5024,12 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" +[[package]] +name = "unicode-linebreak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" + [[package]] name = "unicode-segmentation" version = "1.12.0" diff --git a/codex-rs/tui/Cargo.toml b/codex-rs/tui/Cargo.toml index 09b537c6c3..823fd1428e 100644 --- a/codex-rs/tui/Cargo.toml +++ b/codex-rs/tui/Cargo.toml @@ -48,6 +48,7 @@ serde_json = { version = "1", features = ["preserve_order"] } shlex = "1.3.0" strum = "0.27.2" strum_macros = "0.27.2" +textwrap = "0.16.2" tokio = { version = "1", features = [ "io-std", "macros", @@ -60,7 +61,6 @@ tracing-appender = "0.2.3" tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } tui-input = "0.14.0" tui-markdown = "0.3.3" -tui-textarea = "0.7.0" unicode-segmentation = "1.12.0" unicode-width = "0.1" uuid = "1" @@ -70,3 +70,5 @@ uuid = "1" [dev-dependencies] insta = "1.43.1" pretty_assertions = "1" +rand = "0.8" +chrono = { version = "0.4", features = ["serde"] } diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index efaec7f19a..1142bd87fc 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -438,14 +438,15 @@ impl App<'_> { ); self.pending_history_lines.clear(); } - match &mut self.app_state { + terminal.draw(|frame| match &mut self.app_state { AppState::Chat { widget } => { - terminal.draw(|frame| frame.render_widget_ref(&**widget, frame.area()))?; + if let Some((x, y)) = widget.cursor_pos(frame.area()) { + frame.set_cursor_position((x, y)); + } + frame.render_widget_ref(&**widget, frame.area()) } - AppState::GitWarning { screen } => { - terminal.draw(|frame| frame.render_widget_ref(&*screen, frame.area()))?; - } - } + AppState::GitWarning { screen } => frame.render_widget_ref(&*screen, frame.area()), + })?; Ok(()) } diff --git a/codex-rs/tui/src/bottom_pane/chat_composer.rs b/codex-rs/tui/src/bottom_pane/chat_composer.rs index 7e187bec8b..c9ad719771 100644 --- a/codex-rs/tui/src/bottom_pane/chat_composer.rs +++ b/codex-rs/tui/src/bottom_pane/chat_composer.rs @@ -1,6 +1,11 @@ use codex_core::protocol::TokenUsage; +use crossterm::event::KeyCode; use crossterm::event::KeyEvent; +use crossterm::event::KeyModifiers; use ratatui::buffer::Buffer; +use ratatui::layout::Constraint; +use ratatui::layout::Layout; +use ratatui::layout::Margin; use ratatui::layout::Rect; use ratatui::style::Color; use ratatui::style::Style; @@ -8,13 +13,11 @@ use ratatui::style::Styled; use ratatui::style::Stylize; use ratatui::text::Line; use ratatui::text::Span; +use ratatui::widgets::Block; use ratatui::widgets::BorderType; use ratatui::widgets::Borders; -use ratatui::widgets::Widget; +use ratatui::widgets::StatefulWidgetRef; use ratatui::widgets::WidgetRef; -use tui_textarea::Input; -use tui_textarea::Key; -use tui_textarea::TextArea; use super::chat_composer_history::ChatComposerHistory; use super::command_popup::CommandPopup; @@ -22,7 +25,10 @@ use super::file_search_popup::FileSearchPopup; use crate::app_event::AppEvent; use crate::app_event_sender::AppEventSender; +use crate::bottom_pane::textarea::TextArea; +use crate::bottom_pane::textarea::TextAreaState; use codex_file_search::FileMatch; +use std::cell::RefCell; const BASE_PLACEHOLDER_TEXT: &str = "..."; /// If the pasted content exceeds this number of characters, replace it with a @@ -35,8 +41,14 @@ pub enum InputResult { None, } -pub(crate) struct ChatComposer<'a> { - textarea: TextArea<'a>, +struct TokenUsageInfo { + token_usage: TokenUsage, + model_context_window: Option, +} + +pub(crate) struct ChatComposer { + textarea: TextArea, + textarea_state: RefCell, active_popup: ActivePopup, app_event_tx: AppEventSender, history: ChatComposerHistory, @@ -45,6 +57,8 @@ pub(crate) struct ChatComposer<'a> { dismissed_file_popup_token: Option, current_file_query: Option, pending_pastes: Vec<(String, String)>, + token_usage_info: Option, + has_focus: bool, } /// Popup state – at most one can be visible at any time. @@ -54,20 +68,17 @@ enum ActivePopup { File(FileSearchPopup), } -impl ChatComposer<'_> { +impl ChatComposer { pub fn new( has_input_focus: bool, app_event_tx: AppEventSender, enhanced_keys_supported: bool, ) -> Self { - let mut textarea = TextArea::default(); - textarea.set_placeholder_text(BASE_PLACEHOLDER_TEXT); - textarea.set_cursor_line_style(ratatui::style::Style::default()); - let use_shift_enter_hint = enhanced_keys_supported; - let mut this = Self { - textarea, + Self { + textarea: TextArea::new(), + textarea_state: RefCell::new(TextAreaState::default()), active_popup: ActivePopup::None, app_event_tx, history: ChatComposerHistory::new(), @@ -76,13 +87,13 @@ impl ChatComposer<'_> { dismissed_file_popup_token: None, current_file_query: None, pending_pastes: Vec::new(), - }; - this.update_border(has_input_focus); - this + token_usage_info: None, + has_focus: has_input_focus, + } } - pub fn desired_height(&self) -> u16 { - self.textarea.lines().len().max(1) as u16 + pub fn desired_height(&self, width: u16) -> u16 { + self.textarea.desired_height(width - 1) + match &self.active_popup { ActivePopup::None => 1u16, ActivePopup::Command(c) => c.calculate_required_height(), @@ -90,6 +101,21 @@ impl ChatComposer<'_> { } } + pub fn cursor_pos(&self, area: Rect) -> Option<(u16, u16)> { + let popup_height = match &self.active_popup { + ActivePopup::Command(popup) => popup.calculate_required_height(), + ActivePopup::File(popup) => popup.calculate_required_height(), + ActivePopup::None => 1, + }; + let [textarea_rect, _] = + Layout::vertical([Constraint::Min(0), Constraint::Max(popup_height)]).areas(area); + let mut textarea_rect = textarea_rect; + textarea_rect.width = textarea_rect.width.saturating_sub(1); + textarea_rect.x += 1; + let state = self.textarea_state.borrow(); + self.textarea.cursor_pos_with_state(textarea_rect, &state) + } + /// Returns true if the composer currently contains no user input. pub(crate) fn is_empty(&self) -> bool { self.textarea.is_empty() @@ -103,28 +129,10 @@ impl ChatComposer<'_> { token_usage: TokenUsage, model_context_window: Option, ) { - let placeholder = match (token_usage.total_tokens, model_context_window) { - (total_tokens, Some(context_window)) => { - let percent_remaining: u8 = if context_window > 0 { - // Calculate the percentage of context left. - let percent = 100.0 - (total_tokens as f32 / context_window as f32 * 100.0); - percent.clamp(0.0, 100.0) as u8 - } else { - // If we don't have a context window, we cannot compute the - // percentage. - 100 - }; - // When https://github.com/openai/codex/issues/1257 is resolved, - // check if `percent_remaining < 25`, and if so, recommend - // /compact. - format!("{BASE_PLACEHOLDER_TEXT} — {percent_remaining}% context left") - } - (total_tokens, None) => { - format!("{BASE_PLACEHOLDER_TEXT} — {total_tokens} tokens used") - } - }; - - self.textarea.set_placeholder_text(placeholder); + self.token_usage_info = Some(TokenUsageInfo { + token_usage, + model_context_window, + }); } /// Record the history metadata advertised by `SessionConfiguredEvent` so @@ -142,8 +150,12 @@ impl ChatComposer<'_> { offset: usize, entry: Option, ) -> bool { - self.history - .on_entry_response(log_id, offset, entry, &mut self.textarea) + let Some(text) = self.history.on_entry_response(log_id, offset, entry) else { + return false; + }; + self.textarea.set_text(&text); + self.textarea.set_cursor(0); + true } pub fn handle_paste(&mut self, pasted: String) -> bool { @@ -179,7 +191,7 @@ impl ChatComposer<'_> { pub fn set_ctrl_c_quit_hint(&mut self, show: bool, has_focus: bool) { self.ctrl_c_quit_hint = show; - self.update_border(has_focus); + self.set_has_focus(has_focus); } /// Handle a key event coming from the main UI. @@ -207,49 +219,47 @@ impl ChatComposer<'_> { unreachable!(); }; - match key_event.into() { - Input { key: Key::Up, .. } => { + match key_event { + KeyEvent { + code: KeyCode::Up, .. + } => { popup.move_up(); (InputResult::None, true) } - Input { key: Key::Down, .. } => { + KeyEvent { + code: KeyCode::Down, + .. + } => { popup.move_down(); (InputResult::None, true) } - Input { key: Key::Tab, .. } => { + KeyEvent { + code: KeyCode::Tab, .. + } => { if let Some(cmd) = popup.selected_command() { - let first_line = self - .textarea - .lines() - .first() - .map(|s| s.as_str()) - .unwrap_or(""); + let first_line = self.textarea.text().lines().next().unwrap_or(""); let starts_with_cmd = first_line .trim_start() .starts_with(&format!("/{}", cmd.command())); if !starts_with_cmd { - self.textarea.select_all(); - self.textarea.cut(); - let _ = self.textarea.insert_str(format!("/{} ", cmd.command())); + self.textarea.set_text(&format!("/{} ", cmd.command())); } } (InputResult::None, true) } - Input { - key: Key::Enter, - shift: false, - alt: false, - ctrl: false, + KeyEvent { + code: KeyCode::Enter, + modifiers: KeyModifiers::NONE, + .. } => { if let Some(cmd) = popup.selected_command() { // Send command to the app layer. self.app_event_tx.send(AppEvent::DispatchCommand(*cmd)); // Clear textarea so no residual text remains. - self.textarea.select_all(); - self.textarea.cut(); + self.textarea.set_text(""); // Hide popup since the command has been dispatched. self.active_popup = ActivePopup::None; @@ -268,16 +278,23 @@ impl ChatComposer<'_> { unreachable!(); }; - match key_event.into() { - Input { key: Key::Up, .. } => { + match key_event { + KeyEvent { + code: KeyCode::Up, .. + } => { popup.move_up(); (InputResult::None, true) } - Input { key: Key::Down, .. } => { + KeyEvent { + code: KeyCode::Down, + .. + } => { popup.move_down(); (InputResult::None, true) } - Input { key: Key::Esc, .. } => { + KeyEvent { + code: KeyCode::Esc, .. + } => { // Hide popup without modifying text, remember token to avoid immediate reopen. if let Some(tok) = Self::current_at_token(&self.textarea) { self.dismissed_file_popup_token = Some(tok.to_string()); @@ -285,12 +302,13 @@ impl ChatComposer<'_> { self.active_popup = ActivePopup::None; (InputResult::None, true) } - Input { key: Key::Tab, .. } - | Input { - key: Key::Enter, - ctrl: false, - alt: false, - shift: false, + KeyEvent { + code: KeyCode::Tab, .. + } + | KeyEvent { + code: KeyCode::Enter, + modifiers: KeyModifiers::NONE, + .. } => { if let Some(sel) = popup.selected_match() { let sel_path = sel.to_string(); @@ -315,46 +333,89 @@ impl ChatComposer<'_> { /// - A token is delimited by ASCII whitespace (space, tab, newline). /// - If the token under the cursor starts with `@` and contains at least /// one additional character, that token (without `@`) is returned. - fn current_at_token(textarea: &tui_textarea::TextArea) -> Option { - let (row, col) = textarea.cursor(); + fn current_at_token(textarea: &TextArea) -> Option { + let cursor_offset = textarea.cursor(); + let text = textarea.text(); - // Guard against out-of-bounds rows. - let line = textarea.lines().get(row)?.as_str(); + // Adjust the provided byte offset to the nearest valid char boundary at or before it. + let mut safe_cursor = cursor_offset.min(text.len()); + // If we're not on a char boundary, move back to the start of the current char. + if safe_cursor < text.len() && !text.is_char_boundary(safe_cursor) { + // Find the last valid boundary <= cursor_offset. + safe_cursor = text + .char_indices() + .map(|(i, _)| i) + .take_while(|&i| i <= cursor_offset) + .last() + .unwrap_or(0); + } - // Calculate byte offset for cursor position - let cursor_byte_offset = line.chars().take(col).map(|c| c.len_utf8()).sum::(); + // Split the line around the (now safe) cursor position. + let before_cursor = &text[..safe_cursor]; + let after_cursor = &text[safe_cursor..]; - // Split the line at the cursor position so we can search for word - // boundaries on both sides. - let before_cursor = &line[..cursor_byte_offset]; - let after_cursor = &line[cursor_byte_offset..]; + // Detect whether we're on whitespace at the cursor boundary. + let at_whitespace = if safe_cursor < text.len() { + text[safe_cursor..] + .chars() + .next() + .map(|c| c.is_whitespace()) + .unwrap_or(false) + } else { + false + }; - // Find start index (first character **after** the previous multi-byte whitespace). - let start_idx = before_cursor + // Left candidate: token containing the cursor position. + let start_left = before_cursor .char_indices() .rfind(|(_, c)| c.is_whitespace()) .map(|(idx, c)| idx + c.len_utf8()) .unwrap_or(0); - - // Find end index (first multi-byte whitespace **after** the cursor position). - let end_rel_idx = after_cursor + let end_left_rel = after_cursor .char_indices() .find(|(_, c)| c.is_whitespace()) .map(|(idx, _)| idx) .unwrap_or(after_cursor.len()); - let end_idx = cursor_byte_offset + end_rel_idx; - - if start_idx >= end_idx { - return None; - } - - let token = &line[start_idx..end_idx]; - - if token.starts_with('@') && token.len() > 1 { - Some(token[1..].to_string()) + let end_left = safe_cursor + end_left_rel; + let token_left = if start_left < end_left { + Some(&text[start_left..end_left]) } else { None + }; + + // Right candidate: token immediately after any whitespace from the cursor. + let ws_len_right: usize = after_cursor + .chars() + .take_while(|c| c.is_whitespace()) + .map(|c| c.len_utf8()) + .sum(); + let start_right = safe_cursor + ws_len_right; + let end_right_rel = text[start_right..] + .char_indices() + .find(|(_, c)| c.is_whitespace()) + .map(|(idx, _)| idx) + .unwrap_or(text.len() - start_right); + let end_right = start_right + end_right_rel; + let token_right = if start_right < end_right { + Some(&text[start_right..end_right]) + } else { + None + }; + + let left_at = token_left + .filter(|t| t.starts_with('@') && t.len() > 1) + .map(|t| t[1..].to_string()); + let right_at = token_right + .filter(|t| t.starts_with('@') && t.len() > 1) + .map(|t| t[1..].to_string()); + + if at_whitespace { + return right_at.or(left_at); } + if after_cursor.starts_with('@') { + return right_at.or(left_at); + } + left_at.or(right_at) } /// Replace the active `@token` (the one under the cursor) with `path`. @@ -363,94 +424,73 @@ impl ChatComposer<'_> { /// where the cursor is within the token and regardless of how many /// `@tokens` exist in the line. fn insert_selected_path(&mut self, path: &str) { - let (row, col) = self.textarea.cursor(); + let cursor_offset = self.textarea.cursor(); + let text = self.textarea.text(); - // Materialize the textarea lines so we can mutate them easily. - let mut lines: Vec = self.textarea.lines().to_vec(); + let before_cursor = &text[..cursor_offset]; + let after_cursor = &text[cursor_offset..]; - if let Some(line) = lines.get_mut(row) { - // Calculate byte offset for cursor position - let cursor_byte_offset = line.chars().take(col).map(|c| c.len_utf8()).sum::(); + // Determine token boundaries. + let start_idx = before_cursor + .char_indices() + .rfind(|(_, c)| c.is_whitespace()) + .map(|(idx, c)| idx + c.len_utf8()) + .unwrap_or(0); - let before_cursor = &line[..cursor_byte_offset]; - let after_cursor = &line[cursor_byte_offset..]; + let end_rel_idx = after_cursor + .char_indices() + .find(|(_, c)| c.is_whitespace()) + .map(|(idx, _)| idx) + .unwrap_or(after_cursor.len()); + let end_idx = cursor_offset + end_rel_idx; - // Determine token boundaries. - let start_idx = before_cursor - .char_indices() - .rfind(|(_, c)| c.is_whitespace()) - .map(|(idx, c)| idx + c.len_utf8()) - .unwrap_or(0); + // Replace the slice `[start_idx, end_idx)` with the chosen path and a trailing space. + let mut new_text = + String::with_capacity(text.len() - (end_idx - start_idx) + path.len() + 1); + new_text.push_str(&text[..start_idx]); + new_text.push_str(path); + new_text.push(' '); + new_text.push_str(&text[end_idx..]); - let end_rel_idx = after_cursor - .char_indices() - .find(|(_, c)| c.is_whitespace()) - .map(|(idx, _)| idx) - .unwrap_or(after_cursor.len()); - let end_idx = cursor_byte_offset + end_rel_idx; - - // Replace the slice `[start_idx, end_idx)` with the chosen path and a trailing space. - let mut new_line = - String::with_capacity(line.len() - (end_idx - start_idx) + path.len() + 1); - new_line.push_str(&line[..start_idx]); - new_line.push_str(path); - new_line.push(' '); - new_line.push_str(&line[end_idx..]); - - *line = new_line; - - // Re-populate the textarea. - let new_text = lines.join("\n"); - self.textarea.select_all(); - self.textarea.cut(); - let _ = self.textarea.insert_str(new_text); - - // Note: tui-textarea currently exposes only relative cursor - // movements. Leaving the cursor position unchanged is acceptable - // as subsequent typing will move the cursor naturally. - } + self.textarea.set_text(&new_text); } /// Handle key event when no popup is visible. fn handle_key_event_without_popup(&mut self, key_event: KeyEvent) -> (InputResult, bool) { - let input: Input = key_event.into(); - match input { + match key_event { // ------------------------------------------------------------- // History navigation (Up / Down) – only when the composer is not // empty or when the cursor is at the correct position, to avoid // interfering with normal cursor movement. // ------------------------------------------------------------- - Input { key: Key::Up, .. } => { - if self.history.should_handle_navigation(&self.textarea) { - let consumed = self - .history - .navigate_up(&mut self.textarea, &self.app_event_tx); - if consumed { - return (InputResult::None, true); - } - } - self.handle_input_basic(input) - } - Input { key: Key::Down, .. } => { - if self.history.should_handle_navigation(&self.textarea) { - let consumed = self - .history - .navigate_down(&mut self.textarea, &self.app_event_tx); - if consumed { - return (InputResult::None, true); - } - } - self.handle_input_basic(input) - } - Input { - key: Key::Enter, - shift: false, - alt: false, - ctrl: false, + KeyEvent { + code: KeyCode::Up | KeyCode::Down, + .. } => { - let mut text = self.textarea.lines().join("\n"); - self.textarea.select_all(); - self.textarea.cut(); + if self + .history + .should_handle_navigation(self.textarea.text(), self.textarea.cursor()) + { + let replace_text = match key_event.code { + KeyCode::Up => self.history.navigate_up(&self.app_event_tx), + KeyCode::Down => self.history.navigate_down(&self.app_event_tx), + _ => unreachable!(), + }; + if let Some(text) = replace_text { + self.textarea.set_text(&text); + self.textarea.set_cursor(0); + return (InputResult::None, true); + } + } + self.handle_input_basic(key_event) + } + KeyEvent { + code: KeyCode::Enter, + modifiers: KeyModifiers::NONE, + .. + } => { + let mut text = self.textarea.text().to_string(); + self.textarea.set_text(""); // Replace all pending pastes in the text for (placeholder, actual) in &self.pending_pastes { @@ -467,41 +507,15 @@ impl ChatComposer<'_> { (InputResult::Submitted(text), true) } } - Input { - key: Key::Enter, .. - } - | Input { - key: Key::Char('j'), - ctrl: true, - alt: false, - shift: false, - } => { - self.textarea.insert_newline(); - (InputResult::None, true) - } - Input { - key: Key::Char('d'), - ctrl: true, - alt: false, - shift: false, - } => { - self.textarea.input(Input { - key: Key::Delete, - ctrl: false, - alt: false, - shift: false, - }); - (InputResult::None, true) - } input => self.handle_input_basic(input), } } /// Handle generic Input events that modify the textarea content. - fn handle_input_basic(&mut self, input: Input) -> (InputResult, bool) { + fn handle_input_basic(&mut self, input: KeyEvent) -> (InputResult, bool) { // Special handling for backspace on placeholders - if let Input { - key: Key::Backspace, + if let KeyEvent { + code: KeyCode::Backspace, .. } = input { @@ -510,20 +524,9 @@ impl ChatComposer<'_> { } } - if let Input { - key: Key::Char('u'), - ctrl: true, - alt: false, - .. - } = input - { - self.textarea.delete_line_by_head(); - return (InputResult::None, true); - } - // Normal input handling self.textarea.input(input); - let text_after = self.textarea.lines().join("\n"); + let text_after = self.textarea.text(); // Check if any placeholders were removed and remove their corresponding pending pastes self.pending_pastes @@ -535,21 +538,16 @@ impl ChatComposer<'_> { /// Attempts to remove a placeholder if the cursor is at the end of one. /// Returns true if a placeholder was removed. fn try_remove_placeholder_at_cursor(&mut self) -> bool { - let (row, col) = self.textarea.cursor(); - let line = self - .textarea - .lines() - .get(row) - .map(|s| s.as_str()) - .unwrap_or(""); + let p = self.textarea.cursor(); + let text = self.textarea.text(); // Find any placeholder that ends at the cursor position let placeholder_to_remove = self.pending_pastes.iter().find_map(|(ph, _)| { - if col < ph.len() { + if p < ph.len() { return None; } - let potential_ph_start = col - ph.len(); - if line[potential_ph_start..col] == *ph { + let potential_ph_start = p - ph.len(); + if text[potential_ph_start..p] == *ph { Some(ph.clone()) } else { None @@ -557,17 +555,7 @@ impl ChatComposer<'_> { }); if let Some(placeholder) = placeholder_to_remove { - // Remove the entire placeholder from the text - let placeholder_len = placeholder.len(); - for _ in 0..placeholder_len { - self.textarea.input(Input { - key: Key::Backspace, - ctrl: false, - alt: false, - shift: false, - }); - } - // Remove from pending pastes + self.textarea.replace_range(p - placeholder.len()..p, ""); self.pending_pastes.retain(|(ph, _)| ph != &placeholder); true } else { @@ -579,16 +567,7 @@ impl ChatComposer<'_> { /// textarea. This must be called after every modification that can change /// the text so the popup is shown/updated/hidden as appropriate. fn sync_command_popup(&mut self) { - // Inspect only the first line to decide whether to show the popup. In - // the common case (no leading slash) we avoid copying the entire - // textarea contents. - let first_line = self - .textarea - .lines() - .first() - .map(|s| s.as_str()) - .unwrap_or(""); - + let first_line = self.textarea.text().lines().next().unwrap_or(""); let input_starts_with_slash = first_line.starts_with('/'); match &mut self.active_popup { ActivePopup::Command(popup) => { @@ -644,74 +623,29 @@ impl ChatComposer<'_> { self.dismissed_file_popup_token = None; } - fn update_border(&mut self, has_focus: bool) { - let border_style = if has_focus { - Style::default().fg(Color::Cyan) - } else { - Style::default().dim() - }; - - self.textarea.set_block( - ratatui::widgets::Block::default() - .borders(Borders::LEFT) - .border_type(BorderType::QuadrantOutside) - .border_style(border_style), - ); + fn set_has_focus(&mut self, has_focus: bool) { + self.has_focus = has_focus; } } -impl WidgetRef for &ChatComposer<'_> { +impl WidgetRef for &ChatComposer { fn render_ref(&self, area: Rect, buf: &mut Buffer) { + let popup_height = match &self.active_popup { + ActivePopup::Command(popup) => popup.calculate_required_height(), + ActivePopup::File(popup) => popup.calculate_required_height(), + ActivePopup::None => 1, + }; + let [textarea_rect, popup_rect] = + Layout::vertical([Constraint::Min(0), Constraint::Max(popup_height)]).areas(area); match &self.active_popup { ActivePopup::Command(popup) => { - let popup_height = popup.calculate_required_height(); - - // Split the provided rect so that the popup is rendered at the - // **bottom** and the textarea occupies the remaining space above. - let popup_height = popup_height.min(area.height); - let textarea_rect = Rect { - x: area.x, - y: area.y, - width: area.width, - height: area.height.saturating_sub(popup_height), - }; - let popup_rect = Rect { - x: area.x, - y: area.y + textarea_rect.height, - width: area.width, - height: popup_height, - }; - - popup.render(popup_rect, buf); - self.textarea.render(textarea_rect, buf); + popup.render_ref(popup_rect, buf); } ActivePopup::File(popup) => { - let popup_height = popup.calculate_required_height(); - - let popup_height = popup_height.min(area.height); - let textarea_rect = Rect { - x: area.x, - y: area.y, - width: area.width, - height: area.height.saturating_sub(popup_height), - }; - let popup_rect = Rect { - x: area.x, - y: area.y + textarea_rect.height, - width: area.width, - height: popup_height, - }; - - popup.render(popup_rect, buf); - self.textarea.render(textarea_rect, buf); + popup.render_ref(popup_rect, buf); } ActivePopup::None => { - let mut textarea_rect = area; - textarea_rect.height = textarea_rect.height.saturating_sub(1); - self.textarea.render(textarea_rect, buf); - let mut bottom_line_rect = area; - bottom_line_rect.y += textarea_rect.height; - bottom_line_rect.height = 1; + let bottom_line_rect = popup_rect; let key_hint_style = Style::default().fg(Color::Cyan); let hint = if self.ctrl_c_quit_hint { vec![ @@ -740,6 +674,56 @@ impl WidgetRef for &ChatComposer<'_> { .render_ref(bottom_line_rect, buf); } } + Block::default() + .border_style(Style::default().dim()) + .borders(Borders::LEFT) + .border_type(BorderType::QuadrantOutside) + .border_style(Style::default().fg(if self.has_focus { + Color::Cyan + } else { + Color::Gray + })) + .render_ref( + Rect::new(textarea_rect.x, textarea_rect.y, 1, textarea_rect.height), + buf, + ); + let mut textarea_rect = textarea_rect; + textarea_rect.width = textarea_rect.width.saturating_sub(1); + textarea_rect.x += 1; + let mut state = self.textarea_state.borrow_mut(); + StatefulWidgetRef::render_ref(&(&self.textarea), textarea_rect, buf, &mut state); + if self.textarea.text().is_empty() { + let placeholder = if let Some(token_usage_info) = &self.token_usage_info { + let token_usage = &token_usage_info.token_usage; + let model_context_window = token_usage_info.model_context_window; + match (token_usage.total_tokens, model_context_window) { + (total_tokens, Some(context_window)) => { + let percent_remaining: u8 = if context_window > 0 { + // Calculate the percentage of context left. + let percent = + 100.0 - (total_tokens as f32 / context_window as f32 * 100.0); + percent.clamp(0.0, 100.0) as u8 + } else { + // If we don't have a context window, we cannot compute the + // percentage. + 100 + }; + // When https://github.com/openai/codex/issues/1257 is resolved, + // check if `percent_remaining < 25`, and if so, recommend + // /compact. + format!("{BASE_PLACEHOLDER_TEXT} — {percent_remaining}% context left") + } + (total_tokens, None) => { + format!("{BASE_PLACEHOLDER_TEXT} — {total_tokens} tokens used") + } + } + } else { + BASE_PLACEHOLDER_TEXT.to_string() + }; + Line::from(placeholder) + .style(Style::default().dim()) + .render_ref(textarea_rect.inner(Margin::new(1, 0)), buf); + } } } @@ -749,7 +733,7 @@ mod tests { use crate::bottom_pane::ChatComposer; use crate::bottom_pane::InputResult; use crate::bottom_pane::chat_composer::LARGE_PASTE_CHAR_THRESHOLD; - use tui_textarea::TextArea; + use crate::bottom_pane::textarea::TextArea; #[test] fn test_current_at_token_basic_cases() { @@ -792,9 +776,9 @@ mod tests { ]; for (input, cursor_pos, expected, description) in test_cases { - let mut textarea = TextArea::default(); + let mut textarea = TextArea::new(); textarea.insert_str(input); - textarea.move_cursor(tui_textarea::CursorMove::Jump(0, cursor_pos)); + textarea.set_cursor(cursor_pos); let result = ChatComposer::current_at_token(&textarea); assert_eq!( @@ -826,9 +810,9 @@ mod tests { ]; for (input, cursor_pos, expected, description) in test_cases { - let mut textarea = TextArea::default(); + let mut textarea = TextArea::new(); textarea.insert_str(input); - textarea.move_cursor(tui_textarea::CursorMove::Jump(0, cursor_pos)); + textarea.set_cursor(cursor_pos); let result = ChatComposer::current_at_token(&textarea); assert_eq!( @@ -863,13 +847,13 @@ mod tests { // Full-width space boundaries ( "test　@İstanbul", - 6, + 8, Some("İstanbul".to_string()), "@ token after full-width space", ), ( "@ЙЦУ　@诶", - 6, + 10, Some("诶".to_string()), "Full-width space between Unicode tokens", ), @@ -883,9 +867,9 @@ mod tests { ]; for (input, cursor_pos, expected, description) in test_cases { - let mut textarea = TextArea::default(); + let mut textarea = TextArea::new(); textarea.insert_str(input); - textarea.move_cursor(tui_textarea::CursorMove::Jump(0, cursor_pos)); + textarea.set_cursor(cursor_pos); let result = ChatComposer::current_at_token(&textarea); assert_eq!( @@ -907,7 +891,7 @@ mod tests { let needs_redraw = composer.handle_paste("hello".to_string()); assert!(needs_redraw); - assert_eq!(composer.textarea.lines(), ["hello"]); + assert_eq!(composer.textarea.text(), "hello"); assert!(composer.pending_pastes.is_empty()); let (result, _) = @@ -932,7 +916,7 @@ mod tests { let needs_redraw = composer.handle_paste(large.clone()); assert!(needs_redraw); let placeholder = format!("[Pasted Content {} chars]", large.chars().count()); - assert_eq!(composer.textarea.lines(), [placeholder.as_str()]); + assert_eq!(composer.textarea.text(), placeholder); assert_eq!(composer.pending_pastes.len(), 1); assert_eq!(composer.pending_pastes[0].0, placeholder); assert_eq!(composer.pending_pastes[0].1, large); @@ -1008,7 +992,7 @@ mod tests { composer.handle_paste("b".repeat(LARGE_PASTE_CHAR_THRESHOLD + 4)); composer.handle_paste("c".repeat(LARGE_PASTE_CHAR_THRESHOLD + 6)); // Move cursor to end and press backspace - composer.textarea.move_cursor(tui_textarea::CursorMove::End); + composer.textarea.set_cursor(composer.textarea.text().len()); composer.handle_key_event(KeyEvent::new(KeyCode::Backspace, KeyModifiers::NONE)); } @@ -1123,7 +1107,7 @@ mod tests { current_pos += content.len(); } ( - composer.textarea.lines().join("\n"), + composer.textarea.text().to_string(), composer.pending_pastes.len(), current_pos, ) @@ -1134,25 +1118,18 @@ mod tests { let mut deletion_states = vec![]; // First deletion - composer - .textarea - .move_cursor(tui_textarea::CursorMove::Jump(0, states[0].2 as u16)); + composer.textarea.set_cursor(states[0].2); composer.handle_key_event(KeyEvent::new(KeyCode::Backspace, KeyModifiers::NONE)); deletion_states.push(( - composer.textarea.lines().join("\n"), + composer.textarea.text().to_string(), composer.pending_pastes.len(), )); // Second deletion - composer - .textarea - .move_cursor(tui_textarea::CursorMove::Jump( - 0, - composer.textarea.lines().join("\n").len() as u16, - )); + composer.textarea.set_cursor(composer.textarea.text().len()); composer.handle_key_event(KeyEvent::new(KeyCode::Backspace, KeyModifiers::NONE)); deletion_states.push(( - composer.textarea.lines().join("\n"), + composer.textarea.text().to_string(), composer.pending_pastes.len(), )); @@ -1191,17 +1168,13 @@ mod tests { composer.handle_paste(paste.clone()); composer .textarea - .move_cursor(tui_textarea::CursorMove::Jump( - 0, - (placeholder.len() - pos_from_end) as u16, - )); + .set_cursor((placeholder.len() - pos_from_end) as usize); composer.handle_key_event(KeyEvent::new(KeyCode::Backspace, KeyModifiers::NONE)); let result = ( - composer.textarea.lines().join("\n").contains(&placeholder), + composer.textarea.text().contains(&placeholder), composer.pending_pastes.len(), ); - composer.textarea.select_all(); - composer.textarea.cut(); + composer.textarea.set_text(""); result }) .collect(); diff --git a/codex-rs/tui/src/bottom_pane/chat_composer_history.rs b/codex-rs/tui/src/bottom_pane/chat_composer_history.rs index 5715c99492..a744a409f3 100644 --- a/codex-rs/tui/src/bottom_pane/chat_composer_history.rs +++ b/codex-rs/tui/src/bottom_pane/chat_composer_history.rs @@ -1,8 +1,5 @@ use std::collections::HashMap; -use tui_textarea::CursorMove; -use tui_textarea::TextArea; - use crate::app_event::AppEvent; use crate::app_event_sender::AppEventSender; use codex_core::protocol::Op; @@ -67,59 +64,52 @@ impl ChatComposerHistory { /// Should Up/Down key presses be interpreted as history navigation given /// the current content and cursor position of `textarea`? - pub fn should_handle_navigation(&self, textarea: &TextArea) -> bool { + pub fn should_handle_navigation(&self, text: &str, cursor: usize) -> bool { if self.history_entry_count == 0 && self.local_history.is_empty() { return false; } - if textarea.is_empty() { + if text.is_empty() { return true; } // Textarea is not empty – only navigate when cursor is at start and // text matches last recalled history entry so regular editing is not // hijacked. - let (row, col) = textarea.cursor(); - if row != 0 || col != 0 { + if cursor != 0 { return false; } - let lines = textarea.lines(); - matches!(&self.last_history_text, Some(prev) if prev == &lines.join("\n")) + matches!(&self.last_history_text, Some(prev) if prev == text) } /// Handle . Returns true when the key was consumed and the caller /// should request a redraw. - pub fn navigate_up(&mut self, textarea: &mut TextArea, app_event_tx: &AppEventSender) -> bool { + pub fn navigate_up(&mut self, app_event_tx: &AppEventSender) -> Option { let total_entries = self.history_entry_count + self.local_history.len(); if total_entries == 0 { - return false; + return None; } let next_idx = match self.history_cursor { None => (total_entries as isize) - 1, - Some(0) => return true, // already at oldest + Some(0) => return None, // already at oldest Some(idx) => idx - 1, }; self.history_cursor = Some(next_idx); - self.populate_history_at_index(next_idx as usize, textarea, app_event_tx); - true + self.populate_history_at_index(next_idx as usize, app_event_tx) } /// Handle . - pub fn navigate_down( - &mut self, - textarea: &mut TextArea, - app_event_tx: &AppEventSender, - ) -> bool { + pub fn navigate_down(&mut self, app_event_tx: &AppEventSender) -> Option { let total_entries = self.history_entry_count + self.local_history.len(); if total_entries == 0 { - return false; + return None; } let next_idx_opt = match self.history_cursor { - None => return false, // not browsing + None => return None, // not browsing Some(idx) if (idx as usize) + 1 >= total_entries => None, Some(idx) => Some(idx + 1), }; @@ -127,16 +117,15 @@ impl ChatComposerHistory { match next_idx_opt { Some(idx) => { self.history_cursor = Some(idx); - self.populate_history_at_index(idx as usize, textarea, app_event_tx); + self.populate_history_at_index(idx as usize, app_event_tx) } None => { // Past newest – clear and exit browsing mode. self.history_cursor = None; self.last_history_text = None; - self.replace_textarea_content(textarea, ""); + Some(String::new()) } } - true } /// Integrate a GetHistoryEntryResponse event. @@ -145,19 +134,18 @@ impl ChatComposerHistory { log_id: u64, offset: usize, entry: Option, - textarea: &mut TextArea, - ) -> bool { + ) -> Option { if self.history_log_id != Some(log_id) { - return false; + return None; } - let Some(text) = entry else { return false }; + let text = entry?; self.fetched_history.insert(offset, text.clone()); if self.history_cursor == Some(offset as isize) { - self.replace_textarea_content(textarea, &text); - return true; + self.last_history_text = Some(text.clone()); + return Some(text); } - false + None } // --------------------------------------------------------------------- @@ -167,21 +155,20 @@ impl ChatComposerHistory { fn populate_history_at_index( &mut self, global_idx: usize, - textarea: &mut TextArea, app_event_tx: &AppEventSender, - ) { + ) -> Option { if global_idx >= self.history_entry_count { // Local entry. if let Some(text) = self .local_history .get(global_idx - self.history_entry_count) { - let t = text.clone(); - self.replace_textarea_content(textarea, &t); + self.last_history_text = Some(text.clone()); + return Some(text.clone()); } } else if let Some(text) = self.fetched_history.get(&global_idx) { - let t = text.clone(); - self.replace_textarea_content(textarea, &t); + self.last_history_text = Some(text.clone()); + return Some(text.clone()); } else if let Some(log_id) = self.history_log_id { let op = Op::GetHistoryEntryRequest { offset: global_idx, @@ -189,14 +176,7 @@ impl ChatComposerHistory { }; app_event_tx.send(AppEvent::CodexOp(op)); } - } - - fn replace_textarea_content(&mut self, textarea: &mut TextArea, text: &str) { - textarea.select_all(); - textarea.cut(); - let _ = textarea.insert_str(text); - textarea.move_cursor(CursorMove::Jump(0, 0)); - self.last_history_text = Some(text.to_string()); + None } } @@ -217,11 +197,9 @@ mod tests { // Pretend there are 3 persistent entries. history.set_metadata(1, 3); - let mut textarea = TextArea::default(); - // First Up should request offset 2 (latest) and await async data. - assert!(history.should_handle_navigation(&textarea)); - assert!(history.navigate_up(&mut textarea, &tx)); + assert!(history.should_handle_navigation("", 0)); + assert!(history.navigate_up(&tx).is_none()); // don't replace the text yet // Verify that an AppEvent::CodexOp with the correct GetHistoryEntryRequest was sent. let event = rx.try_recv().expect("expected AppEvent to be sent"); @@ -235,14 +213,15 @@ mod tests { }, history_request1 ); - assert_eq!(textarea.lines().join("\n"), ""); // still empty // Inject the async response. - assert!(history.on_entry_response(1, 2, Some("latest".into()), &mut textarea)); - assert_eq!(textarea.lines().join("\n"), "latest"); + assert_eq!( + Some("latest".into()), + history.on_entry_response(1, 2, Some("latest".into())) + ); // Next Up should move to offset 1. - assert!(history.navigate_up(&mut textarea, &tx)); + assert!(history.navigate_up(&tx).is_none()); // don't replace the text yet // Verify second CodexOp event for offset 1. let event2 = rx.try_recv().expect("expected second event"); @@ -257,7 +236,9 @@ mod tests { history_request_2 ); - history.on_entry_response(1, 1, Some("older".into()), &mut textarea); - assert_eq!(textarea.lines().join("\n"), "older"); + assert_eq!( + Some("older".into()), + history.on_entry_response(1, 1, Some("older".into())) + ); } } diff --git a/codex-rs/tui/src/bottom_pane/mod.rs b/codex-rs/tui/src/bottom_pane/mod.rs index 281f0859a5..cab78bbe3f 100644 --- a/codex-rs/tui/src/bottom_pane/mod.rs +++ b/codex-rs/tui/src/bottom_pane/mod.rs @@ -19,6 +19,7 @@ mod chat_composer_history; mod command_popup; mod file_search_popup; mod status_indicator_view; +mod textarea; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum CancellationEvent { @@ -36,7 +37,7 @@ use status_indicator_view::StatusIndicatorView; pub(crate) struct BottomPane<'a> { /// Composer is retained even when a BottomPaneView is displayed so the /// input state is retained when the view is closed. - composer: ChatComposer<'a>, + composer: ChatComposer, /// If present, this is displayed instead of the `composer`. active_view: Option + 'a>>, @@ -74,7 +75,19 @@ impl BottomPane<'_> { self.active_view .as_ref() .map(|v| v.desired_height(width)) - .unwrap_or(self.composer.desired_height()) + .unwrap_or(self.composer.desired_height(width)) + } + + pub fn cursor_pos(&self, area: Rect) -> Option<(u16, u16)> { + // Hide the cursor whenever an overlay view is active (e.g. the + // status indicator shown while a task is running, or approval modal). + // In these states the textarea is not interactable, so we should not + // show its caret. + if self.active_view.is_some() { + None + } else { + self.composer.cursor_pos(area) + } } /// Forward a key event to the active view or the composer. diff --git a/codex-rs/tui/src/bottom_pane/textarea.rs b/codex-rs/tui/src/bottom_pane/textarea.rs new file mode 100644 index 0000000000..e150135b75 --- /dev/null +++ b/codex-rs/tui/src/bottom_pane/textarea.rs @@ -0,0 +1,1294 @@ +use crossterm::event::KeyCode; +use crossterm::event::KeyEvent; +use crossterm::event::KeyModifiers; +use ratatui::buffer::Buffer; +use ratatui::layout::Rect; +use ratatui::style::Style; +use ratatui::widgets::StatefulWidgetRef; +use ratatui::widgets::WidgetRef; +use std::cell::Ref; +use std::cell::RefCell; +use std::ops::Range; +use textwrap::Options; +use unicode_segmentation::UnicodeSegmentation; +use unicode_width::UnicodeWidthStr; + +#[derive(Debug)] +pub(crate) struct TextArea { + text: String, + cursor_pos: usize, + wrap_cache: RefCell>, + preferred_col: Option, +} + +#[derive(Debug, Clone)] +struct WrapCache { + width: u16, + lines: Vec>, +} + +#[derive(Debug, Default, Clone, Copy)] +pub(crate) struct TextAreaState { + /// Index into wrapped lines of the first visible line. + scroll: u16, +} + +impl TextArea { + pub fn new() -> Self { + Self { + text: String::new(), + cursor_pos: 0, + wrap_cache: RefCell::new(None), + preferred_col: None, + } + } + + pub fn set_text(&mut self, text: &str) { + self.text = text.to_string(); + self.cursor_pos = self.cursor_pos.clamp(0, self.text.len()); + self.wrap_cache.replace(None); + self.preferred_col = None; + } + + pub fn text(&self) -> &str { + &self.text + } + + pub fn insert_str(&mut self, text: &str) { + self.insert_str_at(self.cursor_pos, text); + } + + pub fn insert_str_at(&mut self, pos: usize, text: &str) { + self.text.insert_str(pos, text); + self.wrap_cache.replace(None); + if pos <= self.cursor_pos { + self.cursor_pos += text.len(); + } + self.preferred_col = None; + } + + pub fn replace_range(&mut self, range: std::ops::Range, text: &str) { + assert!(range.start <= range.end); + let start = range.start.clamp(0, self.text.len()); + let end = range.end.clamp(0, self.text.len()); + let removed_len = end - start; + let inserted_len = text.len(); + if removed_len == 0 && inserted_len == 0 { + return; + } + let diff = inserted_len as isize - removed_len as isize; + + self.text.replace_range(range, text); + self.wrap_cache.replace(None); + self.preferred_col = None; + + // Update the cursor position to account for the edit. + self.cursor_pos = if self.cursor_pos < start { + // Cursor was before the edited range – no shift. + self.cursor_pos + } else if self.cursor_pos <= end { + // Cursor was inside the replaced range – move to end of the new text. + start + inserted_len + } else { + // Cursor was after the replaced range – shift by the length diff. + ((self.cursor_pos as isize) + diff) as usize + } + .min(self.text.len()); + } + + pub fn cursor(&self) -> usize { + self.cursor_pos + } + + pub fn set_cursor(&mut self, pos: usize) { + self.cursor_pos = pos.clamp(0, self.text.len()); + self.preferred_col = None; + } + + pub fn desired_height(&self, width: u16) -> u16 { + self.wrapped_lines(width).len() as u16 + } + + #[allow(dead_code)] + pub fn cursor_pos(&self, area: Rect) -> Option<(u16, u16)> { + self.cursor_pos_with_state(area, &TextAreaState::default()) + } + + /// Compute the on-screen cursor position taking scrolling into account. + pub fn cursor_pos_with_state(&self, area: Rect, state: &TextAreaState) -> Option<(u16, u16)> { + let lines = self.wrapped_lines(area.width); + let effective_scroll = self.effective_scroll(area.height, &lines, state.scroll); + let i = Self::wrapped_line_index_by_start(&lines, self.cursor_pos)?; + let ls = &lines[i]; + let col = self.text[ls.start..self.cursor_pos].width() as u16; + let screen_row = i + .saturating_sub(effective_scroll as usize) + .try_into() + .unwrap_or(0); + Some((area.x + col, area.y + screen_row)) + } + + pub fn is_empty(&self) -> bool { + self.text.is_empty() + } + + fn current_display_col(&self) -> usize { + let bol = self.beginning_of_current_line(); + self.text[bol..self.cursor_pos].width() + } + + fn wrapped_line_index_by_start(lines: &[Range], pos: usize) -> Option { + // partition_point returns the index of the first element for which + // the predicate is false, i.e. the count of elements with start <= pos. + let idx = lines.partition_point(|r| r.start <= pos); + if idx == 0 { None } else { Some(idx - 1) } + } + + fn move_to_display_col_on_line( + &mut self, + line_start: usize, + line_end: usize, + target_col: usize, + ) { + let mut width_so_far = 0usize; + for (i, g) in self.text[line_start..line_end].grapheme_indices(true) { + width_so_far += g.width(); + if width_so_far > target_col { + self.cursor_pos = line_start + i; + return; + } + } + self.cursor_pos = line_end; + } + + fn beginning_of_line(&self, pos: usize) -> usize { + self.text[..pos].rfind('\n').map(|i| i + 1).unwrap_or(0) + } + fn beginning_of_current_line(&self) -> usize { + self.beginning_of_line(self.cursor_pos) + } + + fn end_of_line(&self, pos: usize) -> usize { + self.text[pos..] + .find('\n') + .map(|i| i + pos) + .unwrap_or(self.text.len()) + } + fn end_of_current_line(&self) -> usize { + self.end_of_line(self.cursor_pos) + } + + pub(crate) fn beginning_of_previous_word(&self) -> usize { + if let Some(first_non_ws) = self.text[..self.cursor_pos].rfind(|c: char| !c.is_whitespace()) + { + self.text[..first_non_ws] + .rfind(|c: char| c.is_whitespace()) + .map(|i| i + 1) + .unwrap_or(0) + } else { + 0 + } + } + + pub(crate) fn end_of_next_word(&self) -> usize { + let Some(first_non_ws) = self.text[self.cursor_pos..].find(|c: char| !c.is_whitespace()) + else { + return self.text.len(); + }; + let word_start = self.cursor_pos + first_non_ws; + match self.text[word_start..].find(|c: char| c.is_whitespace()) { + Some(rel_idx) => word_start + rel_idx, + None => self.text.len(), + } + } + + pub fn input(&mut self, event: KeyEvent) { + match event { + KeyEvent { + code: KeyCode::Char(c), + modifiers: KeyModifiers::NONE | KeyModifiers::SHIFT | KeyModifiers::ALT, + .. + } => self.insert_str(&c.to_string()), + KeyEvent { + code: KeyCode::Char('j'), + modifiers: KeyModifiers::CONTROL, + .. + } + | KeyEvent { + code: KeyCode::Enter, + .. + } => self.insert_str("\n"), + KeyEvent { + code: KeyCode::Backspace, + .. + } => self.delete_backward(1), + KeyEvent { + code: KeyCode::Delete, + .. + } => self.delete_forward(1), + + KeyEvent { + code: KeyCode::Char('w'), + modifiers: KeyModifiers::CONTROL, + .. + } => { + self.delete_backward_word(); + } + KeyEvent { + code: KeyCode::Char('u'), + modifiers: KeyModifiers::CONTROL, + .. + } => { + self.kill_to_beginning_of_line(); + } + KeyEvent { + code: KeyCode::Char('k'), + modifiers: KeyModifiers::CONTROL, + .. + } => { + self.kill_to_end_of_line(); + } + + // Cursor movement + KeyEvent { + code: KeyCode::Left, + modifiers: KeyModifiers::NONE, + .. + } => { + self.move_cursor_left(); + } + KeyEvent { + code: KeyCode::Right, + modifiers: KeyModifiers::NONE, + .. + } => { + self.move_cursor_right(); + } + KeyEvent { + code: KeyCode::Up, .. + } => { + self.move_cursor_up(); + } + KeyEvent { + code: KeyCode::Down, + .. + } => { + self.move_cursor_down(); + } + KeyEvent { + code: KeyCode::Home, + .. + } => { + self.move_cursor_to_beginning_of_line(false); + } + KeyEvent { + code: KeyCode::Char('a'), + modifiers: KeyModifiers::CONTROL, + .. + } => { + self.move_cursor_to_beginning_of_line(true); + } + + KeyEvent { + code: KeyCode::End, .. + } => { + self.move_cursor_to_end_of_line(false); + } + KeyEvent { + code: KeyCode::Char('e'), + modifiers: KeyModifiers::CONTROL, + .. + } => { + self.move_cursor_to_end_of_line(true); + } + KeyEvent { + code: KeyCode::Left, + modifiers: KeyModifiers::CONTROL, + .. + } => { + self.set_cursor(self.beginning_of_previous_word()); + } + KeyEvent { + code: KeyCode::Right, + modifiers: KeyModifiers::CONTROL, + .. + } => { + self.set_cursor(self.end_of_next_word()); + } + o => { + tracing::debug!("Unhandled key event in TextArea: {:?}", o); + } + } + } + + // ####### Input Functions ####### + pub fn delete_backward(&mut self, n: usize) { + if n == 0 || self.cursor_pos == 0 { + return; + } + let mut gc = + unicode_segmentation::GraphemeCursor::new(self.cursor_pos, self.text.len(), false); + let mut target = self.cursor_pos; + for _ in 0..n { + match gc.prev_boundary(&self.text, 0) { + Ok(Some(b)) => target = b, + Ok(None) => { + target = 0; + break; + } + Err(_) => { + target = target.saturating_sub(1); + } + } + } + self.replace_range(target..self.cursor_pos, ""); + } + + pub fn delete_forward(&mut self, n: usize) { + if n == 0 || self.cursor_pos >= self.text.len() { + return; + } + let mut gc = + unicode_segmentation::GraphemeCursor::new(self.cursor_pos, self.text.len(), false); + let mut target = self.cursor_pos; + for _ in 0..n { + match gc.next_boundary(&self.text, 0) { + Ok(Some(b)) => target = b, + Ok(None) => { + target = self.text.len(); + break; + } + Err(_) => { + target = target.saturating_add(1); + } + } + } + self.replace_range(self.cursor_pos..target, ""); + } + + pub fn delete_backward_word(&mut self) { + self.replace_range(self.beginning_of_previous_word()..self.cursor_pos, ""); + } + + pub fn kill_to_end_of_line(&mut self) { + let eol = self.end_of_current_line(); + if self.cursor_pos == eol { + if eol < self.text.len() { + self.replace_range(self.cursor_pos..eol + 1, ""); + } + } else { + self.replace_range(self.cursor_pos..eol, ""); + } + } + + pub fn kill_to_beginning_of_line(&mut self) { + let bol = self.beginning_of_current_line(); + if self.cursor_pos == bol { + if bol > 0 { + self.replace_range(bol - 1..bol, ""); + } + } else { + self.replace_range(bol..self.cursor_pos, ""); + } + } + + /// Move the cursor left by a single grapheme cluster. + pub fn move_cursor_left(&mut self) { + let mut gc = + unicode_segmentation::GraphemeCursor::new(self.cursor_pos, self.text.len(), false); + match gc.prev_boundary(&self.text, 0) { + Ok(Some(boundary)) => self.cursor_pos = boundary, + Ok(None) => self.cursor_pos = 0, // Already at start. + Err(_) => self.cursor_pos = self.cursor_pos.saturating_sub(1), + } + self.preferred_col = None; + } + + /// Move the cursor right by a single grapheme cluster. + pub fn move_cursor_right(&mut self) { + let mut gc = + unicode_segmentation::GraphemeCursor::new(self.cursor_pos, self.text.len(), false); + match gc.next_boundary(&self.text, 0) { + Ok(Some(boundary)) => self.cursor_pos = boundary, + Ok(None) => self.cursor_pos = self.text.len(), // Already at end. + Err(_) => self.cursor_pos = self.cursor_pos.saturating_add(1), + } + self.preferred_col = None; + } + + pub fn move_cursor_up(&mut self) { + // If we have a wrapping cache, prefer navigating across wrapped (visual) lines. + if let Some((target_col, maybe_line)) = { + let cache_ref = self.wrap_cache.borrow(); + if let Some(cache) = cache_ref.as_ref() { + let lines = &cache.lines; + if let Some(idx) = Self::wrapped_line_index_by_start(lines, self.cursor_pos) { + let cur_range = &lines[idx]; + let target_col = self + .preferred_col + .unwrap_or_else(|| self.text[cur_range.start..self.cursor_pos].width()); + if idx > 0 { + let prev = &lines[idx - 1]; + let line_start = prev.start; + let line_end = prev.end.saturating_sub(1); + Some((target_col, Some((line_start, line_end)))) + } else { + Some((target_col, None)) + } + } else { + None + } + } else { + None + } + } { + // We had wrapping info. Apply movement accordingly. + match maybe_line { + Some((line_start, line_end)) => { + if self.preferred_col.is_none() { + self.preferred_col = Some(target_col); + } + self.move_to_display_col_on_line(line_start, line_end, target_col); + return; + } + None => { + // Already at first visual line -> move to start + self.cursor_pos = 0; + self.preferred_col = None; + return; + } + } + } + + // Fallback to logical line navigation if we don't have wrapping info yet. + if let Some(prev_nl) = self.text[..self.cursor_pos].rfind('\n') { + let target_col = match self.preferred_col { + Some(c) => c, + None => { + let c = self.current_display_col(); + self.preferred_col = Some(c); + c + } + }; + let prev_line_start = self.text[..prev_nl].rfind('\n').map(|i| i + 1).unwrap_or(0); + let prev_line_end = prev_nl; + self.move_to_display_col_on_line(prev_line_start, prev_line_end, target_col); + } else { + self.cursor_pos = 0; + self.preferred_col = None; + } + } + + pub fn move_cursor_down(&mut self) { + // If we have a wrapping cache, prefer navigating across wrapped (visual) lines. + if let Some((target_col, move_to_last)) = { + let cache_ref = self.wrap_cache.borrow(); + if let Some(cache) = cache_ref.as_ref() { + let lines = &cache.lines; + if let Some(idx) = Self::wrapped_line_index_by_start(lines, self.cursor_pos) { + let cur_range = &lines[idx]; + let target_col = self + .preferred_col + .unwrap_or_else(|| self.text[cur_range.start..self.cursor_pos].width()); + if idx + 1 < lines.len() { + let next = &lines[idx + 1]; + let line_start = next.start; + let line_end = next.end.saturating_sub(1); + Some((target_col, Some((line_start, line_end)))) + } else { + Some((target_col, None)) + } + } else { + None + } + } else { + None + } + } { + match move_to_last { + Some((line_start, line_end)) => { + if self.preferred_col.is_none() { + self.preferred_col = Some(target_col); + } + self.move_to_display_col_on_line(line_start, line_end, target_col); + return; + } + None => { + // Already on last visual line -> move to end + self.cursor_pos = self.text.len(); + self.preferred_col = None; + return; + } + } + } + + // Fallback to logical line navigation if we don't have wrapping info yet. + let target_col = match self.preferred_col { + Some(c) => c, + None => { + let c = self.current_display_col(); + self.preferred_col = Some(c); + c + } + }; + if let Some(next_nl) = self.text[self.cursor_pos..] + .find('\n') + .map(|i| i + self.cursor_pos) + { + let next_line_start = next_nl + 1; + let next_line_end = self.text[next_line_start..] + .find('\n') + .map(|i| i + next_line_start) + .unwrap_or(self.text.len()); + self.move_to_display_col_on_line(next_line_start, next_line_end, target_col); + } else { + self.cursor_pos = self.text.len(); + self.preferred_col = None; + } + } + + pub fn move_cursor_to_beginning_of_line(&mut self, move_up_at_bol: bool) { + let bol = self.beginning_of_current_line(); + if move_up_at_bol && self.cursor_pos == bol { + self.set_cursor(self.beginning_of_line(self.cursor_pos.saturating_sub(1))); + } else { + self.set_cursor(bol); + } + self.preferred_col = None; + } + + pub fn move_cursor_to_end_of_line(&mut self, move_down_at_eol: bool) { + let eol = self.end_of_current_line(); + if move_down_at_eol && self.cursor_pos == eol { + let next_pos = (self.cursor_pos.saturating_add(1)).min(self.text.len()); + self.set_cursor(self.end_of_line(next_pos)); + } else { + self.set_cursor(eol); + } + } + + #[allow(clippy::unwrap_used)] + fn wrapped_lines(&self, width: u16) -> Ref<'_, Vec>> { + // Ensure cache is ready (potentially mutably borrow, then drop) + { + let mut cache = self.wrap_cache.borrow_mut(); + let needs_recalc = match cache.as_ref() { + Some(c) => c.width != width, + None => true, + }; + if needs_recalc { + let mut lines: Vec> = Vec::new(); + for line in textwrap::wrap( + &self.text, + Options::new(width as usize).wrap_algorithm(textwrap::WrapAlgorithm::FirstFit), + ) + .iter() + { + match line { + std::borrow::Cow::Borrowed(slice) => { + let start = + unsafe { slice.as_ptr().offset_from(self.text.as_ptr()) as usize }; + let end = start + slice.len(); + let trailing_spaces = + self.text[end..].chars().take_while(|c| *c == ' ').count(); + lines.push(start..end + trailing_spaces + 1); + } + std::borrow::Cow::Owned(_) => unreachable!(), + } + } + *cache = Some(WrapCache { width, lines }); + } + } + + let cache = self.wrap_cache.borrow(); + Ref::map(cache, |c| &c.as_ref().unwrap().lines) + } + + /// Calculate the scroll offset that should be used to satisfy the + /// invariants given the current area size and wrapped lines. + /// + /// - Cursor is always on screen. + /// - No scrolling if content fits in the area. + fn effective_scroll( + &self, + area_height: u16, + lines: &[Range], + current_scroll: u16, + ) -> u16 { + let total_lines = lines.len() as u16; + if area_height >= total_lines { + return 0; + } + + // Where is the cursor within wrapped lines? Prefer assigning boundary positions + // (where pos equals the start of a wrapped line) to that later line. + let cursor_line_idx = + Self::wrapped_line_index_by_start(lines, self.cursor_pos).unwrap_or(0) as u16; + + let max_scroll = total_lines.saturating_sub(area_height); + let mut scroll = current_scroll.min(max_scroll); + + // Ensure cursor is visible within [scroll, scroll + area_height) + if cursor_line_idx < scroll { + scroll = cursor_line_idx; + } else if cursor_line_idx >= scroll + area_height { + scroll = cursor_line_idx + 1 - area_height; + } + scroll + } +} + +impl WidgetRef for &TextArea { + fn render_ref(&self, area: Rect, buf: &mut Buffer) { + let lines = self.wrapped_lines(area.width); + for (i, ls) in lines.iter().enumerate() { + let s = &self.text[ls.start..ls.end - 1]; + buf.set_string(area.x, area.y + i as u16, s, Style::default()); + } + } +} + +impl StatefulWidgetRef for &TextArea { + type State = TextAreaState; + + fn render_ref(&self, area: Rect, buf: &mut Buffer, state: &mut Self::State) { + let lines = self.wrapped_lines(area.width); + let scroll = self.effective_scroll(area.height, &lines, state.scroll); + state.scroll = scroll; + + let start = scroll as usize; + let end = (scroll + area.height).min(lines.len() as u16) as usize; + for (row, ls) in (start..end).enumerate() { + let r = &lines[ls]; + let s = &self.text[r.start..r.end - 1]; + buf.set_string(area.x, area.y + row as u16, s, Style::default()); + } + } +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + // crossterm types are intentionally not imported here to avoid unused warnings + use rand::prelude::*; + + fn rand_grapheme(rng: &mut rand::rngs::StdRng) -> String { + let r: u8 = rng.gen_range(0..100); + match r { + 0..=4 => "\n".to_string(), + 5..=12 => " ".to_string(), + 13..=35 => (rng.gen_range(b'a'..=b'z') as char).to_string(), + 36..=45 => (rng.gen_range(b'A'..=b'Z') as char).to_string(), + 46..=52 => (rng.gen_range(b'0'..=b'9') as char).to_string(), + 53..=65 => { + // Some emoji (wide graphemes) + let choices = ["👍", "😊", "🐍", "🚀", "🧪", "🌟"]; + choices[rng.gen_range(0..choices.len())].to_string() + } + 66..=75 => { + // CJK wide characters + let choices = ["漢", "字", "測", "試", "你", "好", "界", "编", "码"]; + choices[rng.gen_range(0..choices.len())].to_string() + } + 76..=85 => { + // Combining mark sequences + let base = ["e", "a", "o", "n", "u"][rng.gen_range(0..5)]; + let marks = ["\u{0301}", "\u{0308}", "\u{0302}", "\u{0303}"]; + format!("{}{}", base, marks[rng.gen_range(0..marks.len())]) + } + 86..=92 => { + // Some non-latin single codepoints (Greek, Cyrillic, Hebrew) + let choices = ["Ω", "β", "Ж", "ю", "ש", "م", "ह"]; + choices[rng.gen_range(0..choices.len())].to_string() + } + _ => { + // ZWJ sequences (single graphemes but multi-codepoint) + let choices = [ + "👩\u{200D}💻", // woman technologist + "👨\u{200D}💻", // man technologist + "🏳️\u{200D}🌈", // rainbow flag + ]; + choices[rng.gen_range(0..choices.len())].to_string() + } + } + } + + fn ta_with(text: &str) -> TextArea { + let mut t = TextArea::new(); + t.insert_str(text); + t + } + + #[test] + fn insert_and_replace_update_cursor_and_text() { + // insert helpers + let mut t = ta_with("hello"); + t.set_cursor(5); + t.insert_str("!"); + assert_eq!(t.text(), "hello!"); + assert_eq!(t.cursor(), 6); + + t.insert_str_at(0, "X"); + assert_eq!(t.text(), "Xhello!"); + assert_eq!(t.cursor(), 7); + + // Insert after the cursor should not move it + t.set_cursor(1); + let end = t.text().len(); + t.insert_str_at(end, "Y"); + assert_eq!(t.text(), "Xhello!Y"); + assert_eq!(t.cursor(), 1); + + // replace_range cases + // 1) cursor before range + let mut t = ta_with("abcd"); + t.set_cursor(1); + t.replace_range(2..3, "Z"); + assert_eq!(t.text(), "abZd"); + assert_eq!(t.cursor(), 1); + + // 2) cursor inside range + let mut t = ta_with("abcd"); + t.set_cursor(2); + t.replace_range(1..3, "Q"); + assert_eq!(t.text(), "aQd"); + assert_eq!(t.cursor(), 2); + + // 3) cursor after range with shifted by diff + let mut t = ta_with("abcd"); + t.set_cursor(4); + t.replace_range(0..1, "AA"); + assert_eq!(t.text(), "AAbcd"); + assert_eq!(t.cursor(), 5); + } + + #[test] + fn delete_backward_and_forward_edges() { + let mut t = ta_with("abc"); + t.set_cursor(1); + t.delete_backward(1); + assert_eq!(t.text(), "bc"); + assert_eq!(t.cursor(), 0); + + // deleting backward at start is a no-op + t.set_cursor(0); + t.delete_backward(1); + assert_eq!(t.text(), "bc"); + assert_eq!(t.cursor(), 0); + + // forward delete removes next grapheme + t.set_cursor(1); + t.delete_forward(1); + assert_eq!(t.text(), "b"); + assert_eq!(t.cursor(), 1); + + // forward delete at end is a no-op + t.set_cursor(t.text().len()); + t.delete_forward(1); + assert_eq!(t.text(), "b"); + } + + #[test] + fn delete_backward_word_and_kill_line_variants() { + // delete backward word at end removes the whole previous word + let mut t = ta_with("hello world "); + t.set_cursor(t.text().len()); + t.delete_backward_word(); + assert_eq!(t.text(), "hello "); + assert_eq!(t.cursor(), 8); + + // From inside a word, delete from word start to cursor + let mut t = ta_with("foo bar"); + t.set_cursor(6); // inside "bar" (after 'a') + t.delete_backward_word(); + assert_eq!(t.text(), "foo r"); + assert_eq!(t.cursor(), 4); + + // From end, delete the last word only + let mut t = ta_with("foo bar"); + t.set_cursor(t.text().len()); + t.delete_backward_word(); + assert_eq!(t.text(), "foo "); + assert_eq!(t.cursor(), 4); + + // kill_to_end_of_line when not at EOL + let mut t = ta_with("abc\ndef"); + t.set_cursor(1); // on first line, middle + t.kill_to_end_of_line(); + assert_eq!(t.text(), "a\ndef"); + assert_eq!(t.cursor(), 1); + + // kill_to_end_of_line when at EOL deletes newline + let mut t = ta_with("abc\ndef"); + t.set_cursor(3); // EOL of first line + t.kill_to_end_of_line(); + assert_eq!(t.text(), "abcdef"); + assert_eq!(t.cursor(), 3); + + // kill_to_beginning_of_line from middle of line + let mut t = ta_with("abc\ndef"); + t.set_cursor(5); // on second line, after 'e' + t.kill_to_beginning_of_line(); + assert_eq!(t.text(), "abc\nef"); + + // kill_to_beginning_of_line at beginning of non-first line removes the previous newline + let mut t = ta_with("abc\ndef"); + t.set_cursor(4); // beginning of second line + t.kill_to_beginning_of_line(); + assert_eq!(t.text(), "abcdef"); + assert_eq!(t.cursor(), 3); + } + + #[test] + fn cursor_left_and_right_handle_graphemes() { + let mut t = ta_with("a👍b"); + t.set_cursor(t.text().len()); + + t.move_cursor_left(); // before 'b' + let after_first_left = t.cursor(); + t.move_cursor_left(); // before '👍' + let after_second_left = t.cursor(); + t.move_cursor_left(); // before 'a' + let after_third_left = t.cursor(); + + assert!(after_first_left < t.text().len()); + assert!(after_second_left < after_first_left); + assert!(after_third_left < after_second_left); + + // Move right back to end safely + t.move_cursor_right(); + t.move_cursor_right(); + t.move_cursor_right(); + assert_eq!(t.cursor(), t.text().len()); + } + + #[test] + fn cursor_vertical_movement_across_lines_and_bounds() { + let mut t = ta_with("short\nloooooooooong\nmid"); + // Place cursor on second line, column 5 + let second_line_start = 6; // after first '\n' + t.set_cursor(second_line_start + 5); + + // Move up: target column preserved, clamped by line length + t.move_cursor_up(); + assert_eq!(t.cursor(), 5); // first line has len 5 + + // Move up again goes to start of text + t.move_cursor_up(); + assert_eq!(t.cursor(), 0); + + // Move down: from start to target col tracked + t.move_cursor_down(); + // On first move down, we should land on second line, at col 0 (target col remembered as 0) + let pos_after_down = t.cursor(); + assert!(pos_after_down >= second_line_start); + + // Move down again to third line; clamp to its length + t.move_cursor_down(); + let third_line_start = t.text().find("mid").unwrap(); + let third_line_end = third_line_start + 3; + assert!(t.cursor() >= third_line_start && t.cursor() <= third_line_end); + + // Moving down at last line jumps to end + t.move_cursor_down(); + assert_eq!(t.cursor(), t.text().len()); + } + + #[test] + fn home_end_and_emacs_style_home_end() { + let mut t = ta_with("one\ntwo\nthree"); + // Position at middle of second line + let second_line_start = t.text().find("two").unwrap(); + t.set_cursor(second_line_start + 1); + + t.move_cursor_to_beginning_of_line(false); + assert_eq!(t.cursor(), second_line_start); + + // Ctrl-A behavior: if at BOL, go to beginning of previous line + t.move_cursor_to_beginning_of_line(true); + assert_eq!(t.cursor(), 0); // beginning of first line + + // Move to EOL of first line + t.move_cursor_to_end_of_line(false); + assert_eq!(t.cursor(), 3); + + // Ctrl-E: if at EOL, go to end of next line + t.move_cursor_to_end_of_line(true); + // end of second line ("two") is right before its '\n' + let end_second_nl = t.text().find("\nthree").unwrap(); + assert_eq!(t.cursor(), end_second_nl); + } + + #[test] + fn end_of_line_or_down_at_end_of_text() { + let mut t = ta_with("one\ntwo"); + // Place cursor at absolute end of the text + t.set_cursor(t.text().len()); + // Should remain at end without panicking + t.move_cursor_to_end_of_line(true); + assert_eq!(t.cursor(), t.text().len()); + + // Also verify behavior when at EOL of a non-final line: + let eol_first_line = 3; // index of '\n' in "one\ntwo" + t.set_cursor(eol_first_line); + t.move_cursor_to_end_of_line(true); + assert_eq!(t.cursor(), t.text().len()); // moves to end of next (last) line + } + + #[test] + fn word_navigation_helpers() { + let t = ta_with(" alpha beta gamma"); + let mut t = t; // make mutable for set_cursor + // Put cursor after "alpha" + let after_alpha = t.text().find("alpha").unwrap() + "alpha".len(); + t.set_cursor(after_alpha); + assert_eq!(t.beginning_of_previous_word(), 2); // skip initial spaces + + // Put cursor at start of beta + let beta_start = t.text().find("beta").unwrap(); + t.set_cursor(beta_start); + assert_eq!(t.end_of_next_word(), beta_start + "beta".len()); + + // If at end, end_of_next_word returns len + t.set_cursor(t.text().len()); + assert_eq!(t.end_of_next_word(), t.text().len()); + } + + #[test] + fn wrapping_and_cursor_positions() { + let mut t = ta_with("hello world here"); + let area = Rect::new(0, 0, 6, 10); // width 6 -> wraps words + // desired height counts wrapped lines + assert!(t.desired_height(area.width) >= 3); + + // Place cursor in "world" + let world_start = t.text().find("world").unwrap(); + t.set_cursor(world_start + 3); + let (_x, y) = t.cursor_pos(area).unwrap(); + assert_eq!(y, 1); // world should be on second wrapped line + + // With state and small height, cursor is mapped onto visible row + let mut state = TextAreaState::default(); + let small_area = Rect::new(0, 0, 6, 1); + // First call: cursor not visible -> effective scroll ensures it is + let (_x, y) = t.cursor_pos_with_state(small_area, &state).unwrap(); + assert_eq!(y, 0); + + // Render with state to update actual scroll value + let mut buf = Buffer::empty(small_area); + ratatui::widgets::StatefulWidgetRef::render_ref(&(&t), small_area, &mut buf, &mut state); + // After render, state.scroll should be adjusted so cursor row fits + let effective_lines = t.desired_height(small_area.width); + assert!(state.scroll < effective_lines); + } + + #[test] + fn cursor_pos_with_state_basic_and_scroll_behaviors() { + // Case 1: No wrapping needed, height fits — scroll ignored, y maps directly. + let mut t = ta_with("hello world"); + t.set_cursor(3); + let area = Rect::new(2, 5, 20, 3); + // Even if an absurd scroll is provided, when content fits the area the + // effective scroll is 0 and the cursor position matches cursor_pos. + let bad_state = TextAreaState { scroll: 999 }; + let (x1, y1) = t.cursor_pos(area).unwrap(); + let (x2, y2) = t.cursor_pos_with_state(area, &bad_state).unwrap(); + assert_eq!((x2, y2), (x1, y1)); + + // Case 2: Cursor below the current window — y should be clamped to the + // bottom row (area.height - 1) after adjusting effective scroll. + let mut t = ta_with("one two three four five six"); + // Force wrapping to many visual lines. + let wrap_width = 4; + let _ = t.desired_height(wrap_width); + // Put cursor somewhere near the end so it's definitely below the first window. + t.set_cursor(t.text().len().saturating_sub(2)); + let small_area = Rect::new(0, 0, wrap_width, 2); + let state = TextAreaState { scroll: 0 }; + let (_x, y) = t.cursor_pos_with_state(small_area, &state).unwrap(); + assert_eq!(y, small_area.y + small_area.height - 1); + + // Case 3: Cursor above the current window — y should be top row (0) + // when the provided scroll is too large. + let mut t = ta_with("alpha beta gamma delta epsilon zeta"); + let wrap_width = 5; + let lines = t.desired_height(wrap_width); + // Place cursor near start so an excessive scroll moves it to top row. + t.set_cursor(1); + let area = Rect::new(0, 0, wrap_width, 3); + let state = TextAreaState { + scroll: lines.saturating_mul(2), + }; + let (_x, y) = t.cursor_pos_with_state(area, &state).unwrap(); + assert_eq!(y, area.y); + } + + #[test] + fn wrapped_navigation_across_visual_lines() { + let mut t = ta_with("abcdefghij"); + // Force wrapping at width 4: lines -> ["abcd", "efgh", "ij"] + let _ = t.desired_height(4); + + // From the very start, moving down should go to the start of the next wrapped line (index 4) + t.set_cursor(0); + t.move_cursor_down(); + assert_eq!(t.cursor(), 4); + + // Cursor at boundary index 4 should be displayed at start of second wrapped line + t.set_cursor(4); + let area = Rect::new(0, 0, 4, 10); + let (x, y) = t.cursor_pos(area).unwrap(); + assert_eq!((x, y), (0, 1)); + + // With state and small height, cursor should be visible at row 0, col 0 + let small_area = Rect::new(0, 0, 4, 1); + let state = TextAreaState::default(); + let (x, y) = t.cursor_pos_with_state(small_area, &state).unwrap(); + assert_eq!((x, y), (0, 0)); + + // Place cursor in the middle of the second wrapped line ("efgh"), at 'g' + t.set_cursor(6); + // Move up should go to same column on previous wrapped line -> index 2 ('c') + t.move_cursor_up(); + assert_eq!(t.cursor(), 2); + + // Move down should return to same position on the next wrapped line -> back to index 6 ('g') + t.move_cursor_down(); + assert_eq!(t.cursor(), 6); + + // Move down again should go to third wrapped line. Target col is 2, but the line has len 2 -> clamp to end + t.move_cursor_down(); + assert_eq!(t.cursor(), t.text().len()); + } + + #[test] + fn cursor_pos_with_state_after_movements() { + let mut t = ta_with("abcdefghij"); + // Wrap width 4 -> visual lines: abcd | efgh | ij + let _ = t.desired_height(4); + let area = Rect::new(0, 0, 4, 2); + let mut state = TextAreaState::default(); + let mut buf = Buffer::empty(area); + + // Start at beginning + t.set_cursor(0); + ratatui::widgets::StatefulWidgetRef::render_ref(&(&t), area, &mut buf, &mut state); + let (x, y) = t.cursor_pos_with_state(area, &state).unwrap(); + assert_eq!((x, y), (0, 0)); + + // Move down to second visual line; should be at bottom row (row 1) within 2-line viewport + t.move_cursor_down(); + ratatui::widgets::StatefulWidgetRef::render_ref(&(&t), area, &mut buf, &mut state); + let (x, y) = t.cursor_pos_with_state(area, &state).unwrap(); + assert_eq!((x, y), (0, 1)); + + // Move down to third visual line; viewport scrolls and keeps cursor on bottom row + t.move_cursor_down(); + ratatui::widgets::StatefulWidgetRef::render_ref(&(&t), area, &mut buf, &mut state); + let (x, y) = t.cursor_pos_with_state(area, &state).unwrap(); + assert_eq!((x, y), (0, 1)); + + // Move up to second visual line; with current scroll, it appears on top row + t.move_cursor_up(); + ratatui::widgets::StatefulWidgetRef::render_ref(&(&t), area, &mut buf, &mut state); + let (x, y) = t.cursor_pos_with_state(area, &state).unwrap(); + assert_eq!((x, y), (0, 0)); + + // Column preservation across moves: set to col 2 on first line, move down + t.set_cursor(2); + ratatui::widgets::StatefulWidgetRef::render_ref(&(&t), area, &mut buf, &mut state); + let (x0, y0) = t.cursor_pos_with_state(area, &state).unwrap(); + assert_eq!((x0, y0), (2, 0)); + t.move_cursor_down(); + ratatui::widgets::StatefulWidgetRef::render_ref(&(&t), area, &mut buf, &mut state); + let (x1, y1) = t.cursor_pos_with_state(area, &state).unwrap(); + assert_eq!((x1, y1), (2, 1)); + } + + #[test] + fn wrapped_navigation_with_newlines_and_spaces() { + // Include spaces and an explicit newline to exercise boundaries + let mut t = ta_with("word1 word2\nword3"); + // Width 6 will wrap "word1 " and then "word2" before the newline + let _ = t.desired_height(6); + + // Put cursor on the second wrapped line before the newline, at column 1 of "word2" + let start_word2 = t.text().find("word2").unwrap(); + t.set_cursor(start_word2 + 1); + + // Up should go to first wrapped line, column 1 -> index 1 + t.move_cursor_up(); + assert_eq!(t.cursor(), 1); + + // Down should return to the same visual column on "word2" + t.move_cursor_down(); + assert_eq!(t.cursor(), start_word2 + 1); + + // Down again should cross the logical newline to the next visual line ("word3"), clamped to its length if needed + t.move_cursor_down(); + let start_word3 = t.text().find("word3").unwrap(); + assert!(t.cursor() >= start_word3 && t.cursor() <= start_word3 + "word3".len()); + } + + #[test] + fn wrapped_navigation_with_wide_graphemes() { + // Four thumbs up, each of display width 2, with width 3 to force wrapping inside grapheme boundaries + let mut t = ta_with("👍👍👍👍"); + let _ = t.desired_height(3); + + // Put cursor after the second emoji (which should be on first wrapped line) + t.set_cursor("👍👍".len()); + + // Move down should go to the start of the next wrapped line (same column preserved but clamped) + t.move_cursor_down(); + // We expect to land somewhere within the third emoji or at the start of it + let pos_after_down = t.cursor(); + assert!(pos_after_down >= "👍👍".len()); + + // Moving up should take us back to the original position + t.move_cursor_up(); + assert_eq!(t.cursor(), "👍👍".len()); + } + + #[test] + fn fuzz_textarea_randomized() { + // Deterministic seed for reproducibility + // Seed the RNG based on the current day in Pacific Time (PST/PDT). This + // keeps the fuzz test deterministic within a day while still varying + // day-to-day to improve coverage. + #[allow(clippy::unwrap_used)] + let pst_today_seed: u64 = (chrono::Utc::now() - chrono::Duration::hours(8)) + .date_naive() + .and_hms_opt(0, 0, 0) + .unwrap() + .and_utc() + .timestamp() as u64; + let mut rng = rand::rngs::StdRng::seed_from_u64(pst_today_seed); + + for _case in 0..10_000 { + let mut ta = TextArea::new(); + let mut state = TextAreaState::default(); + // Start with a random base string + let base_len = rng.gen_range(0..30); + let mut base = String::new(); + for _ in 0..base_len { + base.push_str(&rand_grapheme(&mut rng)); + } + ta.set_text(&base); + // Choose a valid char boundary for initial cursor + let mut boundaries: Vec = vec![0]; + boundaries.extend(ta.text().char_indices().map(|(i, _)| i).skip(1)); + boundaries.push(ta.text().len()); + let init = boundaries[rng.gen_range(0..boundaries.len())]; + ta.set_cursor(init); + + let mut width: u16 = rng.gen_range(1..=12); + let mut height: u16 = rng.gen_range(1..=4); + + for _step in 0..200 { + // Mostly stable width/height, occasionally change + if rng.gen_bool(0.1) { + width = rng.gen_range(1..=12); + } + if rng.gen_bool(0.1) { + height = rng.gen_range(1..=4); + } + + // Pick an operation + match rng.gen_range(0..14) { + 0 => { + // insert small random string at cursor + let len = rng.gen_range(0..6); + let mut s = String::new(); + for _ in 0..len { + s.push_str(&rand_grapheme(&mut rng)); + } + ta.insert_str(&s); + } + 1 => { + // replace_range with small random slice + let mut b: Vec = vec![0]; + b.extend(ta.text().char_indices().map(|(i, _)| i).skip(1)); + b.push(ta.text().len()); + let i1 = rng.gen_range(0..b.len()); + let i2 = rng.gen_range(0..b.len()); + let (start, end) = if b[i1] <= b[i2] { + (b[i1], b[i2]) + } else { + (b[i2], b[i1]) + }; + let insert_len = rng.gen_range(0..=4); + let mut s = String::new(); + for _ in 0..insert_len { + s.push_str(&rand_grapheme(&mut rng)); + } + let before = ta.text().len(); + ta.replace_range(start..end, &s); + let after = ta.text().len(); + assert_eq!( + after as isize, + before as isize + (s.len() as isize) - ((end - start) as isize) + ); + } + 2 => ta.delete_backward(rng.gen_range(0..=3)), + 3 => ta.delete_forward(rng.gen_range(0..=3)), + 4 => ta.delete_backward_word(), + 5 => ta.kill_to_beginning_of_line(), + 6 => ta.kill_to_end_of_line(), + 7 => ta.move_cursor_left(), + 8 => ta.move_cursor_right(), + 9 => ta.move_cursor_up(), + 10 => ta.move_cursor_down(), + 11 => ta.move_cursor_to_beginning_of_line(true), + 12 => ta.move_cursor_to_end_of_line(true), + _ => { + // Jump to word boundaries + if rng.gen_bool(0.5) { + let p = ta.beginning_of_previous_word(); + ta.set_cursor(p); + } else { + let p = ta.end_of_next_word(); + ta.set_cursor(p); + } + } + } + + // Sanity invariants + assert!(ta.cursor() <= ta.text().len()); + + // Render and compute cursor positions; ensure they are in-bounds and do not panic + let area = Rect::new(0, 0, width, height); + // Stateless render into an area tall enough for all wrapped lines + let total_lines = ta.desired_height(width); + let full_area = Rect::new(0, 0, width, total_lines.max(1)); + let mut buf = Buffer::empty(full_area); + ratatui::widgets::WidgetRef::render_ref(&(&ta), full_area, &mut buf); + + // cursor_pos: x must be within width when present + let _ = ta.cursor_pos(area); + + // cursor_pos_with_state: always within viewport rows + let (_x, _y) = ta + .cursor_pos_with_state(area, &state) + .unwrap_or((area.x, area.y)); + + // Stateful render should not panic, and updates scroll + let mut sbuf = Buffer::empty(area); + ratatui::widgets::StatefulWidgetRef::render_ref( + &(&ta), + area, + &mut sbuf, + &mut state, + ); + + // After wrapping, desired height equals the number of lines we would render without scroll + let total_lines = total_lines as usize; + // state.scroll must not exceed total_lines when content fits within area height + if (height as usize) >= total_lines { + assert_eq!(state.scroll, 0); + } + } + } + } +} diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 374a4b5688..31151d5646 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -509,6 +509,10 @@ impl ChatWidget<'_> { self.bottom_pane .set_token_usage(self.token_usage.clone(), self.config.model_context_window); } + + pub fn cursor_pos(&self, area: Rect) -> Option<(u16, u16)> { + self.bottom_pane.cursor_pos(area) + } } impl WidgetRef for &ChatWidget<'_> { From 78a1d49fac0fbc150e5e862fd3f24c8af50e393c Mon Sep 17 00:00:00 2001 From: Jeremy Rose <172423086+nornagon-openai@users.noreply.github.com> Date: Sun, 3 Aug 2025 11:33:44 -0700 Subject: [PATCH 048/100] fix command duration display (#1806) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit we were always displaying "0ms" before. Screenshot 2025-08-02 at 10 51 22 PM --- codex-rs/core/src/codex.rs | 50 +++++++++---------- codex-rs/core/src/protocol.rs | 2 + codex-rs/core/tests/live_agent.rs | 6 +-- .../src/event_processor_with_human_output.rs | 10 ++-- codex-rs/tui/src/chatwidget.rs | 4 +- 5 files changed, 32 insertions(+), 40 deletions(-) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index e759acb4a2..7004dcfcb7 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -396,11 +396,15 @@ impl Session { &self, sub_id: &str, call_id: &str, - stdout: &str, - stderr: &str, - exit_code: i32, + output: &ExecToolCallOutput, is_apply_patch: bool, ) { + let ExecToolCallOutput { + stdout, + stderr, + duration, + exit_code, + } = output; // Because stdout and stderr could each be up to 100 KiB, we send // truncated versions. const MAX_STREAM_OUTPUT: usize = 5 * 1024; // 5KiB @@ -412,14 +416,15 @@ impl Session { call_id: call_id.to_string(), stdout, stderr, - success: exit_code == 0, + success: *exit_code == 0, }) } else { EventMsg::ExecCommandEnd(ExecCommandEndEvent { call_id: call_id.to_string(), stdout, stderr, - exit_code, + duration: *duration, + exit_code: *exit_code, }) }; @@ -1775,23 +1780,21 @@ async fn handle_container_exec_with_params( stdout, stderr, duration, - } = output; + } = &output; sess.notify_exec_command_end( &sub_id, &call_id, - &stdout, - &stderr, - exit_code, + &output, exec_command_context.apply_patch.is_some(), ) .await; - let is_success = exit_code == 0; + let is_success = *exit_code == 0; let content = format_exec_output( - if is_success { &stdout } else { &stderr }, - exit_code, - duration, + if is_success { stdout } else { stderr }, + *exit_code, + *duration, ); ResponseInputItem::FunctionCallOutput { @@ -1900,23 +1903,16 @@ async fn handle_sandbox_error( stdout, stderr, duration, - } = retry_output; + } = &retry_output; - sess.notify_exec_command_end( - &sub_id, - &call_id, - &stdout, - &stderr, - exit_code, - is_apply_patch, - ) - .await; + sess.notify_exec_command_end(&sub_id, &call_id, &retry_output, is_apply_patch) + .await; - let is_success = exit_code == 0; + let is_success = *exit_code == 0; let content = format_exec_output( - if is_success { &stdout } else { &stderr }, - exit_code, - duration, + if is_success { stdout } else { stderr }, + *exit_code, + *duration, ); ResponseInputItem::FunctionCallOutput { diff --git a/codex-rs/core/src/protocol.rs b/codex-rs/core/src/protocol.rs index 1bfeee56ab..cbb211d955 100644 --- a/codex-rs/core/src/protocol.rs +++ b/codex-rs/core/src/protocol.rs @@ -523,6 +523,8 @@ pub struct ExecCommandEndEvent { pub stderr: String, /// The command's exit code. pub exit_code: i32, + /// The duration of the command execution. + pub duration: Duration, } #[derive(Debug, Clone, Deserialize, Serialize)] diff --git a/codex-rs/core/tests/live_agent.rs b/codex-rs/core/tests/live_agent.rs index 95408e20e5..81b3bb2a12 100644 --- a/codex-rs/core/tests/live_agent.rs +++ b/codex-rs/core/tests/live_agent.rs @@ -177,8 +177,7 @@ async fn live_shell_function_call() { match ev.msg { EventMsg::ExecCommandBegin(codex_core::protocol::ExecCommandBeginEvent { command, - call_id: _, - cwd: _, + .. }) => { assert_eq!(command, vec!["echo", MARKER]); saw_begin = true; @@ -186,8 +185,7 @@ async fn live_shell_function_call() { EventMsg::ExecCommandEnd(codex_core::protocol::ExecCommandEndEvent { stdout, exit_code, - call_id: _, - stderr: _, + .. }) => { assert_eq!(exit_code, 0, "echo returned non‑zero exit code"); assert!(stdout.contains(MARKER)); diff --git a/codex-rs/exec/src/event_processor_with_human_output.rs b/codex-rs/exec/src/event_processor_with_human_output.rs index 7393ab72a0..72e2f9298f 100644 --- a/codex-rs/exec/src/event_processor_with_human_output.rs +++ b/codex-rs/exec/src/event_processor_with_human_output.rs @@ -106,7 +106,6 @@ impl EventProcessorWithHumanOutput { struct ExecCommandBegin { command: Vec, - start_time: Instant, } struct PatchApplyBegin { @@ -228,7 +227,6 @@ impl EventProcessor for EventProcessorWithHumanOutput { call_id.clone(), ExecCommandBegin { command: command.clone(), - start_time: Instant::now(), }, ); ts_println!( @@ -244,16 +242,14 @@ impl EventProcessor for EventProcessorWithHumanOutput { call_id, stdout, stderr, + duration, exit_code, }) => { let exec_command = self.call_id_to_command.remove(&call_id); - let (duration, call) = if let Some(ExecCommandBegin { - command, - start_time, - }) = exec_command + let (duration, call) = if let Some(ExecCommandBegin { command, .. }) = exec_command { ( - format!(" in {}", format_elapsed(start_time)), + format!(" in {}", format_duration(duration)), format!("{}", escape_command(&command).style(self.bold)), ) } else { diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 31151d5646..e5ebf58a07 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -1,7 +1,6 @@ use std::collections::HashMap; use std::path::PathBuf; use std::sync::Arc; -use std::time::Duration; use codex_core::codex_wrapper::CodexConversation; use codex_core::codex_wrapper::init_codex; @@ -390,6 +389,7 @@ impl ChatWidget<'_> { EventMsg::ExecCommandEnd(ExecCommandEndEvent { call_id, exit_code, + duration, stdout, stderr, }) => { @@ -400,7 +400,7 @@ impl ChatWidget<'_> { exit_code, stdout, stderr, - duration: Duration::from_secs(0), + duration, }, )); } From 2576fadc742cc0030800214be55c5c7833521679 Mon Sep 17 00:00:00 2001 From: Jeremy Rose <172423086+nornagon-openai@users.noreply.github.com> Date: Sun, 3 Aug 2025 11:51:33 -0700 Subject: [PATCH 049/100] shimmer on working (#1807) change the animation on "working" to be a text shimmer https://github.com/user-attachments/assets/f64529eb-1c64-493a-8d97-0f68b964bdd0 --- codex-rs/Cargo.lock | 16 +++++ codex-rs/tui/Cargo.toml | 1 + codex-rs/tui/src/status_indicator_widget.rs | 73 +++++++++++++-------- 3 files changed, 61 insertions(+), 29 deletions(-) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index e1a0e162dc..7d4e41d0b1 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -869,6 +869,7 @@ dependencies = [ "shlex", "strum 0.27.2", "strum_macros 0.27.2", + "supports-color", "textwrap 0.16.2", "tokio", "tracing", @@ -2337,6 +2338,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "is_ci" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45" + [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -4378,6 +4385,15 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "supports-color" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c64fc7232dd8d2e4ac5ce4ef302b1d81e0b80d055b9d77c7c4f51f6aa4c867d6" +dependencies = [ + "is_ci", +] + [[package]] name = "syn" version = "1.0.109" diff --git a/codex-rs/tui/Cargo.toml b/codex-rs/tui/Cargo.toml index 823fd1428e..a571b32c8d 100644 --- a/codex-rs/tui/Cargo.toml +++ b/codex-rs/tui/Cargo.toml @@ -48,6 +48,7 @@ serde_json = { version = "1", features = ["preserve_order"] } shlex = "1.3.0" strum = "0.27.2" strum_macros = "0.27.2" +supports-color = "3.0.2" textwrap = "0.16.2" tokio = { version = "1", features = [ "io-std", diff --git a/codex-rs/tui/src/status_indicator_widget.rs b/codex-rs/tui/src/status_indicator_widget.rs index 7e6d267481..aa18ac6fa5 100644 --- a/codex-rs/tui/src/status_indicator_widget.rs +++ b/codex-rs/tui/src/status_indicator_widget.rs @@ -57,7 +57,7 @@ impl StatusIndicatorWidget { thread::spawn(move || { let mut counter = 0usize; while running_clone.load(Ordering::Relaxed) { - std::thread::sleep(Duration::from_millis(200)); + std::thread::sleep(Duration::from_millis(100)); counter = counter.wrapping_add(1); frame_idx_clone.store(counter, Ordering::Relaxed); app_event_tx_clone.send(AppEvent::RequestRedraw); @@ -98,46 +98,51 @@ impl WidgetRef for StatusIndicatorWidget { .borders(Borders::LEFT) .border_type(BorderType::QuadrantOutside) .border_style(widget_style.dim()); - // Animated 3‑dot pattern inside brackets. The *active* dot is bold - // white, the others are dim. - const DOT_COUNT: usize = 3; let idx = self.frame_idx.load(std::sync::atomic::Ordering::Relaxed); - let phase = idx % (DOT_COUNT * 2 - 2); - let active = if phase < DOT_COUNT { - phase - } else { - (DOT_COUNT * 2 - 2) - phase - }; + let header_text = "Working"; + let header_chars: Vec = header_text.chars().collect(); + + let padding = 4usize; // virtual padding around the word for smoother loop + let period = header_chars.len() + padding * 2; + let pos = idx % period; + + let has_true_color = supports_color::on_cached(supports_color::Stream::Stdout) + .map(|level| level.has_16m) + .unwrap_or(false); + + // Width of the bright band (in characters). + let band_half_width = 2.0; let mut header_spans: Vec> = Vec::new(); + for (i, ch) in header_chars.iter().enumerate() { + let i_pos = i as isize + padding as isize; + let pos = pos as isize; + let dist = (i_pos - pos).abs() as f32; - header_spans.push(Span::styled( - "Working ", - Style::default() - .fg(Color::White) - .add_modifier(Modifier::BOLD), - )); + let t = if dist <= band_half_width { + let x = std::f32::consts::PI * (dist / band_half_width); + 0.5 * (1.0 + x.cos()) + } else { + 0.0 + }; - header_spans.push(Span::styled( - "[", - Style::default() - .fg(Color::White) - .add_modifier(Modifier::BOLD), - )); - - for i in 0..DOT_COUNT { - let style = if i == active { + let brightness = 0.4 + 0.6 * t; + let level = (brightness * 255.0).clamp(0.0, 255.0) as u8; + let style = if has_true_color { Style::default() - .fg(Color::White) + .fg(Color::Rgb(level, level, level)) .add_modifier(Modifier::BOLD) } else { - Style::default().dim() + // Bold makes dark gray and gray look the same, so don't use it + // when true color is not supported. + Style::default().fg(color_for_level(level)) }; - header_spans.push(Span::styled(".", style)); + + header_spans.push(Span::styled(ch.to_string(), style)); } header_spans.push(Span::styled( - "] ", + " ", Style::default() .fg(Color::White) .add_modifier(Modifier::BOLD), @@ -189,3 +194,13 @@ impl WidgetRef for StatusIndicatorWidget { paragraph.render_ref(area, buf); } } + +fn color_for_level(level: u8) -> Color { + if level < 128 { + Color::DarkGray + } else if level < 192 { + Color::Gray + } else { + Color::White + } +} From e3565a3f438c30c9d36412d2817346c7accd487c Mon Sep 17 00:00:00 2001 From: Dylan Date: Sun, 3 Aug 2025 13:05:48 -0700 Subject: [PATCH 050/100] [sandbox] Filter out certain non-sandbox errors (#1804) ## Summary Users frequently complain about re-approving commands that have failed for non-sandbox reasons. We can't diagnose with complete accuracy which errors happened because of a sandbox failure, but we can start to eliminate some common simple cases. This PR captures the most common case I've seen, which is a `command not found` error. ## Testing - [x] Added unit tests - [x] Ran a few cases locally --- codex-rs/core/src/exec.rs | 26 +++++++++++--- codex-rs/core/src/lib.rs | 3 +- codex-rs/core/tests/exec.rs | 69 +++++++++++++++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 6 deletions(-) create mode 100644 codex-rs/core/tests/exec.rs diff --git a/codex-rs/core/src/exec.rs b/codex-rs/core/src/exec.rs index 5301f0220d..dce02cc5e2 100644 --- a/codex-rs/core/src/exec.rs +++ b/codex-rs/core/src/exec.rs @@ -140,11 +140,7 @@ pub async fn process_exec_tool_call( let exit_code = raw_output.exit_status.code().unwrap_or(-1); - // NOTE(ragona): This is much less restrictive than the previous check. If we exec - // a command, and it returns anything other than success, we assume that it may have - // been a sandboxing error and allow the user to retry. (The user of course may choose - // not to retry, or in a non-interactive mode, would automatically reject the approval.) - if exit_code != 0 && sandbox_type != SandboxType::None { + if exit_code != 0 && is_likely_sandbox_denied(sandbox_type, exit_code) { return Err(CodexErr::Sandbox(SandboxErr::Denied( exit_code, stdout, stderr, ))); @@ -223,6 +219,26 @@ fn create_linux_sandbox_command_args( linux_cmd } +/// We don't have a fully deterministic way to tell if our command failed +/// because of the sandbox - a command in the user's zshrc file might hit an +/// error, but the command itself might fail or succeed for other reasons. +/// For now, we conservatively check for 'command not found' (exit code 127), +/// and can add additional cases as necessary. +fn is_likely_sandbox_denied(sandbox_type: SandboxType, exit_code: i32) -> bool { + if sandbox_type == SandboxType::None { + return false; + } + + // Quick rejects: well-known non-sandbox shell exit codes + // 127: command not found, 2: misuse of shell builtins + if exit_code == 127 { + return false; + } + + // For all other cases, we assume the sandbox is the cause + true +} + #[derive(Debug)] pub struct RawExecToolCallOutput { pub exit_status: ExitStatus, diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index a33e185afb..80f9014954 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -38,7 +38,7 @@ pub mod plan_tool; mod project_doc; pub mod protocol; mod rollout; -mod safety; +pub(crate) mod safety; pub mod seatbelt; pub mod shell; pub mod spawn; @@ -47,3 +47,4 @@ pub mod util; pub use apply_patch::CODEX_APPLY_PATCH_ARG1; pub use client_common::model_supports_reasoning_summaries; +pub use safety::get_platform_sandbox; diff --git a/codex-rs/core/tests/exec.rs b/codex-rs/core/tests/exec.rs new file mode 100644 index 0000000000..da169296ed --- /dev/null +++ b/codex-rs/core/tests/exec.rs @@ -0,0 +1,69 @@ +#![cfg(target_os = "macos")] +#![expect(clippy::expect_used)] + +use std::collections::HashMap; +use std::sync::Arc; + +use codex_core::exec::ExecParams; +use codex_core::exec::SandboxType; +use codex_core::exec::process_exec_tool_call; +use codex_core::protocol::SandboxPolicy; +use codex_core::spawn::CODEX_SANDBOX_ENV_VAR; +use tempfile::TempDir; +use tokio::sync::Notify; + +use codex_core::get_platform_sandbox; + +async fn run_test_cmd(tmp: TempDir, cmd: Vec<&str>, should_be_ok: bool) { + if std::env::var(CODEX_SANDBOX_ENV_VAR) == Ok("seatbelt".to_string()) { + eprintln!("{CODEX_SANDBOX_ENV_VAR} is set to 'seatbelt', skipping test."); + return; + } + + let sandbox_type = get_platform_sandbox().expect("should be able to get sandbox type"); + assert_eq!(sandbox_type, SandboxType::MacosSeatbelt); + + let params = ExecParams { + command: cmd.iter().map(|s| s.to_string()).collect(), + cwd: tmp.path().to_path_buf(), + timeout_ms: Some(1000), + env: HashMap::new(), + }; + + let ctrl_c = Arc::new(Notify::new()); + let policy = SandboxPolicy::new_read_only_policy(); + + let result = process_exec_tool_call(params, sandbox_type, ctrl_c, &policy, &None, None).await; + + assert!(result.is_ok() == should_be_ok); +} + +/// Command succeeds with exit code 0 normally +#[tokio::test] +async fn exit_code_0_succeeds() { + let tmp = TempDir::new().expect("should be able to create temp dir"); + let cmd = vec!["echo", "hello"]; + + run_test_cmd(tmp, cmd, true).await +} + +/// Command not found returns exit code 127, this is not considered a sandbox error +#[tokio::test] +async fn exit_command_not_found_is_ok() { + let tmp = TempDir::new().expect("should be able to create temp dir"); + let cmd = vec!["/bin/bash", "-c", "nonexistent_command_12345"]; + run_test_cmd(tmp, cmd, true).await +} + +/// Writing a file fails and should be considered a sandbox error +#[tokio::test] +async fn write_file_fails_as_sandbox_error() { + let tmp = TempDir::new().expect("should be able to create temp dir"); + let path = tmp.path().join("test.txt"); + let cmd = vec![ + "/user/bin/touch", + path.to_str().expect("should be able to get path"), + ]; + + run_test_cmd(tmp, cmd, false).await; +} From 1f3318c1c5b83e29b6d6e61eb2bf7d599581023e Mon Sep 17 00:00:00 2001 From: Gabriel Peal Date: Mon, 4 Aug 2025 08:57:04 -0700 Subject: [PATCH 051/100] Add a TurnDiffTracker to create a unified diff for an entire turn (#1770) This lets us show an accumulating diff across all patches in a turn. Refer to the docs for TurnDiffTracker for implementation details. There are multiple ways this could have been done and this felt like the right tradeoff between reliability and completeness: *Pros* * It will pick up all changes to files that the model touched including if they prettier or another command that updates them. * It will not pick up changes made by the user or other agents to files it didn't modify. *Cons* * It will pick up changes that the user made to a file that the model also touched * It will not pick up changes to codegen or files that were not modified with apply_patch --- codex-rs/Cargo.lock | 1 + codex-rs/core/Cargo.toml | 1 + codex-rs/core/src/codex.rs | 111 ++- codex-rs/core/src/lib.rs | 1 + codex-rs/core/src/protocol.rs | 7 + codex-rs/core/src/turn_diff_tracker.rs | 887 ++++++++++++++++++ .../src/event_processor_with_human_output.rs | 6 + codex-rs/mcp-server/src/codex_tool_runner.rs | 1 + codex-rs/mcp-server/src/conversation_loop.rs | 1 + 9 files changed, 998 insertions(+), 18 deletions(-) create mode 100644 codex-rs/core/src/turn_diff_tracker.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 7d4e41d0b1..eb4eccd897 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -699,6 +699,7 @@ dependencies = [ "serde_json", "sha1", "shlex", + "similar", "strum_macros 0.27.2", "tempfile", "thiserror 2.0.12", diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index db3fd4f834..466e9adf02 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -34,6 +34,7 @@ serde_json = "1" serde_bytes = "0.11" sha1 = "0.10.6" shlex = "1.3.0" +similar = "2.7.0" strum_macros = "0.27.2" thiserror = "2.0.12" time = { version = "0.3", features = ["formatting", "local-offset", "macros"] } diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 7004dcfcb7..568d87c4a8 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -85,11 +85,13 @@ use crate::protocol::SandboxPolicy; use crate::protocol::SessionConfiguredEvent; use crate::protocol::Submission; use crate::protocol::TaskCompleteEvent; +use crate::protocol::TurnDiffEvent; use crate::rollout::RolloutRecorder; use crate::safety::SafetyCheck; use crate::safety::assess_command_safety; use crate::safety::assess_safety_for_untrusted_command; use crate::shell; +use crate::turn_diff_tracker::TurnDiffTracker; use crate::user_notification::UserNotification; use crate::util::backoff; @@ -362,7 +364,11 @@ impl Session { } } - async fn notify_exec_command_begin(&self, exec_command_context: ExecCommandContext) { + async fn on_exec_command_begin( + &self, + turn_diff_tracker: &mut TurnDiffTracker, + exec_command_context: ExecCommandContext, + ) { let ExecCommandContext { sub_id, call_id, @@ -374,11 +380,15 @@ impl Session { Some(ApplyPatchCommandContext { user_explicitly_approved_this_action, changes, - }) => EventMsg::PatchApplyBegin(PatchApplyBeginEvent { - call_id, - auto_approved: !user_explicitly_approved_this_action, - changes, - }), + }) => { + turn_diff_tracker.on_patch_begin(&changes); + + EventMsg::PatchApplyBegin(PatchApplyBeginEvent { + call_id, + auto_approved: !user_explicitly_approved_this_action, + changes, + }) + } None => EventMsg::ExecCommandBegin(ExecCommandBeginEvent { call_id, command: command_for_display.clone(), @@ -392,8 +402,10 @@ impl Session { let _ = self.tx_event.send(event).await; } - async fn notify_exec_command_end( + #[allow(clippy::too_many_arguments)] + async fn on_exec_command_end( &self, + turn_diff_tracker: &mut TurnDiffTracker, sub_id: &str, call_id: &str, output: &ExecToolCallOutput, @@ -433,6 +445,20 @@ impl Session { msg, }; let _ = self.tx_event.send(event).await; + + // If this is an apply_patch, after we emit the end patch, emit a second event + // with the full turn diff if there is one. + if is_apply_patch { + let unified_diff = turn_diff_tracker.get_unified_diff(); + if let Ok(Some(unified_diff)) = unified_diff { + let msg = EventMsg::TurnDiff(TurnDiffEvent { unified_diff }); + let event = Event { + id: sub_id.into(), + msg, + }; + let _ = self.tx_event.send(event).await; + } + } } /// Helper that emits a BackgroundEvent with the given message. This keeps @@ -1006,6 +1032,10 @@ async fn run_task(sess: Arc, sub_id: String, input: Vec) { .await; let last_agent_message: Option; + // Although from the perspective of codex.rs, TurnDiffTracker has the lifecycle of a Task which contains + // many turns, from the perspective of the user, it is a single turn. + let mut turn_diff_tracker = TurnDiffTracker::new(); + loop { // Note that pending_input would be something like a message the user // submitted through the UI while the model was running. Though the UI @@ -1037,7 +1067,7 @@ async fn run_task(sess: Arc, sub_id: String, input: Vec) { }) }) .collect(); - match run_turn(&sess, sub_id.clone(), turn_input).await { + match run_turn(&sess, &mut turn_diff_tracker, sub_id.clone(), turn_input).await { Ok(turn_output) => { let mut items_to_record_in_conversation_history = Vec::::new(); let mut responses = Vec::::new(); @@ -1163,6 +1193,7 @@ async fn run_task(sess: Arc, sub_id: String, input: Vec) { async fn run_turn( sess: &Session, + turn_diff_tracker: &mut TurnDiffTracker, sub_id: String, input: Vec, ) -> CodexResult> { @@ -1177,7 +1208,7 @@ async fn run_turn( let mut retries = 0; loop { - match try_run_turn(sess, &sub_id, &prompt).await { + match try_run_turn(sess, turn_diff_tracker, &sub_id, &prompt).await { Ok(output) => return Ok(output), Err(CodexErr::Interrupted) => return Err(CodexErr::Interrupted), Err(CodexErr::EnvVar(var)) => return Err(CodexErr::EnvVar(var)), @@ -1223,6 +1254,7 @@ struct ProcessedResponseItem { async fn try_run_turn( sess: &Session, + turn_diff_tracker: &mut TurnDiffTracker, sub_id: &str, prompt: &Prompt, ) -> CodexResult> { @@ -1310,7 +1342,8 @@ async fn try_run_turn( match event { ResponseEvent::Created => {} ResponseEvent::OutputItemDone(item) => { - let response = handle_response_item(sess, sub_id, item.clone()).await?; + let response = + handle_response_item(sess, turn_diff_tracker, sub_id, item.clone()).await?; output.push(ProcessedResponseItem { item, response }); } @@ -1328,6 +1361,16 @@ async fn try_run_turn( .ok(); } + let unified_diff = turn_diff_tracker.get_unified_diff(); + if let Ok(Some(unified_diff)) = unified_diff { + let msg = EventMsg::TurnDiff(TurnDiffEvent { unified_diff }); + let event = Event { + id: sub_id.to_string(), + msg, + }; + let _ = sess.tx_event.send(event).await; + } + return Ok(output); } ResponseEvent::OutputTextDelta(delta) => { @@ -1432,6 +1475,7 @@ async fn run_compact_task( async fn handle_response_item( sess: &Session, + turn_diff_tracker: &mut TurnDiffTracker, sub_id: &str, item: ResponseItem, ) -> CodexResult> { @@ -1469,7 +1513,17 @@ async fn handle_response_item( .. } => { info!("FunctionCall: {arguments}"); - Some(handle_function_call(sess, sub_id.to_string(), name, arguments, call_id).await) + Some( + handle_function_call( + sess, + turn_diff_tracker, + sub_id.to_string(), + name, + arguments, + call_id, + ) + .await, + ) } ResponseItem::LocalShellCall { id, @@ -1504,6 +1558,7 @@ async fn handle_response_item( handle_container_exec_with_params( exec_params, sess, + turn_diff_tracker, sub_id.to_string(), effective_call_id, ) @@ -1521,6 +1576,7 @@ async fn handle_response_item( async fn handle_function_call( sess: &Session, + turn_diff_tracker: &mut TurnDiffTracker, sub_id: String, name: String, arguments: String, @@ -1534,7 +1590,8 @@ async fn handle_function_call( return *output; } }; - handle_container_exec_with_params(params, sess, sub_id, call_id).await + handle_container_exec_with_params(params, sess, turn_diff_tracker, sub_id, call_id) + .await } "update_plan" => handle_update_plan(sess, arguments, sub_id, call_id).await, _ => { @@ -1608,6 +1665,7 @@ fn maybe_run_with_user_profile(params: ExecParams, sess: &Session) -> ExecParams async fn handle_container_exec_with_params( params: ExecParams, sess: &Session, + turn_diff_tracker: &mut TurnDiffTracker, sub_id: String, call_id: String, ) -> ResponseInputItem { @@ -1755,7 +1813,7 @@ async fn handle_container_exec_with_params( }, ), }; - sess.notify_exec_command_begin(exec_command_context.clone()) + sess.on_exec_command_begin(turn_diff_tracker, exec_command_context.clone()) .await; let params = maybe_run_with_user_profile(params, sess); @@ -1782,7 +1840,8 @@ async fn handle_container_exec_with_params( duration, } = &output; - sess.notify_exec_command_end( + sess.on_exec_command_end( + turn_diff_tracker, &sub_id, &call_id, &output, @@ -1806,7 +1865,15 @@ async fn handle_container_exec_with_params( } } Err(CodexErr::Sandbox(error)) => { - handle_sandbox_error(params, exec_command_context, error, sandbox_type, sess).await + handle_sandbox_error( + turn_diff_tracker, + params, + exec_command_context, + error, + sandbox_type, + sess, + ) + .await } Err(e) => { // Handle non-sandbox errors @@ -1822,6 +1889,7 @@ async fn handle_container_exec_with_params( } async fn handle_sandbox_error( + turn_diff_tracker: &mut TurnDiffTracker, params: ExecParams, exec_command_context: ExecCommandContext, error: SandboxErr, @@ -1878,7 +1946,8 @@ async fn handle_sandbox_error( sess.notify_background_event(&sub_id, "retrying command without sandbox") .await; - sess.notify_exec_command_begin(exec_command_context).await; + sess.on_exec_command_begin(turn_diff_tracker, exec_command_context) + .await; // This is an escalated retry; the policy will not be // examined and the sandbox has been set to `None`. @@ -1905,8 +1974,14 @@ async fn handle_sandbox_error( duration, } = &retry_output; - sess.notify_exec_command_end(&sub_id, &call_id, &retry_output, is_apply_patch) - .await; + sess.on_exec_command_end( + turn_diff_tracker, + &sub_id, + &call_id, + &retry_output, + is_apply_patch, + ) + .await; let is_success = *exit_code == 0; let content = format_exec_output( diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index 80f9014954..4f083d9e56 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -42,6 +42,7 @@ pub(crate) mod safety; pub mod seatbelt; pub mod shell; pub mod spawn; +pub mod turn_diff_tracker; mod user_notification; pub mod util; diff --git a/codex-rs/core/src/protocol.rs b/codex-rs/core/src/protocol.rs index cbb211d955..82591a2c78 100644 --- a/codex-rs/core/src/protocol.rs +++ b/codex-rs/core/src/protocol.rs @@ -387,6 +387,8 @@ pub enum EventMsg { /// Notification that a patch application has finished. PatchApplyEnd(PatchApplyEndEvent), + TurnDiff(TurnDiffEvent), + /// Response to GetHistoryEntryRequest. GetHistoryEntryResponse(GetHistoryEntryResponseEvent), @@ -598,6 +600,11 @@ pub struct PatchApplyEndEvent { pub success: bool, } +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct TurnDiffEvent { + pub unified_diff: String, +} + #[derive(Debug, Clone, Deserialize, Serialize)] pub struct GetHistoryEntryResponseEvent { pub offset: usize, diff --git a/codex-rs/core/src/turn_diff_tracker.rs b/codex-rs/core/src/turn_diff_tracker.rs new file mode 100644 index 0000000000..7026d7bb32 --- /dev/null +++ b/codex-rs/core/src/turn_diff_tracker.rs @@ -0,0 +1,887 @@ +use std::collections::HashMap; +use std::fs; +use std::path::Path; +use std::path::PathBuf; +use std::process::Command; + +use anyhow::Context; +use anyhow::Result; +use anyhow::anyhow; +use sha1::digest::Output; +use uuid::Uuid; + +use crate::protocol::FileChange; + +const ZERO_OID: &str = "0000000000000000000000000000000000000000"; +const DEV_NULL: &str = "/dev/null"; + +struct BaselineFileInfo { + path: PathBuf, + content: Vec, + mode: FileMode, + oid: String, +} + +/// Tracks sets of changes to files and exposes the overall unified diff. +/// Internally, the way this works is now: +/// 1. Maintain an in-memory baseline snapshot of files when they are first seen. +/// For new additions, do not create a baseline so that diffs are shown as proper additions (using /dev/null). +/// 2. Keep a stable internal filename (uuid) per external path for rename tracking. +/// 3. To compute the aggregated unified diff, compare each baseline snapshot to the current file on disk entirely in-memory +/// using the `similar` crate and emit unified diffs with rewritten external paths. +#[derive(Default)] +pub struct TurnDiffTracker { + /// Map external path -> internal filename (uuid). + external_to_temp_name: HashMap, + /// Internal filename -> baseline file info. + baseline_file_info: HashMap, + /// Internal filename -> external path as of current accumulated state (after applying all changes). + /// This is where renames are tracked. + temp_name_to_current_path: HashMap, + /// Cache of known git worktree roots to avoid repeated filesystem walks. + git_root_cache: Vec, +} + +impl TurnDiffTracker { + pub fn new() -> Self { + Self::default() + } + + /// Front-run apply patch calls to track the starting contents of any modified files. + /// - Creates an in-memory baseline snapshot for files that already exist on disk when first seen. + /// - For additions, we intentionally do not create a baseline snapshot so that diffs are proper additions. + /// - Also updates internal mappings for move/rename events. + pub fn on_patch_begin(&mut self, changes: &HashMap) { + for (path, change) in changes.iter() { + // Ensure a stable internal filename exists for this external path. + if !self.external_to_temp_name.contains_key(path) { + let internal = Uuid::new_v4().to_string(); + self.external_to_temp_name + .insert(path.clone(), internal.clone()); + self.temp_name_to_current_path + .insert(internal.clone(), path.clone()); + + // If the file exists on disk now, snapshot as baseline; else leave missing to represent /dev/null. + let baseline_file_info = if path.exists() { + let mode = file_mode_for_path(path); + let mode_val = mode.unwrap_or(FileMode::Regular); + let content = blob_bytes(path, &mode_val).unwrap_or_default(); + let oid = if mode == Some(FileMode::Symlink) { + format!("{:x}", git_blob_sha1_hex_bytes(&content)) + } else { + self.git_blob_oid_for_path(path) + .unwrap_or_else(|| format!("{:x}", git_blob_sha1_hex_bytes(&content))) + }; + Some(BaselineFileInfo { + path: path.clone(), + content, + mode: mode_val, + oid, + }) + } else { + Some(BaselineFileInfo { + path: path.clone(), + content: vec![], + mode: FileMode::Regular, + oid: ZERO_OID.to_string(), + }) + }; + + if let Some(baseline_file_info) = baseline_file_info { + self.baseline_file_info + .insert(internal.clone(), baseline_file_info); + } + } + + // Track rename/move in current mapping if provided in an Update. + if let FileChange::Update { + move_path: Some(dest), + .. + } = change + { + let uuid_filename = match self.external_to_temp_name.get(path) { + Some(i) => i.clone(), + None => { + // This should be rare, but if we haven't mapped the source, create it with no baseline. + let i = Uuid::new_v4().to_string(); + self.baseline_file_info.insert( + i.clone(), + BaselineFileInfo { + path: path.clone(), + content: vec![], + mode: FileMode::Regular, + oid: ZERO_OID.to_string(), + }, + ); + i + } + }; + // Update current external mapping for temp file name. + self.temp_name_to_current_path + .insert(uuid_filename.clone(), dest.clone()); + // Update forward file_mapping: external current -> internal name. + self.external_to_temp_name.remove(path); + self.external_to_temp_name + .insert(dest.clone(), uuid_filename); + }; + } + } + + fn get_path_for_internal(&self, internal: &str) -> Option { + self.temp_name_to_current_path + .get(internal) + .cloned() + .or_else(|| { + self.baseline_file_info + .get(internal) + .map(|info| info.path.clone()) + }) + } + + /// Find the git worktree root for a file/directory by walking up to the first ancestor containing a `.git` entry. + /// Uses a simple cache of known roots and avoids negative-result caching for simplicity. + fn find_git_root_cached(&mut self, start: &Path) -> Option { + let dir = if start.is_dir() { + start + } else { + start.parent()? + }; + + // Fast path: if any cached root is an ancestor of this path, use it. + if let Some(root) = self + .git_root_cache + .iter() + .find(|r| dir.starts_with(r)) + .cloned() + { + return Some(root); + } + + // Walk up to find a `.git` marker. + let mut cur = dir.to_path_buf(); + loop { + let git_marker = cur.join(".git"); + if git_marker.is_dir() || git_marker.is_file() { + if !self.git_root_cache.iter().any(|r| r == &cur) { + self.git_root_cache.push(cur.clone()); + } + return Some(cur); + } + + // On Windows, avoid walking above the drive or UNC share root. + #[cfg(windows)] + { + if is_windows_drive_or_unc_root(&cur) { + return None; + } + } + + if let Some(parent) = cur.parent() { + cur = parent.to_path_buf(); + } else { + return None; + } + } + } + + /// Return a display string for `path` relative to its git root if found, else absolute. + fn relative_to_git_root_str(&mut self, path: &Path) -> String { + let s = if let Some(root) = self.find_git_root_cached(path) { + if let Ok(rel) = path.strip_prefix(&root) { + rel.display().to_string() + } else { + path.display().to_string() + } + } else { + path.display().to_string() + }; + s.replace('\\', "/") + } + + /// Ask git to compute the blob SHA-1 for the file at `path` within its repository. + /// Returns None if no repository is found or git invocation fails. + fn git_blob_oid_for_path(&mut self, path: &Path) -> Option { + let root = self.find_git_root_cached(path)?; + // Compute a path relative to the repo root for better portability across platforms. + let rel = path.strip_prefix(&root).unwrap_or(path); + let output = Command::new("git") + .arg("-C") + .arg(&root) + .arg("hash-object") + .arg("--") + .arg(rel) + .output() + .ok()?; + if !output.status.success() { + return None; + } + let s = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if s.len() == 40 { Some(s) } else { None } + } + + /// Recompute the aggregated unified diff by comparing all of the in-memory snapshots that were + /// collected before the first time they were touched by apply_patch during this turn with + /// the current repo state. + pub fn get_unified_diff(&mut self) -> Result> { + let mut aggregated = String::new(); + + // Compute diffs per tracked internal file in a stable order by external path. + let mut baseline_file_names: Vec = + self.baseline_file_info.keys().cloned().collect(); + // Sort lexicographically by full repo-relative path to match git behavior. + baseline_file_names.sort_by_key(|internal| { + self.get_path_for_internal(internal) + .map(|p| self.relative_to_git_root_str(&p)) + .unwrap_or_default() + }); + + for internal in baseline_file_names { + aggregated.push_str(self.get_file_diff(&internal).as_str()); + if !aggregated.ends_with('\n') { + aggregated.push('\n'); + } + } + + if aggregated.trim().is_empty() { + Ok(None) + } else { + Ok(Some(aggregated)) + } + } + + fn get_file_diff(&mut self, internal_file_name: &str) -> String { + let mut aggregated = String::new(); + + // Snapshot lightweight fields only. + let (baseline_external_path, baseline_mode, left_oid) = { + if let Some(info) = self.baseline_file_info.get(internal_file_name) { + (info.path.clone(), info.mode, info.oid.clone()) + } else { + (PathBuf::new(), FileMode::Regular, ZERO_OID.to_string()) + } + }; + let current_external_path = match self.get_path_for_internal(internal_file_name) { + Some(p) => p, + None => return aggregated, + }; + + let current_mode = file_mode_for_path(¤t_external_path).unwrap_or(FileMode::Regular); + let right_bytes = blob_bytes(¤t_external_path, ¤t_mode); + + // Compute displays with &mut self before borrowing any baseline content. + let left_display = self.relative_to_git_root_str(&baseline_external_path); + let right_display = self.relative_to_git_root_str(¤t_external_path); + + // Compute right oid before borrowing baseline content. + let right_oid = if let Some(b) = right_bytes.as_ref() { + if current_mode == FileMode::Symlink { + format!("{:x}", git_blob_sha1_hex_bytes(b)) + } else { + self.git_blob_oid_for_path(¤t_external_path) + .unwrap_or_else(|| format!("{:x}", git_blob_sha1_hex_bytes(b))) + } + } else { + ZERO_OID.to_string() + }; + + // Borrow baseline content only after all &mut self uses are done. + let left_present = left_oid.as_str() != ZERO_OID; + let left_bytes: Option<&[u8]> = if left_present { + self.baseline_file_info + .get(internal_file_name) + .map(|i| i.content.as_slice()) + } else { + None + }; + + // Fast path: identical bytes or both missing. + if left_bytes == right_bytes.as_deref() { + return aggregated; + } + + aggregated.push_str(&format!("diff --git a/{left_display} b/{right_display}\n")); + + let is_add = !left_present && right_bytes.is_some(); + let is_delete = left_present && right_bytes.is_none(); + + if is_add { + aggregated.push_str(&format!("new file mode {current_mode}\n")); + } else if is_delete { + aggregated.push_str(&format!("deleted file mode {baseline_mode}\n")); + } else if baseline_mode != current_mode { + aggregated.push_str(&format!("old mode {baseline_mode}\n")); + aggregated.push_str(&format!("new mode {current_mode}\n")); + } + + let left_text = left_bytes.and_then(|b| std::str::from_utf8(b).ok()); + let right_text = right_bytes + .as_deref() + .and_then(|b| std::str::from_utf8(b).ok()); + + let can_text_diff = matches!( + (left_text, right_text, is_add, is_delete), + (Some(_), Some(_), _, _) | (_, Some(_), true, _) | (Some(_), _, _, true) + ); + + if can_text_diff { + let l = left_text.unwrap_or(""); + let r = right_text.unwrap_or(""); + + aggregated.push_str(&format!("index {left_oid}..{right_oid}\n")); + + let old_header = if left_present { + format!("a/{left_display}") + } else { + DEV_NULL.to_string() + }; + let new_header = if right_bytes.is_some() { + format!("b/{right_display}") + } else { + DEV_NULL.to_string() + }; + + let diff = similar::TextDiff::from_lines(l, r); + let unified = diff + .unified_diff() + .context_radius(3) + .header(&old_header, &new_header) + .to_string(); + + aggregated.push_str(&unified); + } else { + aggregated.push_str(&format!("index {left_oid}..{right_oid}\n")); + let old_header = if left_present { + format!("a/{left_display}") + } else { + DEV_NULL.to_string() + }; + let new_header = if right_bytes.is_some() { + format!("b/{right_display}") + } else { + DEV_NULL.to_string() + }; + aggregated.push_str(&format!("--- {old_header}\n")); + aggregated.push_str(&format!("+++ {new_header}\n")); + aggregated.push_str("Binary files differ\n"); + } + aggregated + } +} + +/// Compute the Git SHA-1 blob object ID for the given content (bytes). +fn git_blob_sha1_hex_bytes(data: &[u8]) -> Output { + // Git blob hash is sha1 of: "blob \0" + let header = format!("blob {}\0", data.len()); + use sha1::Digest; + let mut hasher = sha1::Sha1::new(); + hasher.update(header.as_bytes()); + hasher.update(data); + hasher.finalize() +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum FileMode { + Regular, + #[cfg(unix)] + Executable, + Symlink, +} + +impl FileMode { + fn as_str(&self) -> &'static str { + match self { + FileMode::Regular => "100644", + #[cfg(unix)] + FileMode::Executable => "100755", + FileMode::Symlink => "120000", + } + } +} + +impl std::fmt::Display for FileMode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +#[cfg(unix)] +fn file_mode_for_path(path: &Path) -> Option { + use std::os::unix::fs::PermissionsExt; + let meta = fs::symlink_metadata(path).ok()?; + let ft = meta.file_type(); + if ft.is_symlink() { + return Some(FileMode::Symlink); + } + let mode = meta.permissions().mode(); + let is_exec = (mode & 0o111) != 0; + Some(if is_exec { + FileMode::Executable + } else { + FileMode::Regular + }) +} + +#[cfg(not(unix))] +fn file_mode_for_path(_path: &Path) -> Option { + // Default to non-executable on non-unix. + Some(FileMode::Regular) +} + +fn blob_bytes(path: &Path, mode: &FileMode) -> Option> { + if path.exists() { + let contents = if *mode == FileMode::Symlink { + symlink_blob_bytes(path) + .ok_or_else(|| anyhow!("failed to read symlink target for {}", path.display())) + } else { + fs::read(path) + .with_context(|| format!("failed to read current file for diff {}", path.display())) + }; + contents.ok() + } else { + None + } +} + +#[cfg(unix)] +fn symlink_blob_bytes(path: &Path) -> Option> { + use std::os::unix::ffi::OsStrExt; + let target = std::fs::read_link(path).ok()?; + Some(target.as_os_str().as_bytes().to_vec()) +} + +#[cfg(not(unix))] +fn symlink_blob_bytes(_path: &Path) -> Option> { + None +} + +#[cfg(windows)] +fn is_windows_drive_or_unc_root(p: &std::path::Path) -> bool { + use std::path::Component; + let mut comps = p.components(); + matches!( + (comps.next(), comps.next(), comps.next()), + (Some(Component::Prefix(_)), Some(Component::RootDir), None) + ) +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used)] + use super::*; + use pretty_assertions::assert_eq; + use tempfile::tempdir; + + /// Compute the Git SHA-1 blob object ID for the given content (string). + /// This delegates to the bytes version to avoid UTF-8 lossy conversions here. + fn git_blob_sha1_hex(data: &str) -> String { + format!("{:x}", git_blob_sha1_hex_bytes(data.as_bytes())) + } + + fn normalize_diff_for_test(input: &str, root: &Path) -> String { + let root_str = root.display().to_string().replace('\\', "/"); + let replaced = input.replace(&root_str, ""); + // Split into blocks on lines starting with "diff --git ", sort blocks for determinism, and rejoin + let mut blocks: Vec = Vec::new(); + let mut current = String::new(); + for line in replaced.lines() { + if line.starts_with("diff --git ") && !current.is_empty() { + blocks.push(current); + current = String::new(); + } + if !current.is_empty() { + current.push('\n'); + } + current.push_str(line); + } + if !current.is_empty() { + blocks.push(current); + } + blocks.sort(); + let mut out = blocks.join("\n"); + if !out.ends_with('\n') { + out.push('\n'); + } + out + } + + #[test] + fn accumulates_add_and_update() { + let mut acc = TurnDiffTracker::new(); + + let dir = tempdir().unwrap(); + let file = dir.path().join("a.txt"); + + // First patch: add file (baseline should be /dev/null). + let add_changes = HashMap::from([( + file.clone(), + FileChange::Add { + content: "foo\n".to_string(), + }, + )]); + acc.on_patch_begin(&add_changes); + + // Simulate apply: create the file on disk. + fs::write(&file, "foo\n").unwrap(); + let first = acc.get_unified_diff().unwrap().unwrap(); + let first = normalize_diff_for_test(&first, dir.path()); + let expected_first = { + let mode = file_mode_for_path(&file).unwrap_or(FileMode::Regular); + let right_oid = git_blob_sha1_hex("foo\n"); + format!( + r#"diff --git a//a.txt b//a.txt +new file mode {mode} +index {ZERO_OID}..{right_oid} +--- {DEV_NULL} ++++ b//a.txt +@@ -0,0 +1 @@ ++foo +"#, + ) + }; + assert_eq!(first, expected_first); + + // Second patch: update the file on disk. + let update_changes = HashMap::from([( + file.clone(), + FileChange::Update { + unified_diff: "".to_owned(), + move_path: None, + }, + )]); + acc.on_patch_begin(&update_changes); + + // Simulate apply: append a new line. + fs::write(&file, "foo\nbar\n").unwrap(); + let combined = acc.get_unified_diff().unwrap().unwrap(); + let combined = normalize_diff_for_test(&combined, dir.path()); + let expected_combined = { + let mode = file_mode_for_path(&file).unwrap_or(FileMode::Regular); + let right_oid = git_blob_sha1_hex("foo\nbar\n"); + format!( + r#"diff --git a//a.txt b//a.txt +new file mode {mode} +index {ZERO_OID}..{right_oid} +--- {DEV_NULL} ++++ b//a.txt +@@ -0,0 +1,2 @@ ++foo ++bar +"#, + ) + }; + assert_eq!(combined, expected_combined); + } + + #[test] + fn accumulates_delete() { + let dir = tempdir().unwrap(); + let file = dir.path().join("b.txt"); + fs::write(&file, "x\n").unwrap(); + + let mut acc = TurnDiffTracker::new(); + let del_changes = HashMap::from([(file.clone(), FileChange::Delete)]); + acc.on_patch_begin(&del_changes); + + // Simulate apply: delete the file from disk. + let baseline_mode = file_mode_for_path(&file).unwrap_or(FileMode::Regular); + fs::remove_file(&file).unwrap(); + let diff = acc.get_unified_diff().unwrap().unwrap(); + let diff = normalize_diff_for_test(&diff, dir.path()); + let expected = { + let left_oid = git_blob_sha1_hex("x\n"); + format!( + r#"diff --git a//b.txt b//b.txt +deleted file mode {baseline_mode} +index {left_oid}..{ZERO_OID} +--- a//b.txt ++++ {DEV_NULL} +@@ -1 +0,0 @@ +-x +"#, + ) + }; + assert_eq!(diff, expected); + } + + #[test] + fn accumulates_move_and_update() { + let dir = tempdir().unwrap(); + let src = dir.path().join("src.txt"); + let dest = dir.path().join("dst.txt"); + fs::write(&src, "line\n").unwrap(); + + let mut acc = TurnDiffTracker::new(); + let mv_changes = HashMap::from([( + src.clone(), + FileChange::Update { + unified_diff: "".to_owned(), + move_path: Some(dest.clone()), + }, + )]); + acc.on_patch_begin(&mv_changes); + + // Simulate apply: move and update content. + fs::rename(&src, &dest).unwrap(); + fs::write(&dest, "line2\n").unwrap(); + + let out = acc.get_unified_diff().unwrap().unwrap(); + let out = normalize_diff_for_test(&out, dir.path()); + let expected = { + let left_oid = git_blob_sha1_hex("line\n"); + let right_oid = git_blob_sha1_hex("line2\n"); + format!( + r#"diff --git a//src.txt b//dst.txt +index {left_oid}..{right_oid} +--- a//src.txt ++++ b//dst.txt +@@ -1 +1 @@ +-line ++line2 +"# + ) + }; + assert_eq!(out, expected); + } + + #[test] + fn move_without_1change_yields_no_diff() { + let dir = tempdir().unwrap(); + let src = dir.path().join("moved.txt"); + let dest = dir.path().join("renamed.txt"); + fs::write(&src, "same\n").unwrap(); + + let mut acc = TurnDiffTracker::new(); + let mv_changes = HashMap::from([( + src.clone(), + FileChange::Update { + unified_diff: "".to_owned(), + move_path: Some(dest.clone()), + }, + )]); + acc.on_patch_begin(&mv_changes); + + // Simulate apply: move only, no content change. + fs::rename(&src, &dest).unwrap(); + + let diff = acc.get_unified_diff().unwrap(); + assert_eq!(diff, None); + } + + #[test] + fn move_declared_but_file_only_appears_at_dest_is_add() { + let dir = tempdir().unwrap(); + let src = dir.path().join("src.txt"); + let dest = dir.path().join("dest.txt"); + let mut acc = TurnDiffTracker::new(); + let mv = HashMap::from([( + src.clone(), + FileChange::Update { + unified_diff: "".into(), + move_path: Some(dest.clone()), + }, + )]); + acc.on_patch_begin(&mv); + // No file existed initially; create only dest + fs::write(&dest, "hello\n").unwrap(); + let diff = acc.get_unified_diff().unwrap().unwrap(); + let diff = normalize_diff_for_test(&diff, dir.path()); + let expected = { + let mode = file_mode_for_path(&dest).unwrap_or(FileMode::Regular); + let right_oid = git_blob_sha1_hex("hello\n"); + format!( + r#"diff --git a//src.txt b//dest.txt +new file mode {mode} +index {ZERO_OID}..{right_oid} +--- {DEV_NULL} ++++ b//dest.txt +@@ -0,0 +1 @@ ++hello +"#, + ) + }; + assert_eq!(diff, expected); + } + + #[test] + fn update_persists_across_new_baseline_for_new_file() { + let dir = tempdir().unwrap(); + let a = dir.path().join("a.txt"); + let b = dir.path().join("b.txt"); + fs::write(&a, "foo\n").unwrap(); + fs::write(&b, "z\n").unwrap(); + + let mut acc = TurnDiffTracker::new(); + + // First: update existing a.txt (baseline snapshot is created for a). + let update_a = HashMap::from([( + a.clone(), + FileChange::Update { + unified_diff: "".to_owned(), + move_path: None, + }, + )]); + acc.on_patch_begin(&update_a); + // Simulate apply: modify a.txt on disk. + fs::write(&a, "foo\nbar\n").unwrap(); + let first = acc.get_unified_diff().unwrap().unwrap(); + let first = normalize_diff_for_test(&first, dir.path()); + let expected_first = { + let left_oid = git_blob_sha1_hex("foo\n"); + let right_oid = git_blob_sha1_hex("foo\nbar\n"); + format!( + r#"diff --git a//a.txt b//a.txt +index {left_oid}..{right_oid} +--- a//a.txt ++++ b//a.txt +@@ -1 +1,2 @@ + foo ++bar +"# + ) + }; + assert_eq!(first, expected_first); + + // Next: introduce a brand-new path b.txt into baseline snapshots via a delete change. + let del_b = HashMap::from([(b.clone(), FileChange::Delete)]); + acc.on_patch_begin(&del_b); + // Simulate apply: delete b.txt. + let baseline_mode = file_mode_for_path(&b).unwrap_or(FileMode::Regular); + fs::remove_file(&b).unwrap(); + + let combined = acc.get_unified_diff().unwrap().unwrap(); + let combined = normalize_diff_for_test(&combined, dir.path()); + let expected = { + let left_oid_a = git_blob_sha1_hex("foo\n"); + let right_oid_a = git_blob_sha1_hex("foo\nbar\n"); + let left_oid_b = git_blob_sha1_hex("z\n"); + format!( + r#"diff --git a//a.txt b//a.txt +index {left_oid_a}..{right_oid_a} +--- a//a.txt ++++ b//a.txt +@@ -1 +1,2 @@ + foo ++bar +diff --git a//b.txt b//b.txt +deleted file mode {baseline_mode} +index {left_oid_b}..{ZERO_OID} +--- a//b.txt ++++ {DEV_NULL} +@@ -1 +0,0 @@ +-z +"#, + ) + }; + assert_eq!(combined, expected); + } + + #[test] + fn binary_files_differ_update() { + let dir = tempdir().unwrap(); + let file = dir.path().join("bin.dat"); + + // Initial non-UTF8 bytes + let left_bytes: Vec = vec![0xff, 0xfe, 0xfd, 0x00]; + // Updated non-UTF8 bytes + let right_bytes: Vec = vec![0x01, 0x02, 0x03, 0x00]; + + fs::write(&file, &left_bytes).unwrap(); + + let mut acc = TurnDiffTracker::new(); + let update_changes = HashMap::from([( + file.clone(), + FileChange::Update { + unified_diff: "".to_owned(), + move_path: None, + }, + )]); + acc.on_patch_begin(&update_changes); + + // Apply update on disk + fs::write(&file, &right_bytes).unwrap(); + + let diff = acc.get_unified_diff().unwrap().unwrap(); + let diff = normalize_diff_for_test(&diff, dir.path()); + let expected = { + let left_oid = format!("{:x}", git_blob_sha1_hex_bytes(&left_bytes)); + let right_oid = format!("{:x}", git_blob_sha1_hex_bytes(&right_bytes)); + format!( + r#"diff --git a//bin.dat b//bin.dat +index {left_oid}..{right_oid} +--- a//bin.dat ++++ b//bin.dat +Binary files differ +"# + ) + }; + assert_eq!(diff, expected); + } + + #[test] + fn filenames_with_spaces_add_and_update() { + let mut acc = TurnDiffTracker::new(); + + let dir = tempdir().unwrap(); + let file = dir.path().join("name with spaces.txt"); + + // First patch: add file (baseline should be /dev/null). + let add_changes = HashMap::from([( + file.clone(), + FileChange::Add { + content: "foo\n".to_string(), + }, + )]); + acc.on_patch_begin(&add_changes); + + // Simulate apply: create the file on disk. + fs::write(&file, "foo\n").unwrap(); + let first = acc.get_unified_diff().unwrap().unwrap(); + let first = normalize_diff_for_test(&first, dir.path()); + let expected_first = { + let mode = file_mode_for_path(&file).unwrap_or(FileMode::Regular); + let right_oid = git_blob_sha1_hex("foo\n"); + format!( + r#"diff --git a//name with spaces.txt b//name with spaces.txt +new file mode {mode} +index {ZERO_OID}..{right_oid} +--- {DEV_NULL} ++++ b//name with spaces.txt +@@ -0,0 +1 @@ ++foo +"#, + ) + }; + assert_eq!(first, expected_first); + + // Second patch: update the file on disk. + let update_changes = HashMap::from([( + file.clone(), + FileChange::Update { + unified_diff: "".to_owned(), + move_path: None, + }, + )]); + acc.on_patch_begin(&update_changes); + + // Simulate apply: append a new line with a space. + fs::write(&file, "foo\nbar baz\n").unwrap(); + let combined = acc.get_unified_diff().unwrap().unwrap(); + let combined = normalize_diff_for_test(&combined, dir.path()); + let expected_combined = { + let mode = file_mode_for_path(&file).unwrap_or(FileMode::Regular); + let right_oid = git_blob_sha1_hex("foo\nbar baz\n"); + format!( + r#"diff --git a//name with spaces.txt b//name with spaces.txt +new file mode {mode} +index {ZERO_OID}..{right_oid} +--- {DEV_NULL} ++++ b//name with spaces.txt +@@ -0,0 +1,2 @@ ++foo ++bar baz +"#, + ) + }; + assert_eq!(combined, expected_combined); + } +} diff --git a/codex-rs/exec/src/event_processor_with_human_output.rs b/codex-rs/exec/src/event_processor_with_human_output.rs index 72e2f9298f..c290d9336b 100644 --- a/codex-rs/exec/src/event_processor_with_human_output.rs +++ b/codex-rs/exec/src/event_processor_with_human_output.rs @@ -20,6 +20,7 @@ use codex_core::protocol::PatchApplyEndEvent; use codex_core::protocol::SessionConfiguredEvent; use codex_core::protocol::TaskCompleteEvent; use codex_core::protocol::TokenUsage; +use codex_core::protocol::TurnDiffEvent; use owo_colors::OwoColorize; use owo_colors::Style; use shlex::try_join; @@ -399,6 +400,7 @@ impl EventProcessor for EventProcessorWithHumanOutput { stdout, stderr, success, + .. }) => { let patch_begin = self.call_id_to_patch.remove(&call_id); @@ -428,6 +430,10 @@ impl EventProcessor for EventProcessorWithHumanOutput { println!("{}", line.style(self.dimmed)); } } + EventMsg::TurnDiff(TurnDiffEvent { unified_diff }) => { + ts_println!(self, "{}", "turn diff:".style(self.magenta)); + println!("{unified_diff}"); + } EventMsg::ExecApprovalRequest(_) => { // Should we exit? } diff --git a/codex-rs/mcp-server/src/codex_tool_runner.rs b/codex-rs/mcp-server/src/codex_tool_runner.rs index d489ffe076..205dfa4631 100644 --- a/codex-rs/mcp-server/src/codex_tool_runner.rs +++ b/codex-rs/mcp-server/src/codex_tool_runner.rs @@ -263,6 +263,7 @@ async fn run_codex_tool_session_inner( | EventMsg::BackgroundEvent(_) | EventMsg::PatchApplyBegin(_) | EventMsg::PatchApplyEnd(_) + | EventMsg::TurnDiff(_) | EventMsg::GetHistoryEntryResponse(_) | EventMsg::PlanUpdate(_) | EventMsg::ShutdownComplete => { diff --git a/codex-rs/mcp-server/src/conversation_loop.rs b/codex-rs/mcp-server/src/conversation_loop.rs index 534275181a..1db39a2306 100644 --- a/codex-rs/mcp-server/src/conversation_loop.rs +++ b/codex-rs/mcp-server/src/conversation_loop.rs @@ -97,6 +97,7 @@ pub async fn run_conversation_loop( | EventMsg::McpToolCallEnd(_) | EventMsg::ExecCommandBegin(_) | EventMsg::ExecCommandEnd(_) + | EventMsg::TurnDiff(_) | EventMsg::BackgroundEvent(_) | EventMsg::ExecCommandOutputDelta(_) | EventMsg::PatchApplyBegin(_) From dc15a5cf0b50eaa7a75dca754d618090d41d51e3 Mon Sep 17 00:00:00 2001 From: ae Date: Mon, 4 Aug 2025 09:34:46 -0700 Subject: [PATCH 052/100] feat: accept custom instructions in profiles (#1803) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allows users to set their experimental_instructions_file in configs. For example the below enables experimental instructions when running `codex -p foo`. ``` [profiles.foo] experimental_instructions_file = "/Users/foo/.codex/prompt.md" ``` # Testing - ✅ Running against a profile with experimental_instructions_file works. - ✅ Running against a profile without experimental_instructions_file works. - ✅ Running against no profile with experimental_instructions_file works. - ✅ Running against no profile without experimental_instructions_file works. --- codex-rs/core/src/config.rs | 10 ++++++---- codex-rs/core/src/config_profile.rs | 2 ++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/codex-rs/core/src/config.rs b/codex-rs/core/src/config.rs index 87629d821a..b43dc56ba0 100644 --- a/codex-rs/core/src/config.rs +++ b/codex-rs/core/src/config.rs @@ -480,10 +480,12 @@ impl Config { // Load base instructions override from a file if specified. If the // path is relative, resolve it against the effective cwd so the // behaviour matches other path-like config values. - let file_base_instructions = Self::get_base_instructions( - cfg.experimental_instructions_file.as_ref(), - &resolved_cwd, - )?; + let experimental_instructions_path = config_profile + .experimental_instructions_file + .as_ref() + .or(cfg.experimental_instructions_file.as_ref()); + let file_base_instructions = + Self::get_base_instructions(experimental_instructions_path, &resolved_cwd)?; let base_instructions = base_instructions.or(file_base_instructions); let config = Self { diff --git a/codex-rs/core/src/config_profile.rs b/codex-rs/core/src/config_profile.rs index 176a9b1500..d945d1df7a 100644 --- a/codex-rs/core/src/config_profile.rs +++ b/codex-rs/core/src/config_profile.rs @@ -1,4 +1,5 @@ use serde::Deserialize; +use std::path::PathBuf; use crate::config_types::ReasoningEffort; use crate::config_types::ReasoningSummary; @@ -17,4 +18,5 @@ pub struct ConfigProfile { pub model_reasoning_effort: Option, pub model_reasoning_summary: Option, pub chatgpt_base_url: Option, + pub experimental_instructions_file: Option, } From a6139aa0035d19d794a3669d6196f9f32a8c8352 Mon Sep 17 00:00:00 2001 From: easong-openai Date: Mon, 4 Aug 2025 10:42:39 -0700 Subject: [PATCH 053/100] Update prompt.md (#1819) The existing prompt is really bad. As a low-hanging fruit, let's correct the apply_patch instructions - this helps smaller models successfully apply patches. --- codex-rs/core/prompt.md | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/codex-rs/core/prompt.md b/codex-rs/core/prompt.md index 0a4578270a..4e55003b9f 100644 --- a/codex-rs/core/prompt.md +++ b/codex-rs/core/prompt.md @@ -10,7 +10,7 @@ You MUST adhere to the following criteria when executing the task: - Showing user code and tool call details is allowed. - User instructions may overwrite the _CODING GUIDELINES_ section in this developer message. - Do not use \`ls -R\`, \`find\`, or \`grep\` - these are slow in large repos. Use \`rg\` and \`rg --files\`. -- Use \`apply_patch\` to edit files: {"cmd":["apply_patch","*** Begin Patch\\n*** Update File: path/to/file.py\\n@@ def example():\\n- pass\\n+ return 123\\n*** End Patch"]} +- Use \`apply_patch\` to edit files: {"command":["apply_patch","*** Begin Patch\\n*** Update File: path/to/file.py\\n@@ def example():\\n- pass\\n+ return 123\\n*** End Patch"]} - If completing the user's task requires writing or modifying files: - Your code and final answer should follow these _CODING GUIDELINES_: - Fix the problem at the root cause rather than applying surface-level patches, when possible. @@ -40,16 +40,16 @@ You MUST adhere to the following criteria when executing the task: Your patch language is a stripped‑down, file‑oriented diff format designed to be easy to parse and safe to apply. You can think of it as a high‑level envelope: -**_ Begin Patch +*** Begin Patch [ one or more file sections ] -_** End Patch +*** End Patch Within that envelope, you get a sequence of file operations. You MUST include a header to specify the action you are taking. Each operation starts with one of three headers: -**_ Add File: - create a new file. Every following line is a + line (the initial contents). -_** Delete File: - remove an existing file. Nothing follows. +*** Add File: - create a new file. Every following line is a + line (the initial contents). +*** Delete File: - remove an existing file. Nothing follows. \*\*\* Update File: - patch an existing file in place (optionally with a rename). May be immediately followed by \*\*\* Move to: if you want to rename the file. @@ -63,28 +63,28 @@ Within a hunk each line starts with: At the end of a truncated hunk you can emit \*\*\* End of File. Patch := Begin { FileOp } End -Begin := "**_ Begin Patch" NEWLINE -End := "_** End Patch" NEWLINE +Begin := "*** Begin Patch" NEWLINE +End := "*** End Patch" NEWLINE FileOp := AddFile | DeleteFile | UpdateFile -AddFile := "**_ Add File: " path NEWLINE { "+" line NEWLINE } -DeleteFile := "_** Delete File: " path NEWLINE -UpdateFile := "**_ Update File: " path NEWLINE [ MoveTo ] { Hunk } -MoveTo := "_** Move to: " newPath NEWLINE +AddFile := "*** Add File: " path NEWLINE { "+" line NEWLINE } +DeleteFile := "*** Delete File: " path NEWLINE +UpdateFile := "*** Update File: " path NEWLINE [ MoveTo ] { Hunk } +MoveTo := "*** Move to: " newPath NEWLINE Hunk := "@@" [ header ] NEWLINE { HunkLine } [ "*** End of File" NEWLINE ] HunkLine := (" " | "-" | "+") text NEWLINE A full patch can combine several operations: -**_ Begin Patch -_** Add File: hello.txt +*** Begin Patch +*** Add File: hello.txt +Hello world -**_ Update File: src/app.py -_** Move to: src/main.py +*** Update File: src/app.py +*** Move to: src/main.py @@ def greet(): -print("Hi") +print("Hello, world!") -**_ Delete File: obsolete.txt -_** End Patch +*** Delete File: obsolete.txt +*** End Patch It is important to remember: @@ -101,7 +101,7 @@ Plan updates A tool named `update_plan` is available. Use it to keep an up‑to‑date, step‑by‑step plan for the task so you can follow your progress. When making your plans, keep in mind that you are a deployed coding agent - `update_plan` calls should not involve doing anything that you aren't capable of doing. For example, `update_plan` calls should NEVER contain tasks to merge your own pull requests. Only stop to ask the user if you genuinely need their feedback on a change. -- At the start of the task, call `update_plan` with an initial plan: a short list of 1‑sentence steps with a `status` for each step (`pending`, `in_progress`, or `completed`). There should always be exactly one `in_progress` step until everything is done. +- At the start of any nontrivial task, call `update_plan` with an initial plan: a short list of 1‑sentence steps with a `status` for each step (`pending`, `in_progress`, or `completed`). There should always be exactly one `in_progress` step until everything is done. - Whenever you finish a step, call `update_plan` again, marking the finished step as `completed` and the next step as `in_progress`. - If your plan needs to change, call `update_plan` with the revised steps and include an `explanation` describing the change. - When all steps are complete, make a final `update_plan` call with all steps marked `completed`. From 64cfbbd3c8609fb1c1a1bfbc1bf86148e12e6cc4 Mon Sep 17 00:00:00 2001 From: Jeremy Rose <172423086+nornagon-openai@users.noreply.github.com> Date: Mon, 4 Aug 2025 11:25:01 -0700 Subject: [PATCH 054/100] support more keys in textarea (#1820) Added: * C-m for newline (not sure if this is actually treated differently to Enter, but tui-textarea handles it and it doesn't hurt) * C-d to delete one char forwards (same as Del) * A-bksp to delete backwards one word * A-arrows to navigate by word --- codex-rs/tui/src/bottom_pane/textarea.rs | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/codex-rs/tui/src/bottom_pane/textarea.rs b/codex-rs/tui/src/bottom_pane/textarea.rs index e150135b75..cb30c2ac7a 100644 --- a/codex-rs/tui/src/bottom_pane/textarea.rs +++ b/codex-rs/tui/src/bottom_pane/textarea.rs @@ -210,7 +210,7 @@ impl TextArea { .. } => self.insert_str(&c.to_string()), KeyEvent { - code: KeyCode::Char('j'), + code: KeyCode::Char('j' | 'm'), modifiers: KeyModifiers::CONTROL, .. } @@ -220,11 +220,22 @@ impl TextArea { } => self.insert_str("\n"), KeyEvent { code: KeyCode::Backspace, + modifiers: KeyModifiers::ALT, + .. + } => self.delete_backward_word(), + KeyEvent { + code: KeyCode::Backspace, + modifiers: KeyModifiers::NONE, .. } => self.delete_backward(1), KeyEvent { code: KeyCode::Delete, .. + } + | KeyEvent { + code: KeyCode::Char('d'), + modifiers: KeyModifiers::CONTROL, + .. } => self.delete_forward(1), KeyEvent { @@ -303,14 +314,14 @@ impl TextArea { } KeyEvent { code: KeyCode::Left, - modifiers: KeyModifiers::CONTROL, + modifiers: KeyModifiers::CONTROL | KeyModifiers::ALT, .. } => { self.set_cursor(self.beginning_of_previous_word()); } KeyEvent { code: KeyCode::Right, - modifiers: KeyModifiers::CONTROL, + modifiers: KeyModifiers::CONTROL | KeyModifiers::ALT, .. } => { self.set_cursor(self.end_of_next_word()); From 2899817c94098caf96009c1d797597df1c298e3f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Aug 2025 14:24:19 -0700 Subject: [PATCH 055/100] chore(deps): bump toml from 0.9.2 to 0.9.4 in /codex-rs (#1815) Bumps [toml](https://github.com/toml-rs/toml) from 0.9.2 to 0.9.4.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=toml&package-manager=cargo&previous-version=0.9.2&new-version=0.9.4)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- codex-rs/Cargo.lock | 10 +++++----- codex-rs/core/Cargo.toml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index eb4eccd897..9d8a027c53 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -661,7 +661,7 @@ dependencies = [ "clap", "codex-core", "serde", - "toml 0.9.2", + "toml 0.9.4", ] [[package]] @@ -707,7 +707,7 @@ dependencies = [ "tokio", "tokio-test", "tokio-util", - "toml 0.9.2", + "toml 0.9.4", "tracing", "tree-sitter", "tree-sitter-bash", @@ -831,7 +831,7 @@ dependencies = [ "tempfile", "tokio", "tokio-test", - "toml 0.9.2", + "toml 0.9.4", "tracing", "tracing-subscriber", "uuid", @@ -4773,9 +4773,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.9.2" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed0aee96c12fa71097902e0bb061a5e1ebd766a6636bb605ba401c45c1650eac" +checksum = "41ae868b5a0f67631c14589f7e250c1ea2c574ee5ba21c6c8dd4b1485705a5a1" dependencies = [ "indexmap 2.10.0", "serde", diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index 466e9adf02..e9d6970ded 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -46,7 +46,7 @@ tokio = { version = "1", features = [ "signal", ] } tokio-util = "0.7.14" -toml = "0.9.2" +toml = "0.9.4" tracing = { version = "0.1.41", features = ["log"] } tree-sitter = "0.25.8" tree-sitter-bash = "0.25.0" From 6db597ec0c6833018973bbcbe97139d100913304 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Aug 2025 14:25:00 -0700 Subject: [PATCH 056/100] chore(deps-dev): bump typescript from 5.8.3 to 5.9.2 in /.github/actions/codex (#1814) [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=typescript&package-manager=bun&previous-version=5.8.3&new-version=5.9.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/actions/codex/bun.lock | 4 ++-- .github/actions/codex/package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/codex/bun.lock b/.github/actions/codex/bun.lock index 8b546a5ac6..82e12cc4b6 100644 --- a/.github/actions/codex/bun.lock +++ b/.github/actions/codex/bun.lock @@ -11,7 +11,7 @@ "@types/bun": "^1.2.19", "@types/node": "^24.1.0", "prettier": "^3.6.2", - "typescript": "^5.8.3", + "typescript": "^5.9.2", }, }, }, @@ -68,7 +68,7 @@ "tunnel": ["tunnel@0.0.6", "", {}, "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg=="], - "typescript": ["typescript@5.8.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ=="], + "typescript": ["typescript@5.9.2", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-CWBzXQrc/qOkhidw1OzBTQuYRbfyxDXJMVJ1XNwUHGROVmuaeiEm3OslpZ1RV96d7SKKjZKrSJu3+t/xlw3R9A=="], "undici": ["undici@5.29.0", "", { "dependencies": { "@fastify/busboy": "^2.0.0" } }, "sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg=="], diff --git a/.github/actions/codex/package.json b/.github/actions/codex/package.json index 21817b8a59..6c7ae9002b 100644 --- a/.github/actions/codex/package.json +++ b/.github/actions/codex/package.json @@ -16,6 +16,6 @@ "@types/bun": "^1.2.19", "@types/node": "^24.1.0", "prettier": "^3.6.2", - "typescript": "^5.8.3" + "typescript": "^5.9.2" } } From 89ab5c3f74f6efcf5ac3b5ddb7390a90aecc9df3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Aug 2025 14:26:14 -0700 Subject: [PATCH 057/100] chore(deps): bump serde_json from 1.0.141 to 1.0.142 in /codex-rs (#1817) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [serde_json](https://github.com/serde-rs/json) from 1.0.141 to 1.0.142.
Release notes

Sourced from serde_json's releases.

v1.0.142

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=serde_json&package-manager=cargo&previous-version=1.0.141&new-version=1.0.142)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- codex-rs/Cargo.lock | 4 ++-- codex-rs/execpolicy/Cargo.toml | 2 +- codex-rs/file-search/Cargo.toml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 9d8a027c53..0ad32c3cd4 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -3997,9 +3997,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.141" +version = "1.0.142" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b9eff21ebe718216c6ec64e1d9ac57087aad11efc64e32002bce4a0d4c03d3" +checksum = "030fedb782600dcbd6f02d479bf0d817ac3bb40d644745b769d6a96bc3afc5a7" dependencies = [ "indexmap 2.10.0", "itoa", diff --git a/codex-rs/execpolicy/Cargo.toml b/codex-rs/execpolicy/Cargo.toml index ad003e66c4..9693d5c41f 100644 --- a/codex-rs/execpolicy/Cargo.toml +++ b/codex-rs/execpolicy/Cargo.toml @@ -26,7 +26,7 @@ multimap = "0.10.0" path-absolutize = "3.1.1" regex-lite = "0.1" serde = { version = "1.0.194", features = ["derive"] } -serde_json = "1.0.110" +serde_json = "1.0.142" serde_with = { version = "3", features = ["macros"] } [dev-dependencies] diff --git a/codex-rs/file-search/Cargo.toml b/codex-rs/file-search/Cargo.toml index 3f70377183..bf1e8e687f 100644 --- a/codex-rs/file-search/Cargo.toml +++ b/codex-rs/file-search/Cargo.toml @@ -17,5 +17,5 @@ clap = { version = "4", features = ["derive"] } ignore = "0.4.23" nucleo-matcher = "0.3.1" serde = { version = "1", features = ["derive"] } -serde_json = "1.0.110" +serde_json = "1.0.142" tokio = { version = "1", features = ["full"] } From 7279080edd35fb005d3c02200d1b06165b57f7dc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Aug 2025 14:50:53 -0700 Subject: [PATCH 058/100] chore(deps): bump tokio from 1.46.1 to 1.47.1 in /codex-rs (#1816) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [tokio](https://github.com/tokio-rs/tokio) from 1.46.1 to 1.47.1.
Release notes

Sourced from tokio's releases.

Tokio v1.47.1

1.47.1 (August 1st, 2025)

Fixed

  • process: fix panic from spurious pidfd wakeup (#7494)
  • sync: fix broken link of Python asyncio.Event in SetOnce docs (#7485)

#7485: tokio-rs/tokio#7485 #7494: tokio-rs/tokio#7494

Tokio v1.47.0

1.47.0 (July 25th, 2025)

This release adds poll_proceed and cooperative to the coop module for cooperative scheduling, adds SetOnce to the sync module which provides similar functionality to [std::sync::OnceLock], and adds a new method sync::Notify::notified_owned() which returns an OwnedNotified without a lifetime parameter.

Added

  • coop: add cooperative and poll_proceed (#7405)
  • sync: add SetOnce (#7418)
  • sync: add sync::Notify::notified_owned() (#7465)

Changed

  • deps: upgrade windows-sys 0.52 → 0.59 (#7117)
  • deps: update to socket2 v0.6 (#7443)
  • sync: improve AtomicWaker::wake performance (#7450)

Documented

  • metrics: fix listed feature requirements for some metrics (#7449)
  • runtime: improve safety comments of Readiness<'_> (#7415)

#7405: tokio-rs/tokio#7405 #7415: tokio-rs/tokio#7415 #7418: tokio-rs/tokio#7418 #7449: tokio-rs/tokio#7449 #7450: tokio-rs/tokio#7450 #7465: tokio-rs/tokio#7465

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=tokio&package-manager=cargo&previous-version=1.46.1&new-version=1.47.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- codex-rs/Cargo.lock | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 0ad32c3cd4..4daae977b0 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -2017,7 +2017,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.0", + "socket2", "system-configuration", "tokio", "tower-service", @@ -4188,16 +4188,6 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c" -[[package]] -name = "socket2" -version = "0.5.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - [[package]] name = "socket2" version = "0.6.0" @@ -4673,9 +4663,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.46.1" +version = "1.47.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc3a2344dafbe23a245241fe8b09735b521110d30fcefbbd5feb1797ca35d17" +checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038" dependencies = [ "backtrace", "bytes", @@ -4686,9 +4676,9 @@ dependencies = [ "pin-project-lite", "signal-hook-registry", "slab", - "socket2 0.5.10", + "socket2", "tokio-macros", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] From 3f13ebce10209ab3645f51e7606892b3fd71d47e Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Mon, 4 Aug 2025 15:56:32 -0700 Subject: [PATCH 059/100] [codex] stop printing error message when --output-last-message is not specified (#1828) Previously, `codex exec` was printing `Warning: no file to write last message to` as a warning to stderr even though `--output-last-message` was not specified, which is wrong. This fixes the code and changes `handle_last_message()` so that it is only called when `last_message_path` is `Some`. --- codex-rs/exec/src/event_processor.rs | 22 +++++++------------ .../src/event_processor_with_human_output.rs | 7 +++--- .../src/event_processor_with_json_output.rs | 7 +++--- 3 files changed, 14 insertions(+), 22 deletions(-) diff --git a/codex-rs/exec/src/event_processor.rs b/codex-rs/exec/src/event_processor.rs index 741f89d7cb..0f189f3fa2 100644 --- a/codex-rs/exec/src/event_processor.rs +++ b/codex-rs/exec/src/event_processor.rs @@ -44,20 +44,14 @@ pub(crate) fn create_config_summary_entries(config: &Config) -> Vec<(&'static st entries } -pub(crate) fn handle_last_message( - last_agent_message: Option<&str>, - last_message_path: Option<&Path>, -) { - match (last_message_path, last_agent_message) { - (Some(path), Some(msg)) => write_last_message_file(msg, Some(path)), - (Some(path), None) => { - write_last_message_file("", Some(path)); - eprintln!( - "Warning: no last agent message; wrote empty content to {}", - path.display() - ); - } - (None, _) => eprintln!("Warning: no file to write last message to."), +pub(crate) fn handle_last_message(last_agent_message: Option<&str>, output_file: &Path) { + let message = last_agent_message.unwrap_or_default(); + write_last_message_file(message, Some(output_file)); + if last_agent_message.is_none() { + eprintln!( + "Warning: no last agent message; wrote empty content to {}", + output_file.display() + ); } } diff --git a/codex-rs/exec/src/event_processor_with_human_output.rs b/codex-rs/exec/src/event_processor_with_human_output.rs index c290d9336b..7703c138fc 100644 --- a/codex-rs/exec/src/event_processor_with_human_output.rs +++ b/codex-rs/exec/src/event_processor_with_human_output.rs @@ -170,10 +170,9 @@ impl EventProcessor for EventProcessorWithHumanOutput { // Ignore. } EventMsg::TaskComplete(TaskCompleteEvent { last_agent_message }) => { - handle_last_message( - last_agent_message.as_deref(), - self.last_message_path.as_deref(), - ); + if let Some(output_file) = self.last_message_path.as_deref() { + handle_last_message(last_agent_message.as_deref(), output_file); + } return CodexStatus::InitiateShutdown; } EventMsg::TokenCount(TokenUsage { total_tokens, .. }) => { diff --git a/codex-rs/exec/src/event_processor_with_json_output.rs b/codex-rs/exec/src/event_processor_with_json_output.rs index e7a658b76f..1d153add6e 100644 --- a/codex-rs/exec/src/event_processor_with_json_output.rs +++ b/codex-rs/exec/src/event_processor_with_json_output.rs @@ -46,10 +46,9 @@ impl EventProcessor for EventProcessorWithJsonOutput { CodexStatus::Running } EventMsg::TaskComplete(TaskCompleteEvent { last_agent_message }) => { - handle_last_message( - last_agent_message.as_deref(), - self.last_message_path.as_deref(), - ); + if let Some(output_file) = self.last_message_path.as_deref() { + handle_last_message(last_agent_message.as_deref(), output_file); + } CodexStatus::InitiateShutdown } EventMsg::ShutdownComplete => CodexStatus::Shutdown, From bd171e5206465593a616cd65344a77d17477d51f Mon Sep 17 00:00:00 2001 From: Ahmed Ibrahim Date: Mon, 4 Aug 2025 16:49:42 -0700 Subject: [PATCH 060/100] add raw reasoning --- codex-rs/core/src/chat_completions.rs | 163 ++++++++++++++---- codex-rs/core/src/client.rs | 10 +- codex-rs/core/src/codex.rs | 60 +++++-- codex-rs/core/src/config.rs | 27 ++- codex-rs/core/src/models.rs | 8 + codex-rs/core/src/protocol.rs | 8 + .../src/event_processor_with_human_output.rs | 9 + codex-rs/mcp-server/src/codex_tool_runner.rs | 3 +- codex-rs/mcp-server/src/conversation_loop.rs | 3 +- 9 files changed, 236 insertions(+), 55 deletions(-) diff --git a/codex-rs/core/src/chat_completions.rs b/codex-rs/core/src/chat_completions.rs index 5ede774b1c..d1b8338987 100644 --- a/codex-rs/core/src/chat_completions.rs +++ b/codex-rs/core/src/chat_completions.rs @@ -207,6 +207,7 @@ async fn process_chat_sse( } let mut fn_call_state = FunctionCallState::default(); + let mut assistant_text = String::new(); loop { let sse = match timeout(idle_timeout, stream.next()).await { @@ -254,21 +255,42 @@ async fn process_chat_sse( let choice_opt = chunk.get("choices").and_then(|c| c.get(0)); if let Some(choice) = choice_opt { - // Handle assistant content tokens. + // Handle assistant content tokens as streaming deltas. if let Some(content) = choice .get("delta") .and_then(|d| d.get("content")) .and_then(|c| c.as_str()) { - let item = ResponseItem::Message { - role: "assistant".to_string(), - content: vec![ContentItem::OutputText { - text: content.to_string(), - }], - id: None, - }; + if !content.is_empty() { + assistant_text.push_str(content); + let _ = tx_event + .send(Ok(ResponseEvent::OutputTextDelta(content.to_string()))) + .await; + } + } - let _ = tx_event.send(Ok(ResponseEvent::OutputItemDone(item))).await; + // Forward any reasoning/thinking deltas if present. + if let Some(reasoning) = choice + .get("delta") + .and_then(|d| d.get("reasoning")) + .and_then(|c| c.as_str()) + { + let _ = tx_event + .send(Ok(ResponseEvent::ReasoningSummaryDelta( + reasoning.to_string(), + ))) + .await; + } + if let Some(reasoning_content) = choice + .get("delta") + .and_then(|d| d.get("reasoning_content")) + .and_then(|c| c.as_str()) + { + let _ = tx_event + .send(Ok(ResponseEvent::ReasoningSummaryDelta( + reasoning_content.to_string(), + ))) + .await; } // Handle streaming function / tool calls. @@ -317,7 +339,18 @@ async fn process_chat_sse( let _ = tx_event.send(Ok(ResponseEvent::OutputItemDone(item))).await; } "stop" => { - // Regular turn without tool-call. + // Regular turn without tool-call. Emit the final assistant message + // as a single OutputItemDone so non-delta consumers see the result. + if !assistant_text.is_empty() { + let item = ResponseItem::Message { + role: "assistant".to_string(), + content: vec![ContentItem::OutputText { + text: std::mem::take(&mut assistant_text), + }], + id: None, + }; + let _ = tx_event.send(Ok(ResponseEvent::OutputItemDone(item))).await; + } } _ => {} } @@ -358,7 +391,10 @@ async fn process_chat_sse( pub(crate) struct AggregatedChatStream { inner: S, cumulative: String, - pending_completed: Option, + cumulative_reasoning: String, + pending: std::collections::VecDeque, + // When true, do not emit a cumulative assistant message at Completed. + streaming_mode: bool, } impl Stream for AggregatedChatStream @@ -370,8 +406,8 @@ where fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { let this = self.get_mut(); - // First, flush any buffered Completed event from the previous call. - if let Some(ev) = this.pending_completed.take() { + // First, flush any buffered events from the previous call. + if let Some(ev) = this.pending.pop_front() { return Poll::Ready(Some(Ok(ev))); } @@ -388,16 +424,21 @@ where let is_assistant_delta = matches!(&item, crate::models::ResponseItem::Message { role, .. } if role == "assistant"); if is_assistant_delta { - if let crate::models::ResponseItem::Message { content, .. } = &item { - if let Some(text) = content.iter().find_map(|c| match c { - crate::models::ContentItem::OutputText { text } => Some(text), - _ => None, - }) { - this.cumulative.push_str(text); + // Only use the final assistant message if we have not + // seen any deltas; otherwise, deltas already built the + // cumulative text and this would duplicate it. + if this.cumulative.is_empty() { + if let crate::models::ResponseItem::Message { content, .. } = &item { + if let Some(text) = content.iter().find_map(|c| match c { + crate::models::ContentItem::OutputText { text } => Some(text), + _ => None, + }) { + this.cumulative.push_str(text); + } } } - // Swallow partial assistant chunk; keep polling. + // Swallow assistant message here; emit on Completed. continue; } @@ -408,24 +449,48 @@ where response_id, token_usage, }))) => { + // Build any aggregated items in the correct order: Reasoning first, then Message. + let mut emitted_any = false; + + if !this.cumulative_reasoning.is_empty() { + let aggregated_reasoning = crate::models::ResponseItem::Reasoning { + id: String::new(), + summary: vec![ + crate::models::ReasoningItemReasoningSummary::SummaryText { + text: std::mem::take(&mut this.cumulative_reasoning), + }, + ], + content: None, + encrypted_content: None, + }; + this.pending + .push_back(ResponseEvent::OutputItemDone(aggregated_reasoning)); + emitted_any = true; + } + if !this.cumulative.is_empty() { - let aggregated_item = crate::models::ResponseItem::Message { + let aggregated_message = crate::models::ResponseItem::Message { id: None, role: "assistant".to_string(), content: vec![crate::models::ContentItem::OutputText { text: std::mem::take(&mut this.cumulative), }], }; + this.pending + .push_back(ResponseEvent::OutputItemDone(aggregated_message)); + emitted_any = true; + } - // Buffer Completed so it is returned *after* the aggregated message. - this.pending_completed = Some(ResponseEvent::Completed { - response_id, - token_usage, + // Always emit Completed last when anything was aggregated. + if emitted_any { + this.pending.push_back(ResponseEvent::Completed { + response_id: response_id.clone(), + token_usage: token_usage.clone(), }); - - return Poll::Ready(Some(Ok(ResponseEvent::OutputItemDone( - aggregated_item, - )))); + // Return the first pending event now. + if let Some(ev) = this.pending.pop_front() { + return Poll::Ready(Some(Ok(ev))); + } } // Nothing aggregated – forward Completed directly. @@ -439,11 +504,25 @@ where // will never appear in a Chat Completions stream. continue; } - Poll::Ready(Some(Ok(ResponseEvent::OutputTextDelta(_)))) - | Poll::Ready(Some(Ok(ResponseEvent::ReasoningSummaryDelta(_)))) => { - // Deltas are ignored here since aggregation waits for the - // final OutputItemDone. - continue; + Poll::Ready(Some(Ok(ResponseEvent::OutputTextDelta(delta)))) => { + // Always accumulate deltas so we can emit a final OutputItemDone at Completed. + this.cumulative.push_str(&delta); + if this.streaming_mode { + // In streaming mode, also forward the delta immediately. + return Poll::Ready(Some(Ok(ResponseEvent::OutputTextDelta(delta)))); + } else { + continue; + } + } + Poll::Ready(Some(Ok(ResponseEvent::ReasoningSummaryDelta(delta)))) => { + // Always accumulate reasoning deltas so we can emit a final Reasoning item at Completed. + this.cumulative_reasoning.push_str(&delta); + if this.streaming_mode { + // In streaming mode, also forward the delta immediately. + return Poll::Ready(Some(Ok(ResponseEvent::ReasoningSummaryDelta(delta)))); + } else { + continue; + } } } } @@ -475,9 +554,23 @@ pub(crate) trait AggregateStreamExt: Stream> + Size AggregatedChatStream { inner: self, cumulative: String::new(), - pending_completed: None, + cumulative_reasoning: String::new(), + pending: std::collections::VecDeque::new(), + streaming_mode: false, } } } impl AggregateStreamExt for T where T: Stream> + Sized {} + +impl AggregatedChatStream { + pub(crate) fn streaming_mode(inner: S) -> Self { + AggregatedChatStream { + inner, + cumulative: String::new(), + cumulative_reasoning: String::new(), + pending: std::collections::VecDeque::new(), + streaming_mode: true, + } + } +} diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index b9ea6b13f4..8685bc54d3 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -93,7 +93,13 @@ impl ModelClient { // Wrap it with the aggregation adapter so callers see *only* // the final assistant message per turn (matching the // behaviour of the Responses API). - let mut aggregated = response_stream.aggregate(); + let mut aggregated = if self.config.show_reasoning_content + && !self.config.hide_agent_reasoning + { + crate::chat_completions::AggregatedChatStream::streaming_mode(response_stream) + } else { + response_stream.aggregate() + }; // Bridge the aggregated stream back into a standard // `ResponseStream` by forwarding events through a channel. @@ -438,7 +444,7 @@ async fn process_sse( } } } - "response.reasoning_summary_text.delta" => { + "response.reasoning_summary_text.delta" | "response.reasoning_text.delta" => { if let Some(delta) = event.delta { let event = ResponseEvent::ReasoningSummaryDelta(delta); if tx_event.send(Ok(event)).await.is_err() { diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 568d87c4a8..18bcf6261d 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -56,6 +56,7 @@ use crate::mcp_tool_call::handle_mcp_tool_call; use crate::models::ContentItem; use crate::models::FunctionCallOutputPayload; use crate::models::LocalShellAction; +use crate::models::ReasoningItemContent; use crate::models::ReasoningItemReasoningSummary; use crate::models::ResponseInputItem; use crate::models::ResponseItem; @@ -64,6 +65,7 @@ use crate::plan_tool::handle_update_plan; use crate::project_doc::get_user_instructions; use crate::protocol::AgentMessageDeltaEvent; use crate::protocol::AgentMessageEvent; +use crate::protocol::AgentReasoningContentEvent; use crate::protocol::AgentReasoningDeltaEvent; use crate::protocol::AgentReasoningEvent; use crate::protocol::ApplyPatchApprovalRequestEvent; @@ -227,6 +229,8 @@ pub(crate) struct Session { state: Mutex, codex_linux_sandbox_exe: Option, user_shell: shell::Shell, + show_reasoning_content: bool, + hide_agent_reasoning: bool, } impl Session { @@ -822,6 +826,8 @@ async fn submission_loop( codex_linux_sandbox_exe: config.codex_linux_sandbox_exe.clone(), disable_response_storage, user_shell: default_shell, + show_reasoning_content: config.show_reasoning_content, + hide_agent_reasoning: config.hide_agent_reasoning, })); // Patch restored state into the newly created session. @@ -1132,6 +1138,7 @@ async fn run_task(sess: Arc, sub_id: String, input: Vec) { ResponseItem::Reasoning { id, summary, + content, encrypted_content, }, None, @@ -1139,6 +1146,7 @@ async fn run_task(sess: Arc, sub_id: String, input: Vec) { items_to_record_in_conversation_history.push(ResponseItem::Reasoning { id: id.clone(), summary: summary.clone(), + content: content.clone(), encrypted_content: encrypted_content.clone(), }); } @@ -1381,11 +1389,13 @@ async fn try_run_turn( sess.tx_event.send(event).await.ok(); } ResponseEvent::ReasoningSummaryDelta(delta) => { - let event = Event { - id: sub_id.to_string(), - msg: EventMsg::AgentReasoningDelta(AgentReasoningDeltaEvent { delta }), - }; - sess.tx_event.send(event).await.ok(); + if !sess.hide_agent_reasoning { + let event = Event { + id: sub_id.to_string(), + msg: EventMsg::AgentReasoningDelta(AgentReasoningDeltaEvent { delta }), + }; + sess.tx_event.send(event).await.ok(); + } } } } @@ -1493,16 +1503,36 @@ async fn handle_response_item( } None } - ResponseItem::Reasoning { summary, .. } => { - for item in summary { - let text = match item { - ReasoningItemReasoningSummary::SummaryText { text } => text, - }; - let event = Event { - id: sub_id.to_string(), - msg: EventMsg::AgentReasoning(AgentReasoningEvent { text }), - }; - sess.tx_event.send(event).await.ok(); + ResponseItem::Reasoning { + id: _, + summary, + content, + encrypted_content: _, + } => { + if !sess.hide_agent_reasoning { + for item in summary { + let text = match item { + ReasoningItemReasoningSummary::SummaryText { text } => text, + }; + let event = Event { + id: sub_id.to_string(), + msg: EventMsg::AgentReasoning(AgentReasoningEvent { text }), + }; + sess.tx_event.send(event).await.ok(); + } + } + if !sess.hide_agent_reasoning && sess.show_reasoning_content && content.is_some() { + let content = content.unwrap(); + for item in content { + let text = match item { + ReasoningItemContent::ReasoningText { text } => text, + }; + let event = Event { + id: sub_id.to_string(), + msg: EventMsg::AgentReasoningContent(AgentReasoningContentEvent { text }), + }; + sess.tx_event.send(event).await.ok(); + } } None } diff --git a/codex-rs/core/src/config.rs b/codex-rs/core/src/config.rs index b43dc56ba0..3277ca08e9 100644 --- a/codex-rs/core/src/config.rs +++ b/codex-rs/core/src/config.rs @@ -57,6 +57,10 @@ pub struct Config { /// users are only interested in the final agent responses. pub hide_agent_reasoning: bool, + /// When `true`, the raw chain-of-thought text from reasoning events will be + /// displayed in the UI in addition to the reasoning summaries. + pub show_reasoning_content: bool, + /// Disable server-side response storage (sends the full conversation /// context with every request). Currently necessary for OpenAI customers /// who have opted into Zero Data Retention (ZDR). @@ -325,6 +329,10 @@ pub struct ConfigToml { /// UI/output. Defaults to `false`. pub hide_agent_reasoning: Option, + /// When set to `true`, raw chain-of-thought text from reasoning events will + /// be shown in the UI. + pub show_reasoning_content: Option, + pub model_reasoning_effort: Option, pub model_reasoning_summary: Option, @@ -488,6 +496,19 @@ impl Config { Self::get_base_instructions(experimental_instructions_path, &resolved_cwd)?; let base_instructions = base_instructions.or(file_base_instructions); + // Resolve hide/show reasoning flags with consistent precedence: + // if hide is true, force show_reasoning_content to false. + let hide_agent_reasoning_val = cfg.hide_agent_reasoning.unwrap_or(false); + let show_reasoning_content_val = if hide_agent_reasoning_val { + false + } else { + cfg.show_reasoning_content.unwrap_or(false) + }; + + if cfg.hide_agent_reasoning == Some(true) && cfg.show_reasoning_content == Some(true) { + tracing::warn!("Ignoring show_reasoning_content because hide_agent_reasoning is true"); + } + let config = Self { model, model_context_window, @@ -517,7 +538,8 @@ impl Config { tui: cfg.tui.unwrap_or_default(), codex_linux_sandbox_exe, - hide_agent_reasoning: cfg.hide_agent_reasoning.unwrap_or(false), + hide_agent_reasoning: hide_agent_reasoning_val, + show_reasoning_content: show_reasoning_content_val, model_reasoning_effort: config_profile .model_reasoning_effort .or(cfg.model_reasoning_effort) @@ -891,6 +913,7 @@ disable_response_storage = true tui: Tui::default(), codex_linux_sandbox_exe: None, hide_agent_reasoning: false, + show_reasoning_content: false, model_reasoning_effort: ReasoningEffort::High, model_reasoning_summary: ReasoningSummary::Detailed, model_supports_reasoning_summaries: false, @@ -941,6 +964,7 @@ disable_response_storage = true tui: Tui::default(), codex_linux_sandbox_exe: None, hide_agent_reasoning: false, + show_reasoning_content: false, model_reasoning_effort: ReasoningEffort::default(), model_reasoning_summary: ReasoningSummary::default(), model_supports_reasoning_summaries: false, @@ -1006,6 +1030,7 @@ disable_response_storage = true tui: Tui::default(), codex_linux_sandbox_exe: None, hide_agent_reasoning: false, + show_reasoning_content: false, model_reasoning_effort: ReasoningEffort::default(), model_reasoning_summary: ReasoningSummary::default(), model_supports_reasoning_summaries: false, diff --git a/codex-rs/core/src/models.rs b/codex-rs/core/src/models.rs index 166404915a..98d8727e77 100644 --- a/codex-rs/core/src/models.rs +++ b/codex-rs/core/src/models.rs @@ -45,6 +45,8 @@ pub enum ResponseItem { Reasoning { id: String, summary: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + content: Option>, encrypted_content: Option, }, LocalShellCall { @@ -136,6 +138,12 @@ pub enum ReasoningItemReasoningSummary { SummaryText { text: String }, } +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ReasoningItemContent { + ReasoningText { text: String }, +} + impl From> for ResponseInputItem { fn from(items: Vec) -> Self { Self::Message { diff --git a/codex-rs/core/src/protocol.rs b/codex-rs/core/src/protocol.rs index 82591a2c78..1e0733628c 100644 --- a/codex-rs/core/src/protocol.rs +++ b/codex-rs/core/src/protocol.rs @@ -359,6 +359,9 @@ pub enum EventMsg { /// Agent reasoning delta event from agent. AgentReasoningDelta(AgentReasoningDeltaEvent), + /// Raw chain-of-thought from agent. + AgentReasoningContent(AgentReasoningContentEvent), + /// Ack the client's configure message. SessionConfigured(SessionConfiguredEvent), @@ -464,6 +467,11 @@ pub struct AgentReasoningEvent { pub text: String, } +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct AgentReasoningContentEvent { + pub text: String, +} + #[derive(Debug, Clone, Deserialize, Serialize)] pub struct AgentReasoningDeltaEvent { pub delta: String, diff --git a/codex-rs/exec/src/event_processor_with_human_output.rs b/codex-rs/exec/src/event_processor_with_human_output.rs index 7703c138fc..1f8fe3c031 100644 --- a/codex-rs/exec/src/event_processor_with_human_output.rs +++ b/codex-rs/exec/src/event_processor_with_human_output.rs @@ -4,6 +4,7 @@ use codex_core::config::Config; use codex_core::plan_tool::UpdatePlanArgs; use codex_core::protocol::AgentMessageDeltaEvent; use codex_core::protocol::AgentMessageEvent; +use codex_core::protocol::AgentReasoningContentEvent; use codex_core::protocol::AgentReasoningDeltaEvent; use codex_core::protocol::BackgroundEventEvent; use codex_core::protocol::ErrorEvent; @@ -203,6 +204,14 @@ impl EventProcessor for EventProcessorWithHumanOutput { #[allow(clippy::expect_used)] std::io::stdout().flush().expect("could not flush stdout"); } + EventMsg::AgentReasoningContent(AgentReasoningContentEvent { text }) => { + if !self.show_agent_reasoning { + return CodexStatus::Running; + } + print!("{text}"); + #[allow(clippy::expect_used)] + std::io::stdout().flush().expect("could not flush stdout"); + } EventMsg::AgentMessage(AgentMessageEvent { message }) => { // if answer_started is false, this means we haven't received any // delta. Thus, we need to print the message as a new answer. diff --git a/codex-rs/mcp-server/src/codex_tool_runner.rs b/codex-rs/mcp-server/src/codex_tool_runner.rs index 205dfa4631..3d32d8de52 100644 --- a/codex-rs/mcp-server/src/codex_tool_runner.rs +++ b/codex-rs/mcp-server/src/codex_tool_runner.rs @@ -252,7 +252,8 @@ async fn run_codex_tool_session_inner( EventMsg::AgentMessage(AgentMessageEvent { .. }) => { // TODO: think how we want to support this in the MCP } - EventMsg::TaskStarted + EventMsg::AgentReasoningContent(_) + | EventMsg::TaskStarted | EventMsg::TokenCount(_) | EventMsg::AgentReasoning(_) | EventMsg::McpToolCallBegin(_) diff --git a/codex-rs/mcp-server/src/conversation_loop.rs b/codex-rs/mcp-server/src/conversation_loop.rs index 1db39a2306..5b95c313f4 100644 --- a/codex-rs/mcp-server/src/conversation_loop.rs +++ b/codex-rs/mcp-server/src/conversation_loop.rs @@ -90,7 +90,8 @@ pub async fn run_conversation_loop( EventMsg::AgentMessage(AgentMessageEvent { .. }) => { // TODO: think how we want to support this in the MCP } - EventMsg::TaskStarted + EventMsg::AgentReasoningContent(_) + | EventMsg::TaskStarted | EventMsg::TokenCount(_) | EventMsg::AgentReasoning(_) | EventMsg::McpToolCallBegin(_) From 1a33de34b06eea9731ae27d2a2ab532af586e782 Mon Sep 17 00:00:00 2001 From: Ahmed Ibrahim Date: Mon, 4 Aug 2025 16:56:52 -0700 Subject: [PATCH 061/100] unify flag --- codex-rs/core/src/client.rs | 4 +--- codex-rs/core/src/codex.rs | 4 +--- codex-rs/core/src/config.rs | 21 --------------------- 3 files changed, 2 insertions(+), 27 deletions(-) diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index 8685bc54d3..fd530e0c6d 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -93,9 +93,7 @@ impl ModelClient { // Wrap it with the aggregation adapter so callers see *only* // the final assistant message per turn (matching the // behaviour of the Responses API). - let mut aggregated = if self.config.show_reasoning_content - && !self.config.hide_agent_reasoning - { + let mut aggregated = if !self.config.hide_agent_reasoning { crate::chat_completions::AggregatedChatStream::streaming_mode(response_stream) } else { response_stream.aggregate() diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 18bcf6261d..caebaed233 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -229,7 +229,6 @@ pub(crate) struct Session { state: Mutex, codex_linux_sandbox_exe: Option, user_shell: shell::Shell, - show_reasoning_content: bool, hide_agent_reasoning: bool, } @@ -826,7 +825,6 @@ async fn submission_loop( codex_linux_sandbox_exe: config.codex_linux_sandbox_exe.clone(), disable_response_storage, user_shell: default_shell, - show_reasoning_content: config.show_reasoning_content, hide_agent_reasoning: config.hide_agent_reasoning, })); @@ -1521,7 +1519,7 @@ async fn handle_response_item( sess.tx_event.send(event).await.ok(); } } - if !sess.hide_agent_reasoning && sess.show_reasoning_content && content.is_some() { + if !sess.hide_agent_reasoning && content.is_some() { let content = content.unwrap(); for item in content { let text = match item { diff --git a/codex-rs/core/src/config.rs b/codex-rs/core/src/config.rs index 3277ca08e9..302c468b66 100644 --- a/codex-rs/core/src/config.rs +++ b/codex-rs/core/src/config.rs @@ -57,10 +57,6 @@ pub struct Config { /// users are only interested in the final agent responses. pub hide_agent_reasoning: bool, - /// When `true`, the raw chain-of-thought text from reasoning events will be - /// displayed in the UI in addition to the reasoning summaries. - pub show_reasoning_content: bool, - /// Disable server-side response storage (sends the full conversation /// context with every request). Currently necessary for OpenAI customers /// who have opted into Zero Data Retention (ZDR). @@ -329,10 +325,6 @@ pub struct ConfigToml { /// UI/output. Defaults to `false`. pub hide_agent_reasoning: Option, - /// When set to `true`, raw chain-of-thought text from reasoning events will - /// be shown in the UI. - pub show_reasoning_content: Option, - pub model_reasoning_effort: Option, pub model_reasoning_summary: Option, @@ -499,15 +491,6 @@ impl Config { // Resolve hide/show reasoning flags with consistent precedence: // if hide is true, force show_reasoning_content to false. let hide_agent_reasoning_val = cfg.hide_agent_reasoning.unwrap_or(false); - let show_reasoning_content_val = if hide_agent_reasoning_val { - false - } else { - cfg.show_reasoning_content.unwrap_or(false) - }; - - if cfg.hide_agent_reasoning == Some(true) && cfg.show_reasoning_content == Some(true) { - tracing::warn!("Ignoring show_reasoning_content because hide_agent_reasoning is true"); - } let config = Self { model, @@ -539,7 +522,6 @@ impl Config { codex_linux_sandbox_exe, hide_agent_reasoning: hide_agent_reasoning_val, - show_reasoning_content: show_reasoning_content_val, model_reasoning_effort: config_profile .model_reasoning_effort .or(cfg.model_reasoning_effort) @@ -913,7 +895,6 @@ disable_response_storage = true tui: Tui::default(), codex_linux_sandbox_exe: None, hide_agent_reasoning: false, - show_reasoning_content: false, model_reasoning_effort: ReasoningEffort::High, model_reasoning_summary: ReasoningSummary::Detailed, model_supports_reasoning_summaries: false, @@ -964,7 +945,6 @@ disable_response_storage = true tui: Tui::default(), codex_linux_sandbox_exe: None, hide_agent_reasoning: false, - show_reasoning_content: false, model_reasoning_effort: ReasoningEffort::default(), model_reasoning_summary: ReasoningSummary::default(), model_supports_reasoning_summaries: false, @@ -1030,7 +1010,6 @@ disable_response_storage = true tui: Tui::default(), codex_linux_sandbox_exe: None, hide_agent_reasoning: false, - show_reasoning_content: false, model_reasoning_effort: ReasoningEffort::default(), model_reasoning_summary: ReasoningSummary::default(), model_supports_reasoning_summaries: false, From e38ce39c514d035d0638c82c85d8fe1c4e120fc1 Mon Sep 17 00:00:00 2001 From: Ahmed Ibrahim Date: Mon, 4 Aug 2025 17:03:24 -0700 Subject: [PATCH 062/100] Revert to 3f13ebce10209ab3645f51e7606892b3fd71d47e without rewriting history. Wrong merge --- codex-rs/core/src/chat_completions.rs | 163 ++++-------------- codex-rs/core/src/client.rs | 8 +- codex-rs/core/src/codex.rs | 58 ++----- codex-rs/core/src/config.rs | 6 +- codex-rs/core/src/models.rs | 8 - codex-rs/core/src/protocol.rs | 8 - .../src/event_processor_with_human_output.rs | 9 - codex-rs/mcp-server/src/codex_tool_runner.rs | 3 +- codex-rs/mcp-server/src/conversation_loop.rs | 3 +- 9 files changed, 55 insertions(+), 211 deletions(-) diff --git a/codex-rs/core/src/chat_completions.rs b/codex-rs/core/src/chat_completions.rs index d1b8338987..5ede774b1c 100644 --- a/codex-rs/core/src/chat_completions.rs +++ b/codex-rs/core/src/chat_completions.rs @@ -207,7 +207,6 @@ async fn process_chat_sse( } let mut fn_call_state = FunctionCallState::default(); - let mut assistant_text = String::new(); loop { let sse = match timeout(idle_timeout, stream.next()).await { @@ -255,42 +254,21 @@ async fn process_chat_sse( let choice_opt = chunk.get("choices").and_then(|c| c.get(0)); if let Some(choice) = choice_opt { - // Handle assistant content tokens as streaming deltas. + // Handle assistant content tokens. if let Some(content) = choice .get("delta") .and_then(|d| d.get("content")) .and_then(|c| c.as_str()) { - if !content.is_empty() { - assistant_text.push_str(content); - let _ = tx_event - .send(Ok(ResponseEvent::OutputTextDelta(content.to_string()))) - .await; - } - } + let item = ResponseItem::Message { + role: "assistant".to_string(), + content: vec![ContentItem::OutputText { + text: content.to_string(), + }], + id: None, + }; - // Forward any reasoning/thinking deltas if present. - if let Some(reasoning) = choice - .get("delta") - .and_then(|d| d.get("reasoning")) - .and_then(|c| c.as_str()) - { - let _ = tx_event - .send(Ok(ResponseEvent::ReasoningSummaryDelta( - reasoning.to_string(), - ))) - .await; - } - if let Some(reasoning_content) = choice - .get("delta") - .and_then(|d| d.get("reasoning_content")) - .and_then(|c| c.as_str()) - { - let _ = tx_event - .send(Ok(ResponseEvent::ReasoningSummaryDelta( - reasoning_content.to_string(), - ))) - .await; + let _ = tx_event.send(Ok(ResponseEvent::OutputItemDone(item))).await; } // Handle streaming function / tool calls. @@ -339,18 +317,7 @@ async fn process_chat_sse( let _ = tx_event.send(Ok(ResponseEvent::OutputItemDone(item))).await; } "stop" => { - // Regular turn without tool-call. Emit the final assistant message - // as a single OutputItemDone so non-delta consumers see the result. - if !assistant_text.is_empty() { - let item = ResponseItem::Message { - role: "assistant".to_string(), - content: vec![ContentItem::OutputText { - text: std::mem::take(&mut assistant_text), - }], - id: None, - }; - let _ = tx_event.send(Ok(ResponseEvent::OutputItemDone(item))).await; - } + // Regular turn without tool-call. } _ => {} } @@ -391,10 +358,7 @@ async fn process_chat_sse( pub(crate) struct AggregatedChatStream { inner: S, cumulative: String, - cumulative_reasoning: String, - pending: std::collections::VecDeque, - // When true, do not emit a cumulative assistant message at Completed. - streaming_mode: bool, + pending_completed: Option, } impl Stream for AggregatedChatStream @@ -406,8 +370,8 @@ where fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { let this = self.get_mut(); - // First, flush any buffered events from the previous call. - if let Some(ev) = this.pending.pop_front() { + // First, flush any buffered Completed event from the previous call. + if let Some(ev) = this.pending_completed.take() { return Poll::Ready(Some(Ok(ev))); } @@ -424,21 +388,16 @@ where let is_assistant_delta = matches!(&item, crate::models::ResponseItem::Message { role, .. } if role == "assistant"); if is_assistant_delta { - // Only use the final assistant message if we have not - // seen any deltas; otherwise, deltas already built the - // cumulative text and this would duplicate it. - if this.cumulative.is_empty() { - if let crate::models::ResponseItem::Message { content, .. } = &item { - if let Some(text) = content.iter().find_map(|c| match c { - crate::models::ContentItem::OutputText { text } => Some(text), - _ => None, - }) { - this.cumulative.push_str(text); - } + if let crate::models::ResponseItem::Message { content, .. } = &item { + if let Some(text) = content.iter().find_map(|c| match c { + crate::models::ContentItem::OutputText { text } => Some(text), + _ => None, + }) { + this.cumulative.push_str(text); } } - // Swallow assistant message here; emit on Completed. + // Swallow partial assistant chunk; keep polling. continue; } @@ -449,48 +408,24 @@ where response_id, token_usage, }))) => { - // Build any aggregated items in the correct order: Reasoning first, then Message. - let mut emitted_any = false; - - if !this.cumulative_reasoning.is_empty() { - let aggregated_reasoning = crate::models::ResponseItem::Reasoning { - id: String::new(), - summary: vec![ - crate::models::ReasoningItemReasoningSummary::SummaryText { - text: std::mem::take(&mut this.cumulative_reasoning), - }, - ], - content: None, - encrypted_content: None, - }; - this.pending - .push_back(ResponseEvent::OutputItemDone(aggregated_reasoning)); - emitted_any = true; - } - if !this.cumulative.is_empty() { - let aggregated_message = crate::models::ResponseItem::Message { + let aggregated_item = crate::models::ResponseItem::Message { id: None, role: "assistant".to_string(), content: vec![crate::models::ContentItem::OutputText { text: std::mem::take(&mut this.cumulative), }], }; - this.pending - .push_back(ResponseEvent::OutputItemDone(aggregated_message)); - emitted_any = true; - } - // Always emit Completed last when anything was aggregated. - if emitted_any { - this.pending.push_back(ResponseEvent::Completed { - response_id: response_id.clone(), - token_usage: token_usage.clone(), + // Buffer Completed so it is returned *after* the aggregated message. + this.pending_completed = Some(ResponseEvent::Completed { + response_id, + token_usage, }); - // Return the first pending event now. - if let Some(ev) = this.pending.pop_front() { - return Poll::Ready(Some(Ok(ev))); - } + + return Poll::Ready(Some(Ok(ResponseEvent::OutputItemDone( + aggregated_item, + )))); } // Nothing aggregated – forward Completed directly. @@ -504,25 +439,11 @@ where // will never appear in a Chat Completions stream. continue; } - Poll::Ready(Some(Ok(ResponseEvent::OutputTextDelta(delta)))) => { - // Always accumulate deltas so we can emit a final OutputItemDone at Completed. - this.cumulative.push_str(&delta); - if this.streaming_mode { - // In streaming mode, also forward the delta immediately. - return Poll::Ready(Some(Ok(ResponseEvent::OutputTextDelta(delta)))); - } else { - continue; - } - } - Poll::Ready(Some(Ok(ResponseEvent::ReasoningSummaryDelta(delta)))) => { - // Always accumulate reasoning deltas so we can emit a final Reasoning item at Completed. - this.cumulative_reasoning.push_str(&delta); - if this.streaming_mode { - // In streaming mode, also forward the delta immediately. - return Poll::Ready(Some(Ok(ResponseEvent::ReasoningSummaryDelta(delta)))); - } else { - continue; - } + Poll::Ready(Some(Ok(ResponseEvent::OutputTextDelta(_)))) + | Poll::Ready(Some(Ok(ResponseEvent::ReasoningSummaryDelta(_)))) => { + // Deltas are ignored here since aggregation waits for the + // final OutputItemDone. + continue; } } } @@ -554,23 +475,9 @@ pub(crate) trait AggregateStreamExt: Stream> + Size AggregatedChatStream { inner: self, cumulative: String::new(), - cumulative_reasoning: String::new(), - pending: std::collections::VecDeque::new(), - streaming_mode: false, + pending_completed: None, } } } impl AggregateStreamExt for T where T: Stream> + Sized {} - -impl AggregatedChatStream { - pub(crate) fn streaming_mode(inner: S) -> Self { - AggregatedChatStream { - inner, - cumulative: String::new(), - cumulative_reasoning: String::new(), - pending: std::collections::VecDeque::new(), - streaming_mode: true, - } - } -} diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index fd530e0c6d..b9ea6b13f4 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -93,11 +93,7 @@ impl ModelClient { // Wrap it with the aggregation adapter so callers see *only* // the final assistant message per turn (matching the // behaviour of the Responses API). - let mut aggregated = if !self.config.hide_agent_reasoning { - crate::chat_completions::AggregatedChatStream::streaming_mode(response_stream) - } else { - response_stream.aggregate() - }; + let mut aggregated = response_stream.aggregate(); // Bridge the aggregated stream back into a standard // `ResponseStream` by forwarding events through a channel. @@ -442,7 +438,7 @@ async fn process_sse( } } } - "response.reasoning_summary_text.delta" | "response.reasoning_text.delta" => { + "response.reasoning_summary_text.delta" => { if let Some(delta) = event.delta { let event = ResponseEvent::ReasoningSummaryDelta(delta); if tx_event.send(Ok(event)).await.is_err() { diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index caebaed233..568d87c4a8 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -56,7 +56,6 @@ use crate::mcp_tool_call::handle_mcp_tool_call; use crate::models::ContentItem; use crate::models::FunctionCallOutputPayload; use crate::models::LocalShellAction; -use crate::models::ReasoningItemContent; use crate::models::ReasoningItemReasoningSummary; use crate::models::ResponseInputItem; use crate::models::ResponseItem; @@ -65,7 +64,6 @@ use crate::plan_tool::handle_update_plan; use crate::project_doc::get_user_instructions; use crate::protocol::AgentMessageDeltaEvent; use crate::protocol::AgentMessageEvent; -use crate::protocol::AgentReasoningContentEvent; use crate::protocol::AgentReasoningDeltaEvent; use crate::protocol::AgentReasoningEvent; use crate::protocol::ApplyPatchApprovalRequestEvent; @@ -229,7 +227,6 @@ pub(crate) struct Session { state: Mutex, codex_linux_sandbox_exe: Option, user_shell: shell::Shell, - hide_agent_reasoning: bool, } impl Session { @@ -825,7 +822,6 @@ async fn submission_loop( codex_linux_sandbox_exe: config.codex_linux_sandbox_exe.clone(), disable_response_storage, user_shell: default_shell, - hide_agent_reasoning: config.hide_agent_reasoning, })); // Patch restored state into the newly created session. @@ -1136,7 +1132,6 @@ async fn run_task(sess: Arc, sub_id: String, input: Vec) { ResponseItem::Reasoning { id, summary, - content, encrypted_content, }, None, @@ -1144,7 +1139,6 @@ async fn run_task(sess: Arc, sub_id: String, input: Vec) { items_to_record_in_conversation_history.push(ResponseItem::Reasoning { id: id.clone(), summary: summary.clone(), - content: content.clone(), encrypted_content: encrypted_content.clone(), }); } @@ -1387,13 +1381,11 @@ async fn try_run_turn( sess.tx_event.send(event).await.ok(); } ResponseEvent::ReasoningSummaryDelta(delta) => { - if !sess.hide_agent_reasoning { - let event = Event { - id: sub_id.to_string(), - msg: EventMsg::AgentReasoningDelta(AgentReasoningDeltaEvent { delta }), - }; - sess.tx_event.send(event).await.ok(); - } + let event = Event { + id: sub_id.to_string(), + msg: EventMsg::AgentReasoningDelta(AgentReasoningDeltaEvent { delta }), + }; + sess.tx_event.send(event).await.ok(); } } } @@ -1501,36 +1493,16 @@ async fn handle_response_item( } None } - ResponseItem::Reasoning { - id: _, - summary, - content, - encrypted_content: _, - } => { - if !sess.hide_agent_reasoning { - for item in summary { - let text = match item { - ReasoningItemReasoningSummary::SummaryText { text } => text, - }; - let event = Event { - id: sub_id.to_string(), - msg: EventMsg::AgentReasoning(AgentReasoningEvent { text }), - }; - sess.tx_event.send(event).await.ok(); - } - } - if !sess.hide_agent_reasoning && content.is_some() { - let content = content.unwrap(); - for item in content { - let text = match item { - ReasoningItemContent::ReasoningText { text } => text, - }; - let event = Event { - id: sub_id.to_string(), - msg: EventMsg::AgentReasoningContent(AgentReasoningContentEvent { text }), - }; - sess.tx_event.send(event).await.ok(); - } + ResponseItem::Reasoning { summary, .. } => { + for item in summary { + let text = match item { + ReasoningItemReasoningSummary::SummaryText { text } => text, + }; + let event = Event { + id: sub_id.to_string(), + msg: EventMsg::AgentReasoning(AgentReasoningEvent { text }), + }; + sess.tx_event.send(event).await.ok(); } None } diff --git a/codex-rs/core/src/config.rs b/codex-rs/core/src/config.rs index 302c468b66..b43dc56ba0 100644 --- a/codex-rs/core/src/config.rs +++ b/codex-rs/core/src/config.rs @@ -488,10 +488,6 @@ impl Config { Self::get_base_instructions(experimental_instructions_path, &resolved_cwd)?; let base_instructions = base_instructions.or(file_base_instructions); - // Resolve hide/show reasoning flags with consistent precedence: - // if hide is true, force show_reasoning_content to false. - let hide_agent_reasoning_val = cfg.hide_agent_reasoning.unwrap_or(false); - let config = Self { model, model_context_window, @@ -521,7 +517,7 @@ impl Config { tui: cfg.tui.unwrap_or_default(), codex_linux_sandbox_exe, - hide_agent_reasoning: hide_agent_reasoning_val, + hide_agent_reasoning: cfg.hide_agent_reasoning.unwrap_or(false), model_reasoning_effort: config_profile .model_reasoning_effort .or(cfg.model_reasoning_effort) diff --git a/codex-rs/core/src/models.rs b/codex-rs/core/src/models.rs index 98d8727e77..166404915a 100644 --- a/codex-rs/core/src/models.rs +++ b/codex-rs/core/src/models.rs @@ -45,8 +45,6 @@ pub enum ResponseItem { Reasoning { id: String, summary: Vec, - #[serde(default, skip_serializing_if = "Option::is_none")] - content: Option>, encrypted_content: Option, }, LocalShellCall { @@ -138,12 +136,6 @@ pub enum ReasoningItemReasoningSummary { SummaryText { text: String }, } -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum ReasoningItemContent { - ReasoningText { text: String }, -} - impl From> for ResponseInputItem { fn from(items: Vec) -> Self { Self::Message { diff --git a/codex-rs/core/src/protocol.rs b/codex-rs/core/src/protocol.rs index 1e0733628c..82591a2c78 100644 --- a/codex-rs/core/src/protocol.rs +++ b/codex-rs/core/src/protocol.rs @@ -359,9 +359,6 @@ pub enum EventMsg { /// Agent reasoning delta event from agent. AgentReasoningDelta(AgentReasoningDeltaEvent), - /// Raw chain-of-thought from agent. - AgentReasoningContent(AgentReasoningContentEvent), - /// Ack the client's configure message. SessionConfigured(SessionConfiguredEvent), @@ -467,11 +464,6 @@ pub struct AgentReasoningEvent { pub text: String, } -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct AgentReasoningContentEvent { - pub text: String, -} - #[derive(Debug, Clone, Deserialize, Serialize)] pub struct AgentReasoningDeltaEvent { pub delta: String, diff --git a/codex-rs/exec/src/event_processor_with_human_output.rs b/codex-rs/exec/src/event_processor_with_human_output.rs index 1f8fe3c031..7703c138fc 100644 --- a/codex-rs/exec/src/event_processor_with_human_output.rs +++ b/codex-rs/exec/src/event_processor_with_human_output.rs @@ -4,7 +4,6 @@ use codex_core::config::Config; use codex_core::plan_tool::UpdatePlanArgs; use codex_core::protocol::AgentMessageDeltaEvent; use codex_core::protocol::AgentMessageEvent; -use codex_core::protocol::AgentReasoningContentEvent; use codex_core::protocol::AgentReasoningDeltaEvent; use codex_core::protocol::BackgroundEventEvent; use codex_core::protocol::ErrorEvent; @@ -204,14 +203,6 @@ impl EventProcessor for EventProcessorWithHumanOutput { #[allow(clippy::expect_used)] std::io::stdout().flush().expect("could not flush stdout"); } - EventMsg::AgentReasoningContent(AgentReasoningContentEvent { text }) => { - if !self.show_agent_reasoning { - return CodexStatus::Running; - } - print!("{text}"); - #[allow(clippy::expect_used)] - std::io::stdout().flush().expect("could not flush stdout"); - } EventMsg::AgentMessage(AgentMessageEvent { message }) => { // if answer_started is false, this means we haven't received any // delta. Thus, we need to print the message as a new answer. diff --git a/codex-rs/mcp-server/src/codex_tool_runner.rs b/codex-rs/mcp-server/src/codex_tool_runner.rs index 3d32d8de52..205dfa4631 100644 --- a/codex-rs/mcp-server/src/codex_tool_runner.rs +++ b/codex-rs/mcp-server/src/codex_tool_runner.rs @@ -252,8 +252,7 @@ async fn run_codex_tool_session_inner( EventMsg::AgentMessage(AgentMessageEvent { .. }) => { // TODO: think how we want to support this in the MCP } - EventMsg::AgentReasoningContent(_) - | EventMsg::TaskStarted + EventMsg::TaskStarted | EventMsg::TokenCount(_) | EventMsg::AgentReasoning(_) | EventMsg::McpToolCallBegin(_) diff --git a/codex-rs/mcp-server/src/conversation_loop.rs b/codex-rs/mcp-server/src/conversation_loop.rs index 5b95c313f4..1db39a2306 100644 --- a/codex-rs/mcp-server/src/conversation_loop.rs +++ b/codex-rs/mcp-server/src/conversation_loop.rs @@ -90,8 +90,7 @@ pub async fn run_conversation_loop( EventMsg::AgentMessage(AgentMessageEvent { .. }) => { // TODO: think how we want to support this in the MCP } - EventMsg::AgentReasoningContent(_) - | EventMsg::TaskStarted + EventMsg::TaskStarted | EventMsg::TokenCount(_) | EventMsg::AgentReasoning(_) | EventMsg::McpToolCallBegin(_) From 84bcadb8d92d6a0b694cb19c865d6aafac14d6f3 Mon Sep 17 00:00:00 2001 From: pakrym-oai Date: Mon, 4 Aug 2025 18:07:49 -0700 Subject: [PATCH 063/100] Restore API key and query param overrides (#1826) Addresses https://github.com/openai/codex/issues/1796 --- codex-rs/core/src/chat_completions.rs | 4 +- codex-rs/core/src/client.rs | 51 ++++++---------- codex-rs/core/src/model_provider_info.rs | 62 +++++++++++++------ codex-rs/core/tests/client.rs | 78 ++++++++++++++++++++++++ codex-rs/login/src/lib.rs | 2 +- 5 files changed, 144 insertions(+), 53 deletions(-) diff --git a/codex-rs/core/src/chat_completions.rs b/codex-rs/core/src/chat_completions.rs index 5ede774b1c..b1dee853f0 100644 --- a/codex-rs/core/src/chat_completions.rs +++ b/codex-rs/core/src/chat_completions.rs @@ -120,7 +120,7 @@ pub(crate) async fn stream_chat_completions( debug!( "POST to {}: {}", - provider.get_full_url(), + provider.get_full_url(&None), serde_json::to_string_pretty(&payload).unwrap_or_default() ); @@ -129,7 +129,7 @@ pub(crate) async fn stream_chat_completions( loop { attempt += 1; - let req_builder = provider.create_request_builder(client)?; + let req_builder = provider.create_request_builder(client, &None).await?; let res = req_builder .header(reqwest::header::ACCEPT, "text/event-stream") diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index b9ea6b13f4..1a8ae94f81 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -30,7 +30,6 @@ use crate::config::Config; use crate::config_types::ReasoningEffort as ReasoningEffortConfig; use crate::config_types::ReasoningSummary as ReasoningSummaryConfig; use crate::error::CodexErr; -use crate::error::EnvVarError; use crate::error::Result; use crate::flags::CODEX_RS_SSE_FIXTURE; use crate::model_provider_info::ModelProviderInfo; @@ -122,24 +121,11 @@ impl ModelClient { return stream_from_fixture(path, self.provider.clone()).await; } - let auth = self.auth.as_ref().ok_or_else(|| { - CodexErr::EnvVar(EnvVarError { - var: "OPENAI_API_KEY".to_string(), - instructions: Some("Create an API key (https://platform.openai.com) and export it as an environment variable.".to_string()), - }) - })?; + let auth = self.auth.clone(); - let store = prompt.store && auth.mode != AuthMode::ChatGPT; + let auth_mode = auth.as_ref().map(|a| a.mode); - let base_url = match self.provider.base_url.clone() { - Some(url) => url, - None => match auth.mode { - AuthMode::ChatGPT => "https://chatgpt.com/backend-api/codex".to_string(), - AuthMode::ApiKey => "https://api.openai.com/v1".to_string(), - }, - }; - - let token = auth.get_token().await?; + let store = prompt.store && auth_mode != Some(AuthMode::ChatGPT); let full_instructions = prompt.get_full_instructions(&self.config.model); let tools_json = create_tools_json_for_responses_api( @@ -180,35 +166,36 @@ impl ModelClient { include, }; - trace!( - "POST to {}: {}", - self.provider.get_full_url(), - serde_json::to_string(&payload)? - ); - let mut attempt = 0; let max_retries = self.provider.request_max_retries(); + trace!( + "POST to {}: {}", + self.provider.get_full_url(&auth), + serde_json::to_string(&payload)? + ); + loop { attempt += 1; let mut req_builder = self - .client - .post(format!("{base_url}/responses")) + .provider + .create_request_builder(&self.client, &auth) + .await?; + + req_builder = req_builder .header("OpenAI-Beta", "responses=experimental") .header("session_id", self.session_id.to_string()) - .bearer_auth(&token) .header(reqwest::header::ACCEPT, "text/event-stream") .json(&payload); - if auth.mode == AuthMode::ChatGPT { - if let Some(account_id) = auth.get_account_id().await { - req_builder = req_builder.header("chatgpt-account-id", account_id); - } + if let Some(auth) = auth.as_ref() + && auth.mode == AuthMode::ChatGPT + && let Some(account_id) = auth.get_account_id().await + { + req_builder = req_builder.header("chatgpt-account-id", account_id); } - req_builder = self.provider.apply_http_headers(req_builder); - let originator = self .config .internal_originator diff --git a/codex-rs/core/src/model_provider_info.rs b/codex-rs/core/src/model_provider_info.rs index 2936637779..49478660f4 100644 --- a/codex-rs/core/src/model_provider_info.rs +++ b/codex-rs/core/src/model_provider_info.rs @@ -5,8 +5,11 @@ //! 2. User-defined entries inside `~/.codex/config.toml` under the `model_providers` //! key. These override or extend the defaults at runtime. +use codex_login::AuthMode; +use codex_login::CodexAuth; use serde::Deserialize; use serde::Serialize; +use std::borrow::Cow; use std::collections::HashMap; use std::env::VarError; use std::time::Duration; @@ -88,25 +91,30 @@ impl ModelProviderInfo { /// When `require_api_key` is true and the provider declares an `env_key` /// but the variable is missing/empty, returns an [`Err`] identical to the /// one produced by [`ModelProviderInfo::api_key`]. - pub fn create_request_builder<'a>( + pub async fn create_request_builder<'a>( &'a self, client: &'a reqwest::Client, + auth: &Option, ) -> crate::error::Result { - let url = self.get_full_url(); + let auth: Cow<'_, Option> = if auth.is_some() { + Cow::Borrowed(auth) + } else { + Cow::Owned(self.get_fallback_auth()?) + }; + + let url = self.get_full_url(&auth); let mut builder = client.post(url); - let api_key = self.api_key()?; - if let Some(key) = api_key { - builder = builder.bearer_auth(key); + if let Some(auth) = auth.as_ref() { + builder = builder.bearer_auth(auth.get_token().await?); } Ok(self.apply_http_headers(builder)) } - pub(crate) fn get_full_url(&self) -> String { - let query_string = self - .query_params + fn get_query_string(&self) -> String { + self.query_params .as_ref() .map_or_else(String::new, |params| { let full_params = params @@ -115,16 +123,29 @@ impl ModelProviderInfo { .collect::>() .join("&"); format!("?{full_params}") - }); + }) + } + + pub(crate) fn get_full_url(&self, auth: &Option) -> String { + let default_base_url = if matches!( + auth, + Some(CodexAuth { + mode: AuthMode::ChatGPT, + .. + }) + ) { + "https://chatgpt.com/backend-api/codex" + } else { + "https://api.openai.com/v1" + }; + let query_string = self.get_query_string(); let base_url = self .base_url .clone() - .unwrap_or("https://api.openai.com/v1".to_string()); + .unwrap_or(default_base_url.to_string()); match self.wire_api { - WireApi::Responses => { - format!("{base_url}/responses{query_string}") - } + WireApi::Responses => format!("{base_url}/responses{query_string}"), WireApi::Chat => format!("{base_url}/chat/completions{query_string}"), } } @@ -132,10 +153,7 @@ impl ModelProviderInfo { /// Apply provider-specific HTTP headers (both static and environment-based) /// onto an existing `reqwest::RequestBuilder` and return the updated /// builder. - pub fn apply_http_headers( - &self, - mut builder: reqwest::RequestBuilder, - ) -> reqwest::RequestBuilder { + fn apply_http_headers(&self, mut builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { if let Some(extra) = &self.http_headers { for (k, v) in extra { builder = builder.header(k, v); @@ -157,7 +175,7 @@ impl ModelProviderInfo { /// If `env_key` is Some, returns the API key for this provider if present /// (and non-empty) in the environment. If `env_key` is required but /// cannot be found, returns an error. - fn api_key(&self) -> crate::error::Result> { + pub fn api_key(&self) -> crate::error::Result> { match &self.env_key { Some(env_key) => { let env_value = std::env::var(env_key); @@ -198,6 +216,14 @@ impl ModelProviderInfo { .map(Duration::from_millis) .unwrap_or(Duration::from_millis(DEFAULT_STREAM_IDLE_TIMEOUT_MS)) } + + fn get_fallback_auth(&self) -> crate::error::Result> { + let api_key = self.api_key()?; + if let Some(api_key) = api_key { + return Ok(Some(CodexAuth::from_api_key(api_key))); + } + Ok(None) + } } /// Built-in default provider list. diff --git a/codex-rs/core/tests/client.rs b/codex-rs/core/tests/client.rs index a22a94388b..06a110ea3a 100644 --- a/codex-rs/core/tests/client.rs +++ b/codex-rs/core/tests/client.rs @@ -4,6 +4,7 @@ use chrono::Utc; use codex_core::Codex; use codex_core::CodexSpawnOk; use codex_core::ModelProviderInfo; +use codex_core::WireApi; use codex_core::built_in_model_providers; use codex_core::protocol::EventMsg; use codex_core::protocol::InputItem; @@ -21,8 +22,10 @@ use tempfile::TempDir; use wiremock::Mock; use wiremock::MockServer; use wiremock::ResponseTemplate; +use wiremock::matchers::header_regex; use wiremock::matchers::method; use wiremock::matchers::path; +use wiremock::matchers::query_param; /// Build minimal SSE stream with completed marker using the JSON fixture. fn sse_completed(id: &str) -> String { @@ -376,6 +379,81 @@ async fn includes_user_instructions_message_in_request() { .starts_with("be nice") ); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn azure_overrides_assign_properties_used_for_responses_url() { + #![allow(clippy::unwrap_used)] + + let existing_env_var_with_random_value = if cfg!(windows) { "USERNAME" } else { "USER" }; + + // Mock server + let server = MockServer::start().await; + + // First request – must NOT include `previous_response_id`. + let first = ResponseTemplate::new(200) + .insert_header("content-type", "text/event-stream") + .set_body_raw(sse_completed("resp1"), "text/event-stream"); + + // Expect POST to /openai/responses with api-version query param + Mock::given(method("POST")) + .and(path("/openai/responses")) + .and(query_param("api-version", "2025-04-01-preview")) + .and(header_regex("Custom-Header", "Value")) + .and(header_regex( + "Authorization", + format!( + "Bearer {}", + std::env::var(existing_env_var_with_random_value).unwrap() + ) + .as_str(), + )) + .respond_with(first) + .expect(1) + .mount(&server) + .await; + + let provider = ModelProviderInfo { + name: "custom".to_string(), + base_url: Some(format!("{}/openai", server.uri())), + // Reuse the existing environment variable to avoid using unsafe code + env_key: Some(existing_env_var_with_random_value.to_string()), + query_params: Some(std::collections::HashMap::from([( + "api-version".to_string(), + "2025-04-01-preview".to_string(), + )])), + env_key_instructions: None, + wire_api: WireApi::Responses, + http_headers: Some(std::collections::HashMap::from([( + "Custom-Header".to_string(), + "Value".to_string(), + )])), + env_http_headers: None, + request_max_retries: None, + stream_max_retries: None, + stream_idle_timeout_ms: None, + requires_auth: false, + }; + + // Init session + let codex_home = TempDir::new().unwrap(); + let mut config = load_default_config_for_test(&codex_home); + config.model_provider = provider; + + let ctrl_c = std::sync::Arc::new(tokio::sync::Notify::new()); + let CodexSpawnOk { codex, .. } = Codex::spawn(config, None, ctrl_c.clone()).await.unwrap(); + + codex + .submit(Op::UserInput { + items: vec![InputItem::Text { + text: "hello".into(), + }], + }) + .await + .unwrap(); + + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TaskComplete(_))).await; +} + fn auth_from_token(id_token: String) -> CodexAuth { CodexAuth::new( None, diff --git a/codex-rs/login/src/lib.rs b/codex-rs/login/src/lib.rs index 3d55c20264..35f67e7109 100644 --- a/codex-rs/login/src/lib.rs +++ b/codex-rs/login/src/lib.rs @@ -22,7 +22,7 @@ const SOURCE_FOR_PYTHON_SERVER: &str = include_str!("./login_with_chatgpt.py"); const CLIENT_ID: &str = "app_EMoamEEZ73f0CkXaXp7hrann"; pub const OPENAI_API_KEY_ENV_VAR: &str = "OPENAI_API_KEY"; -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq, Copy)] pub enum AuthMode { ApiKey, ChatGPT, From f58401e203b08e64d7aff829f165d0a11c940ae7 Mon Sep 17 00:00:00 2001 From: pakrym-oai Date: Mon, 4 Aug 2025 18:45:13 -0700 Subject: [PATCH 064/100] Request the simplified auth flow (#1834) --- codex-rs/login/src/login_with_chatgpt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/codex-rs/login/src/login_with_chatgpt.py b/codex-rs/login/src/login_with_chatgpt.py index 4c07feeba0..14ccfa9ed4 100644 --- a/codex-rs/login/src/login_with_chatgpt.py +++ b/codex-rs/login/src/login_with_chatgpt.py @@ -458,6 +458,7 @@ class _ApiKeyHTTPServer(http.server.HTTPServer): "code_challenge": self.pkce.code_challenge, "code_challenge_method": "S256", "id_token_add_organizations": "true", + "codex_cli_simplified_flow": "true", "state": self.state, } return f"{self.issuer}/oauth/authorize?" + urllib.parse.urlencode(params) From 063083af157dcf57703462c07789c54695861dff Mon Sep 17 00:00:00 2001 From: Dylan Date: Mon, 4 Aug 2025 18:55:57 -0700 Subject: [PATCH 065/100] [prompts] Better user_instructions handling (#1836) ## Summary Our recent change in #1737 can sometimes lead to the model confusing AGENTS.md context as part of the message. But a little prompting and formatting can help fix this! ## Testing - Ran locally with a few different prompts to verify the model behaves well. - Updated unit tests --- codex-rs/core/prompt.md | 2 ++ codex-rs/core/src/chat_completions.rs | 2 +- codex-rs/core/src/client.rs | 4 ++-- codex-rs/core/src/client_common.rs | 10 ++++++++++ codex-rs/core/tests/client.rs | 8 +++++++- 5 files changed, 22 insertions(+), 4 deletions(-) diff --git a/codex-rs/core/prompt.md b/codex-rs/core/prompt.md index 4e55003b9f..f194eba4e2 100644 --- a/codex-rs/core/prompt.md +++ b/codex-rs/core/prompt.md @@ -9,6 +9,8 @@ You MUST adhere to the following criteria when executing the task: - Analyzing code for vulnerabilities is allowed. - Showing user code and tool call details is allowed. - User instructions may overwrite the _CODING GUIDELINES_ section in this developer message. +- `user_instructions` are not part of the user's request, but guidance for how to complete the task. +- Do not cite `user_instructions` back to the user unless a specific piece is relevant. - Do not use \`ls -R\`, \`find\`, or \`grep\` - these are slow in large repos. Use \`rg\` and \`rg --files\`. - Use \`apply_patch\` to edit files: {"command":["apply_patch","*** Begin Patch\\n*** Update File: path/to/file.py\\n@@ def example():\\n- pass\\n+ return 123\\n*** End Patch"]} - If completing the user's task requires writing or modifying files: diff --git a/codex-rs/core/src/chat_completions.rs b/codex-rs/core/src/chat_completions.rs index b1dee853f0..b5ade23b9d 100644 --- a/codex-rs/core/src/chat_completions.rs +++ b/codex-rs/core/src/chat_completions.rs @@ -40,7 +40,7 @@ pub(crate) async fn stream_chat_completions( let full_instructions = prompt.get_full_instructions(model); messages.push(json!({"role": "system", "content": full_instructions})); - if let Some(instr) = &prompt.user_instructions { + if let Some(instr) = &prompt.get_formatted_user_instructions() { messages.push(json!({"role": "user", "content": instr})); } diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index 1a8ae94f81..00762a8a67 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -144,11 +144,11 @@ impl ModelClient { }; let mut input_with_instructions = Vec::with_capacity(prompt.input.len() + 1); - if let Some(ui) = &prompt.user_instructions { + if let Some(ui) = prompt.get_formatted_user_instructions() { input_with_instructions.push(ResponseItem::Message { id: None, role: "user".to_string(), - content: vec![ContentItem::InputText { text: ui.clone() }], + content: vec![ContentItem::InputText { text: ui }], }); } input_with_instructions.extend(prompt.input.clone()); diff --git a/codex-rs/core/src/client_common.rs b/codex-rs/core/src/client_common.rs index 157f35872a..6d9524cc92 100644 --- a/codex-rs/core/src/client_common.rs +++ b/codex-rs/core/src/client_common.rs @@ -17,6 +17,10 @@ use tokio::sync::mpsc; /// with this content. const BASE_INSTRUCTIONS: &str = include_str!("../prompt.md"); +/// wraps user instructions message in a tag for the model to parse more easily. +const USER_INSTRUCTIONS_START: &str = "\n\n"; +const USER_INSTRUCTIONS_END: &str = "\n\n"; + /// API request payload for a single model turn. #[derive(Default, Debug, Clone)] pub struct Prompt { @@ -49,6 +53,12 @@ impl Prompt { } Cow::Owned(sections.join("\n")) } + + pub(crate) fn get_formatted_user_instructions(&self) -> Option { + self.user_instructions + .as_ref() + .map(|ui| format!("{USER_INSTRUCTIONS_START}{ui}{USER_INSTRUCTIONS_END}")) + } } #[derive(Debug)] diff --git a/codex-rs/core/tests/client.rs b/codex-rs/core/tests/client.rs index 06a110ea3a..f493020210 100644 --- a/codex-rs/core/tests/client.rs +++ b/codex-rs/core/tests/client.rs @@ -376,7 +376,13 @@ async fn includes_user_instructions_message_in_request() { request_body["input"][0]["content"][0]["text"] .as_str() .unwrap() - .starts_with("be nice") + .starts_with("\n\nbe nice") + ); + assert!( + request_body["input"][0]["content"][0]["text"] + .as_str() + .unwrap() + .ends_with("") ); } From 906d44976001aa6fbd51c9a40129e20b92e819ce Mon Sep 17 00:00:00 2001 From: easong-openai Date: Mon, 4 Aug 2025 21:23:22 -0700 Subject: [PATCH 066/100] Stream model responses (#1810) Stream models thoughts and responses instead of waiting for the whole thing to come through. Very rough right now, but I'm making the risk call to push through. --- codex-rs/Cargo.lock | 50 ++- codex-rs/core/src/chat_completions.rs | 18 +- codex-rs/core/src/codex.rs | 12 +- codex-rs/core/src/conversation_history.rs | 186 +++++++- codex-rs/core/src/models.rs | 17 +- codex-rs/tui/Cargo.toml | 5 + .../tui/src/bottom_pane/live_ring_widget.rs | 45 ++ codex-rs/tui/src/bottom_pane/mod.rs | 399 ++++++++++++++++-- codex-rs/tui/src/chatwidget.rs | 174 ++++++-- codex-rs/tui/src/history_cell.rs | 32 +- codex-rs/tui/src/insert_history.rs | 81 +++- codex-rs/tui/src/lib.rs | 5 +- codex-rs/tui/src/live_wrap.rs | 290 +++++++++++++ codex-rs/tui/src/markdown.rs | 6 +- codex-rs/tui/src/status_indicator_widget.rs | 215 +++++++--- codex-rs/tui/tests/vt100_history.rs | 214 ++++++++++ codex-rs/tui/tests/vt100_live_commit.rs | 101 +++++ 17 files changed, 1616 insertions(+), 234 deletions(-) create mode 100644 codex-rs/tui/src/bottom_pane/live_ring_widget.rs create mode 100644 codex-rs/tui/src/live_wrap.rs create mode 100644 codex-rs/tui/tests/vt100_history.rs create mode 100644 codex-rs/tui/tests/vt100_live_commit.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 4daae977b0..2e20a7d624 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -881,6 +881,7 @@ dependencies = [ "unicode-segmentation", "unicode-width 0.1.14", "uuid", + "vt100", ] [[package]] @@ -1473,7 +1474,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.52.0", ] [[package]] @@ -1553,7 +1554,7 @@ checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" dependencies = [ "cfg-if", "rustix 1.0.8", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -1756,7 +1757,7 @@ version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cba6ae63eb948698e300f645f87c70f76630d505f23b8907cf1e193ee85048c1" dependencies = [ - "unicode-width 0.2.0", + "unicode-width 0.2.1", ] [[package]] @@ -2336,7 +2337,7 @@ checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -3392,7 +3393,7 @@ dependencies = [ [[package]] name = "ratatui" version = "0.29.0" -source = "git+https://github.com/nornagon/ratatui?branch=nornagon-v0.29.0-patch#bca287ddc5d38fe088c79e2eda22422b96226f2e" +source = "git+https://github.com/nornagon/ratatui?branch=nornagon-v0.29.0-patch#9b2ad1298408c45918ee9f8241a6f95498cdbed2" dependencies = [ "bitflags 2.9.1", "cassowary", @@ -3406,7 +3407,7 @@ dependencies = [ "strum 0.26.3", "unicode-segmentation", "unicode-truncate", - "unicode-width 0.2.0", + "unicode-width 0.2.1", ] [[package]] @@ -3720,7 +3721,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -3733,7 +3734,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.9.4", - "windows-sys 0.60.2", + "windows-sys 0.52.0", ] [[package]] @@ -4499,7 +4500,7 @@ dependencies = [ "getrandom 0.3.3", "once_cell", "rustix 1.0.8", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -4546,7 +4547,7 @@ checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" dependencies = [ "smawk", "unicode-linebreak", - "unicode-width 0.2.0", + "unicode-width 0.2.1", ] [[package]] @@ -4994,7 +4995,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "911e93158bf80bbc94bad533b2b16e3d711e1132d69a6a6980c3920a63422c19" dependencies = [ "ratatui", - "unicode-width 0.2.0", + "unicode-width 0.2.1", ] [[package]] @@ -5062,9 +5063,9 @@ checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" [[package]] name = "unicode-width" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" +checksum = "4a1a07cc7db3810833284e8d372ccdc6da29741639ecc70c9ec107df0fa6154c" [[package]] name = "unicode-xid" @@ -5149,6 +5150,27 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vt100" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "054ff75fb8fa83e609e685106df4faeffdf3a735d3c74ebce97ec557d5d36fd9" +dependencies = [ + "itoa", + "unicode-width 0.2.1", + "vte", +] + +[[package]] +name = "vte" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5924018406ce0063cd67f8e008104968b74b563ee1b85dde3ed1f7cb87d3dbd" +dependencies = [ + "arrayvec", + "memchr", +] + [[package]] name = "wait-timeout" version = "0.2.1" @@ -5337,7 +5359,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] diff --git a/codex-rs/core/src/chat_completions.rs b/codex-rs/core/src/chat_completions.rs index b5ade23b9d..e1804b191e 100644 --- a/codex-rs/core/src/chat_completions.rs +++ b/codex-rs/core/src/chat_completions.rs @@ -260,6 +260,11 @@ async fn process_chat_sse( .and_then(|d| d.get("content")) .and_then(|c| c.as_str()) { + // Emit a delta so downstream consumers can stream text live. + let _ = tx_event + .send(Ok(ResponseEvent::OutputTextDelta(content.to_string()))) + .await; + let item = ResponseItem::Message { role: "assistant".to_string(), content: vec![ContentItem::OutputText { @@ -439,11 +444,14 @@ where // will never appear in a Chat Completions stream. continue; } - Poll::Ready(Some(Ok(ResponseEvent::OutputTextDelta(_)))) - | Poll::Ready(Some(Ok(ResponseEvent::ReasoningSummaryDelta(_)))) => { - // Deltas are ignored here since aggregation waits for the - // final OutputItemDone. - continue; + Poll::Ready(Some(Ok(ResponseEvent::OutputTextDelta(delta)))) => { + // Forward deltas unchanged so callers can stream text + // live while still receiving a single aggregated + // OutputItemDone at the end of the turn. + return Poll::Ready(Some(Ok(ResponseEvent::OutputTextDelta(delta)))); + } + Poll::Ready(Some(Ok(ResponseEvent::ReasoningSummaryDelta(delta)))) => { + return Poll::Ready(Some(Ok(ResponseEvent::ReasoningSummaryDelta(delta)))); } } } diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 568d87c4a8..8d24356460 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -123,7 +123,7 @@ impl Codex { let resume_path = config.experimental_resume.clone(); info!("resume_path: {resume_path:?}"); let (tx_sub, rx_sub) = async_channel::bounded(64); - let (tx_event, rx_event) = async_channel::bounded(1600); + let (tx_event, rx_event) = async_channel::unbounded(); let user_instructions = get_user_instructions(&config).await; @@ -701,7 +701,7 @@ async fn submission_loop( cwd, resume_path, } => { - info!( + debug!( "Configuring session: model={model}; provider={provider:?}; resume={resume_path:?}" ); if !cwd.is_absolute() { @@ -1374,6 +1374,11 @@ async fn try_run_turn( return Ok(output); } ResponseEvent::OutputTextDelta(delta) => { + { + let mut st = sess.state.lock().unwrap(); + st.history.append_assistant_text(&delta); + } + let event = Event { id: sub_id.to_string(), msg: EventMsg::AgentMessageDelta(AgentMessageDeltaEvent { delta }), @@ -1921,7 +1926,8 @@ async fn handle_sandbox_error( // include additional metadata on the command to indicate whether non-zero // exit codes merit a retry. - // For now, we categorically ask the user to retry without sandbox. + // For now, we categorically ask the user to retry without sandbox and + // emit the raw error as a background event. sess.notify_background_event(&sub_id, format!("Execution failed: {error}")) .await; diff --git a/codex-rs/core/src/conversation_history.rs b/codex-rs/core/src/conversation_history.rs index f5254f339e..1d55b125bc 100644 --- a/codex-rs/core/src/conversation_history.rs +++ b/codex-rs/core/src/conversation_history.rs @@ -24,9 +24,52 @@ impl ConversationHistory { I::Item: std::ops::Deref, { for item in items { - if is_api_message(&item) { - // Note agent-loop.ts also does filtering on some of the fields. - self.items.push(item.clone()); + if !is_api_message(&item) { + continue; + } + + // Merge adjacent assistant messages into a single history entry. + // This prevents duplicates when a partial assistant message was + // streamed into history earlier in the turn and the final full + // message is recorded at turn end. + match (&*item, self.items.last_mut()) { + ( + ResponseItem::Message { + role: new_role, + content: new_content, + .. + }, + Some(ResponseItem::Message { + role: last_role, + content: last_content, + .. + }), + ) if new_role == "assistant" && last_role == "assistant" => { + append_text_content(last_content, new_content); + } + _ => { + self.items.push(item.clone()); + } + } + } + } + + /// Append a text `delta` to the latest assistant message, creating a new + /// assistant entry if none exists yet (e.g. first delta for this turn). + pub(crate) fn append_assistant_text(&mut self, delta: &str) { + match self.items.last_mut() { + Some(ResponseItem::Message { role, content, .. }) if role == "assistant" => { + append_text_delta(content, delta); + } + _ => { + // Start a new assistant message with the delta. + self.items.push(ResponseItem::Message { + id: None, + role: "assistant".to_string(), + content: vec![crate::models::ContentItem::OutputText { + text: delta.to_string(), + }], + }); } } } @@ -72,3 +115,140 @@ fn is_api_message(message: &ResponseItem) -> bool { ResponseItem::Other => false, } } + +/// Helper to append the textual content from `src` into `dst` in place. +fn append_text_content( + dst: &mut Vec, + src: &Vec, +) { + for c in src { + if let crate::models::ContentItem::OutputText { text } = c { + append_text_delta(dst, text); + } + } +} + +/// Append a single text delta to the last OutputText item in `content`, or +/// push a new OutputText item if none exists. +fn append_text_delta(content: &mut Vec, delta: &str) { + if let Some(crate::models::ContentItem::OutputText { text }) = content + .iter_mut() + .rev() + .find(|c| matches!(c, crate::models::ContentItem::OutputText { .. })) + { + text.push_str(delta); + } else { + content.push(crate::models::ContentItem::OutputText { + text: delta.to_string(), + }); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::models::ContentItem; + + fn assistant_msg(text: &str) -> ResponseItem { + ResponseItem::Message { + id: None, + role: "assistant".to_string(), + content: vec![ContentItem::OutputText { + text: text.to_string(), + }], + } + } + + fn user_msg(text: &str) -> ResponseItem { + ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::OutputText { + text: text.to_string(), + }], + } + } + + #[test] + fn merges_adjacent_assistant_messages() { + let mut h = ConversationHistory::default(); + let a1 = assistant_msg("Hello"); + let a2 = assistant_msg(", world!"); + h.record_items([&a1, &a2]); + + let items = h.contents(); + assert_eq!( + items, + vec![ResponseItem::Message { + id: None, + role: "assistant".to_string(), + content: vec![ContentItem::OutputText { + text: "Hello, world!".to_string() + }] + }] + ); + } + + #[test] + fn append_assistant_text_creates_and_appends() { + let mut h = ConversationHistory::default(); + h.append_assistant_text("Hello"); + h.append_assistant_text(", world"); + + // Now record a final full assistant message and verify it merges. + let final_msg = assistant_msg("!"); + h.record_items([&final_msg]); + + let items = h.contents(); + assert_eq!( + items, + vec![ResponseItem::Message { + id: None, + role: "assistant".to_string(), + content: vec![ContentItem::OutputText { + text: "Hello, world!".to_string() + }] + }] + ); + } + + #[test] + fn filters_non_api_messages() { + let mut h = ConversationHistory::default(); + // System message is not an API message; Other is ignored. + let system = ResponseItem::Message { + id: None, + role: "system".to_string(), + content: vec![ContentItem::OutputText { + text: "ignored".to_string(), + }], + }; + h.record_items([&system, &ResponseItem::Other]); + + // User and assistant should be retained. + let u = user_msg("hi"); + let a = assistant_msg("hello"); + h.record_items([&u, &a]); + + let items = h.contents(); + assert_eq!( + items, + vec![ + ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::OutputText { + text: "hi".to_string() + }] + }, + ResponseItem::Message { + id: None, + role: "assistant".to_string(), + content: vec![ContentItem::OutputText { + text: "hello".to_string() + }] + } + ] + ); + } +} diff --git a/codex-rs/core/src/models.rs b/codex-rs/core/src/models.rs index 166404915a..91bfb3bc8c 100644 --- a/codex-rs/core/src/models.rs +++ b/codex-rs/core/src/models.rs @@ -9,7 +9,7 @@ use serde::ser::Serializer; use crate::protocol::InputItem; -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(tag = "type", rename_all = "snake_case")] pub enum ResponseInputItem { Message { @@ -26,7 +26,7 @@ pub enum ResponseInputItem { }, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(tag = "type", rename_all = "snake_case")] pub enum ContentItem { InputText { text: String }, @@ -34,7 +34,7 @@ pub enum ContentItem { OutputText { text: String }, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(tag = "type", rename_all = "snake_case")] pub enum ResponseItem { Message { @@ -107,7 +107,7 @@ impl From for ResponseItem { } } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(rename_all = "snake_case")] pub enum LocalShellStatus { Completed, @@ -115,13 +115,13 @@ pub enum LocalShellStatus { Incomplete, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(tag = "type", rename_all = "snake_case")] pub enum LocalShellAction { Exec(LocalShellExecAction), } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct LocalShellExecAction { pub command: Vec, pub timeout_ms: Option, @@ -130,7 +130,7 @@ pub struct LocalShellExecAction { pub user: Option, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(tag = "type", rename_all = "snake_case")] pub enum ReasoningItemReasoningSummary { SummaryText { text: String }, @@ -185,10 +185,9 @@ pub struct ShellToolCallParams { pub timeout_ms: Option, } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq)] pub struct FunctionCallOutputPayload { pub content: String, - #[expect(dead_code)] pub success: Option, } diff --git a/codex-rs/tui/Cargo.toml b/codex-rs/tui/Cargo.toml index a571b32c8d..60af056a2d 100644 --- a/codex-rs/tui/Cargo.toml +++ b/codex-rs/tui/Cargo.toml @@ -11,6 +11,10 @@ path = "src/main.rs" name = "codex_tui" path = "src/lib.rs" +[features] +# Enable vt100-based tests (emulator) when running with `--features vt100-tests`. +vt100-tests = [] + [lints] workspace = true @@ -73,3 +77,4 @@ insta = "1.43.1" pretty_assertions = "1" rand = "0.8" chrono = { version = "0.4", features = ["serde"] } +vt100 = "0.16.2" diff --git a/codex-rs/tui/src/bottom_pane/live_ring_widget.rs b/codex-rs/tui/src/bottom_pane/live_ring_widget.rs new file mode 100644 index 0000000000..13f91acc5d --- /dev/null +++ b/codex-rs/tui/src/bottom_pane/live_ring_widget.rs @@ -0,0 +1,45 @@ +use ratatui::buffer::Buffer; +use ratatui::layout::Rect; +use ratatui::text::Line; +use ratatui::widgets::Paragraph; +use ratatui::widgets::WidgetRef; + +/// Minimal rendering-only widget for the transient ring rows. +pub(crate) struct LiveRingWidget { + max_rows: u16, + rows: Vec>, // newest at the end +} + +impl LiveRingWidget { + pub fn new() -> Self { + Self { + max_rows: 3, + rows: Vec::new(), + } + } + + pub fn set_max_rows(&mut self, n: u16) { + self.max_rows = n.max(1); + } + + pub fn set_rows(&mut self, rows: Vec>) { + self.rows = rows; + } + + pub fn desired_height(&self, _width: u16) -> u16 { + let len = self.rows.len() as u16; + len.min(self.max_rows) + } +} + +impl WidgetRef for LiveRingWidget { + fn render_ref(&self, area: Rect, buf: &mut Buffer) { + if area.height == 0 { + return; + } + let visible = self.rows.len().saturating_sub(self.max_rows as usize); + let slice = &self.rows[visible..]; + let para = Paragraph::new(slice.to_vec()); + para.render_ref(area, buf); + } +} diff --git a/codex-rs/tui/src/bottom_pane/mod.rs b/codex-rs/tui/src/bottom_pane/mod.rs index cab78bbe3f..fde0b3bde8 100644 --- a/codex-rs/tui/src/bottom_pane/mod.rs +++ b/codex-rs/tui/src/bottom_pane/mod.rs @@ -4,12 +4,12 @@ use crate::app_event::AppEvent; use crate::app_event_sender::AppEventSender; use crate::user_approval_widget::ApprovalRequest; use bottom_pane_view::BottomPaneView; -use bottom_pane_view::ConditionalUpdate; use codex_core::protocol::TokenUsage; use codex_file_search::FileMatch; use crossterm::event::KeyEvent; use ratatui::buffer::Buffer; use ratatui::layout::Rect; +use ratatui::text::Line; use ratatui::widgets::WidgetRef; mod approval_modal_view; @@ -18,6 +18,7 @@ mod chat_composer; mod chat_composer_history; mod command_popup; mod file_search_popup; +mod live_ring_widget; mod status_indicator_view; mod textarea; @@ -30,6 +31,7 @@ pub(crate) enum CancellationEvent { pub(crate) use chat_composer::ChatComposer; pub(crate) use chat_composer::InputResult; +use crate::status_indicator_widget::StatusIndicatorWidget; use approval_modal_view::ApprovalModalView; use status_indicator_view::StatusIndicatorView; @@ -46,6 +48,19 @@ pub(crate) struct BottomPane<'a> { has_input_focus: bool, is_task_running: bool, ctrl_c_quit_hint: bool, + + /// Optional live, multi‑line status/"live cell" rendered directly above + /// the composer while a task is running. Unlike `active_view`, this does + /// not replace the composer; it augments it. + live_status: Option, + + /// Optional transient ring shown above the composer. This is a rendering-only + /// container used during development before we wire it to ChatWidget events. + live_ring: Option, + + /// True if the active view is the StatusIndicatorView that replaces the + /// composer during a running task. + status_view_active: bool, } pub(crate) struct BottomPaneParams { @@ -55,6 +70,7 @@ pub(crate) struct BottomPaneParams { } impl BottomPane<'_> { + const BOTTOM_PAD_LINES: u16 = 2; pub fn new(params: BottomPaneParams) -> Self { let enhanced_keys_supported = params.enhanced_keys_supported; Self { @@ -68,14 +84,40 @@ impl BottomPane<'_> { has_input_focus: params.has_input_focus, is_task_running: false, ctrl_c_quit_hint: false, + live_status: None, + live_ring: None, + status_view_active: false, } } pub fn desired_height(&self, width: u16) -> u16 { - self.active_view + let overlay_status_h = self + .live_status .as_ref() - .map(|v| v.desired_height(width)) - .unwrap_or(self.composer.desired_height(width)) + .map(|s| s.desired_height(width)) + .unwrap_or(0); + let ring_h = self + .live_ring + .as_ref() + .map(|r| r.desired_height(width)) + .unwrap_or(0); + + let view_height = if let Some(view) = self.active_view.as_ref() { + // Add a single blank spacer line between live ring and status view when active. + let spacer = if self.live_ring.is_some() && self.status_view_active { + 1 + } else { + 0 + }; + spacer + view.desired_height(width) + } else { + self.composer.desired_height(width) + }; + + overlay_status_h + .saturating_add(ring_h) + .saturating_add(view_height) + .saturating_add(Self::BOTTOM_PAD_LINES) } pub fn cursor_pos(&self, area: Rect) -> Option<(u16, u16)> { @@ -96,10 +138,6 @@ impl BottomPane<'_> { view.handle_key_event(self, key_event); if !view.is_complete() { self.active_view = Some(view); - } else if self.is_task_running { - self.active_view = Some(Box::new(StatusIndicatorView::new( - self.app_event_tx.clone(), - ))); } self.request_redraw(); InputResult::None @@ -125,10 +163,6 @@ impl BottomPane<'_> { CancellationEvent::Handled => { if !view.is_complete() { self.active_view = Some(view); - } else if self.is_task_running { - self.active_view = Some(Box::new(StatusIndicatorView::new( - self.app_event_tx.clone(), - ))); } self.show_ctrl_c_quit_hint(); } @@ -148,19 +182,37 @@ impl BottomPane<'_> { } } - /// Update the status indicator text (only when the `StatusIndicatorView` is - /// active). + /// Update the status indicator text. Prefer replacing the composer with + /// the StatusIndicatorView so the input pane shows a single-line status + /// like: `▌ Working waiting for model`. pub(crate) fn update_status_text(&mut self, text: String) { - if let Some(view) = &mut self.active_view { - match view.update_status_text(text) { - ConditionalUpdate::NeedsRedraw => { - self.request_redraw(); - } - ConditionalUpdate::NoRedraw => { - // No redraw needed. - } + let mut handled_by_view = false; + if let Some(view) = self.active_view.as_mut() { + if matches!( + view.update_status_text(text.clone()), + bottom_pane_view::ConditionalUpdate::NeedsRedraw + ) { + handled_by_view = true; + } + } else { + let mut v = StatusIndicatorView::new(self.app_event_tx.clone()); + v.update_text(text.clone()); + self.active_view = Some(Box::new(v)); + self.status_view_active = true; + handled_by_view = true; + } + + // Fallback: if the current active view did not consume status updates, + // present an overlay above the composer. + if !handled_by_view { + if self.live_status.is_none() { + self.live_status = Some(StatusIndicatorWidget::new(self.app_event_tx.clone())); + } + if let Some(status) = &mut self.live_status { + status.update_text(text); } } + self.request_redraw(); } pub(crate) fn show_ctrl_c_quit_hint(&mut self) { @@ -186,27 +238,23 @@ impl BottomPane<'_> { pub fn set_task_running(&mut self, running: bool) { self.is_task_running = running; - match (running, self.active_view.is_some()) { - (true, false) => { - // Show status indicator overlay. + if running { + if self.active_view.is_none() { self.active_view = Some(Box::new(StatusIndicatorView::new( self.app_event_tx.clone(), ))); - self.request_redraw(); + self.status_view_active = true; } - (false, true) => { - if let Some(mut view) = self.active_view.take() { - if view.should_hide_when_task_is_done() { - // Leave self.active_view as None. - self.request_redraw(); - } else { - // Preserve the view. - self.active_view = Some(view); - } + self.request_redraw(); + } else { + self.live_status = None; + // Drop the status view when a task completes, but keep other + // modal views (e.g. approval dialogs). + if let Some(mut view) = self.active_view.take() { + if !view.should_hide_when_task_is_done() { + self.active_view = Some(view); } - } - _ => { - // No change. + self.status_view_active = false; } } } @@ -248,6 +296,7 @@ impl BottomPane<'_> { // Otherwise create a new approval modal overlay. let modal = ApprovalModalView::new(request, self.app_event_tx.clone()); self.active_view = Some(Box::new(modal)); + self.status_view_active = false; self.request_redraw() } @@ -281,15 +330,80 @@ impl BottomPane<'_> { self.composer.on_file_search_result(query, matches); self.request_redraw(); } + + /// Set the rows and cap for the transient live ring overlay. + pub(crate) fn set_live_ring_rows(&mut self, max_rows: u16, rows: Vec>) { + let mut w = live_ring_widget::LiveRingWidget::new(); + w.set_max_rows(max_rows); + w.set_rows(rows); + self.live_ring = Some(w); + } + + pub(crate) fn clear_live_ring(&mut self) { + self.live_ring = None; + } + + // Removed restart_live_status_with_text – no longer used by the current streaming UI. } impl WidgetRef for &BottomPane<'_> { fn render_ref(&self, area: Rect, buf: &mut Buffer) { - // Show BottomPaneView if present. - if let Some(ov) = &self.active_view { - ov.render(area, buf); - } else { - (&self.composer).render_ref(area, buf); + let mut y_offset = 0u16; + if let Some(ring) = &self.live_ring { + let live_h = ring.desired_height(area.width).min(area.height); + if live_h > 0 { + let live_rect = Rect { + x: area.x, + y: area.y, + width: area.width, + height: live_h, + }; + ring.render_ref(live_rect, buf); + y_offset = live_h; + } + } + // Spacer between live ring and status view when active + if self.live_ring.is_some() && self.status_view_active && y_offset < area.height { + // Leave one empty line + y_offset = y_offset.saturating_add(1); + } + if let Some(status) = &self.live_status { + let live_h = status.desired_height(area.width).min(area.height); + if live_h > 0 { + let live_rect = Rect { + x: area.x, + y: area.y, + width: area.width, + height: live_h, + }; + status.render_ref(live_rect, buf); + y_offset = live_h; + } + } + + if let Some(view) = &self.active_view { + if y_offset < area.height { + // Reserve bottom padding lines; keep at least 1 line for the view. + let avail = area.height - y_offset; + let pad = BottomPane::BOTTOM_PAD_LINES.min(avail.saturating_sub(1)); + let view_rect = Rect { + x: area.x, + y: area.y + y_offset, + width: area.width, + height: avail - pad, + }; + view.render(view_rect, buf); + } + } else if y_offset < area.height { + let composer_rect = Rect { + x: area.x, + y: area.y + y_offset, + width: area.width, + // Reserve bottom padding + height: (area.height - y_offset) + - BottomPane::BOTTOM_PAD_LINES.min((area.height - y_offset).saturating_sub(1)), + }; + (&self.composer).render_ref(composer_rect, buf); } } } @@ -298,6 +412,9 @@ impl WidgetRef for &BottomPane<'_> { mod tests { use super::*; use crate::app_event::AppEvent; + use ratatui::buffer::Buffer; + use ratatui::layout::Rect; + use ratatui::text::Line; use std::path::PathBuf; use std::sync::mpsc::channel; @@ -324,4 +441,200 @@ mod tests { assert!(pane.ctrl_c_quit_hint_visible()); assert_eq!(CancellationEvent::Ignored, pane.on_ctrl_c()); } + + #[test] + fn live_ring_renders_above_composer() { + let (tx_raw, _rx) = channel::(); + let tx = AppEventSender::new(tx_raw); + let mut pane = BottomPane::new(BottomPaneParams { + app_event_tx: tx, + has_input_focus: true, + enhanced_keys_supported: false, + }); + + // Provide 4 rows with max_rows=3; only the last 3 should be visible. + pane.set_live_ring_rows( + 3, + vec![ + Line::from("one".to_string()), + Line::from("two".to_string()), + Line::from("three".to_string()), + Line::from("four".to_string()), + ], + ); + + let area = Rect::new(0, 0, 10, 5); + let mut buf = Buffer::empty(area); + (&pane).render_ref(area, &mut buf); + + // Extract the first 3 rows and assert they contain the last three lines. + let mut lines: Vec = Vec::new(); + for y in 0..3 { + let mut s = String::new(); + for x in 0..area.width { + s.push(buf[(x, y)].symbol().chars().next().unwrap_or(' ')); + } + lines.push(s.trim_end().to_string()); + } + assert_eq!(lines, vec!["two", "three", "four"]); + } + + #[test] + fn status_indicator_visible_with_live_ring() { + let (tx_raw, _rx) = channel::(); + let tx = AppEventSender::new(tx_raw); + let mut pane = BottomPane::new(BottomPaneParams { + app_event_tx: tx, + has_input_focus: true, + enhanced_keys_supported: false, + }); + + // Simulate task running which replaces composer with the status indicator. + pane.set_task_running(true); + pane.update_status_text("waiting for model".to_string()); + + // Provide 2 rows in the live ring (e.g., streaming CoT) and ensure the + // status indicator remains visible below them. + pane.set_live_ring_rows( + 2, + vec![ + Line::from("cot1".to_string()), + Line::from("cot2".to_string()), + ], + ); + + // Allow some frames so the dot animation is present. + std::thread::sleep(std::time::Duration::from_millis(120)); + + // Height should include both ring rows, 1 spacer, and the 1-line status. + let area = Rect::new(0, 0, 30, 4); + let mut buf = Buffer::empty(area); + (&pane).render_ref(area, &mut buf); + + // Top two rows are the live ring. + let mut r0 = String::new(); + let mut r1 = String::new(); + for x in 0..area.width { + r0.push(buf[(x, 0)].symbol().chars().next().unwrap_or(' ')); + r1.push(buf[(x, 1)].symbol().chars().next().unwrap_or(' ')); + } + assert!(r0.contains("cot1"), "expected first live row: {r0:?}"); + assert!(r1.contains("cot2"), "expected second live row: {r1:?}"); + + // Row 2 is the spacer (blank) + let mut r2 = String::new(); + for x in 0..area.width { + r2.push(buf[(x, 2)].symbol().chars().next().unwrap_or(' ')); + } + assert!(r2.trim().is_empty(), "expected blank spacer line: {r2:?}"); + + // Bottom row is the status line; it should contain the left bar and "Working". + let mut r3 = String::new(); + for x in 0..area.width { + r3.push(buf[(x, 3)].symbol().chars().next().unwrap_or(' ')); + } + assert_eq!(buf[(0, 3)].symbol().chars().next().unwrap_or(' '), '▌'); + assert!( + r3.contains("Working"), + "expected Working header in status line: {r3:?}" + ); + } + + #[test] + fn bottom_padding_present_for_status_view() { + let (tx_raw, _rx) = channel::(); + let tx = AppEventSender::new(tx_raw); + let mut pane = BottomPane::new(BottomPaneParams { + app_event_tx: tx, + has_input_focus: true, + enhanced_keys_supported: false, + }); + + // Activate spinner (status view replaces composer) with no live ring. + pane.set_task_running(true); + pane.update_status_text("waiting for model".to_string()); + + // Use height == desired_height; expect 1 status row at top and 2 bottom padding rows. + let height = pane.desired_height(30); + assert!( + height >= 3, + "expected at least 3 rows with bottom padding; got {height}" + ); + let area = Rect::new(0, 0, 30, height); + let mut buf = Buffer::empty(area); + (&pane).render_ref(area, &mut buf); + + // Top row contains the status header + let mut top = String::new(); + for x in 0..area.width { + top.push(buf[(x, 0)].symbol().chars().next().unwrap_or(' ')); + } + assert_eq!(buf[(0, 0)].symbol().chars().next().unwrap_or(' '), '▌'); + assert!( + top.contains("Working"), + "expected Working header on top row: {top:?}" + ); + + // Bottom two rows are blank padding + let mut r_last = String::new(); + let mut r_last2 = String::new(); + for x in 0..area.width { + r_last.push(buf[(x, height - 1)].symbol().chars().next().unwrap_or(' ')); + r_last2.push(buf[(x, height - 2)].symbol().chars().next().unwrap_or(' ')); + } + assert!( + r_last.trim().is_empty(), + "expected last row blank: {r_last:?}" + ); + assert!( + r_last2.trim().is_empty(), + "expected second-to-last row blank: {r_last2:?}" + ); + } + + #[test] + fn bottom_padding_shrinks_when_tiny() { + let (tx_raw, _rx) = channel::(); + let tx = AppEventSender::new(tx_raw); + let mut pane = BottomPane::new(BottomPaneParams { + app_event_tx: tx, + has_input_focus: true, + enhanced_keys_supported: false, + }); + + pane.set_task_running(true); + pane.update_status_text("waiting for model".to_string()); + + // Height=2 → pad shrinks to 1; bottom row is blank, top row has spinner. + let area2 = Rect::new(0, 0, 20, 2); + let mut buf2 = Buffer::empty(area2); + (&pane).render_ref(area2, &mut buf2); + let mut row0 = String::new(); + let mut row1 = String::new(); + for x in 0..area2.width { + row0.push(buf2[(x, 0)].symbol().chars().next().unwrap_or(' ')); + row1.push(buf2[(x, 1)].symbol().chars().next().unwrap_or(' ')); + } + assert!( + row0.contains("Working"), + "expected Working header on row 0: {row0:?}" + ); + assert!( + row1.trim().is_empty(), + "expected bottom padding on row 1: {row1:?}" + ); + + // Height=1 → no padding; single row is the spinner. + let area1 = Rect::new(0, 0, 20, 1); + let mut buf1 = Buffer::empty(area1); + (&pane).render_ref(area1, &mut buf1); + let mut only = String::new(); + for x in 0..area1.width { + only.push(buf1[(x, 0)].symbol().chars().next().unwrap_or(' ')); + } + assert!( + only.contains("Working"), + "expected Working header with no padding: {only:?}" + ); + } } diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index e5ebf58a07..f63810b62a 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -42,8 +42,10 @@ use crate::exec_command::strip_bash_lc_and_escape; use crate::history_cell::CommandOutput; use crate::history_cell::HistoryCell; use crate::history_cell::PatchEventType; +use crate::live_wrap::RowBuilder; use crate::user_approval_widget::ApprovalRequest; use codex_file_search::FileMatch; +use ratatui::style::Stylize; struct RunningCommand { command: Vec, @@ -64,6 +66,10 @@ pub(crate) struct ChatWidget<'a> { // at once into scrollback so the history contains a single message. answer_buffer: String, running_commands: HashMap, + live_builder: RowBuilder, + current_stream: Option, + stream_header_emitted: bool, + live_max_rows: u16, } struct UserMessage { @@ -71,6 +77,12 @@ struct UserMessage { image_paths: Vec, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum StreamKind { + Answer, + Reasoning, +} + impl From for UserMessage { fn from(text: String) -> Self { Self { @@ -151,6 +163,10 @@ impl ChatWidget<'_> { reasoning_buffer: String::new(), answer_buffer: String::new(), running_commands: HashMap::new(), + live_builder: RowBuilder::new(80), + current_stream: None, + stream_header_emitted: false, + live_max_rows: 3, } } @@ -234,58 +250,45 @@ impl ChatWidget<'_> { self.request_redraw(); } - EventMsg::AgentMessage(AgentMessageEvent { message }) => { - // Final assistant answer. Prefer the fully provided message - // from the event; if it is empty fall back to any accumulated - // delta buffer (some providers may only stream deltas and send - // an empty final message). - let full = if message.is_empty() { - std::mem::take(&mut self.answer_buffer) - } else { - self.answer_buffer.clear(); - message - }; - if !full.is_empty() { - self.add_to_history(HistoryCell::new_agent_message(&self.config, full)); - } + EventMsg::AgentMessage(AgentMessageEvent { message: _ }) => { + // Final assistant answer: commit all remaining rows and close with + // a blank line. Use the final text if provided, otherwise rely on + // streamed deltas already in the builder. + self.finalize_stream(StreamKind::Answer); self.request_redraw(); } EventMsg::AgentMessageDelta(AgentMessageDeltaEvent { delta }) => { - // Buffer only – do not emit partial lines. This avoids cases - // where long responses appear truncated if the terminal - // wrapped early. The full message is emitted on - // AgentMessage. + self.begin_stream(StreamKind::Answer); self.answer_buffer.push_str(&delta); + self.stream_push_and_maybe_commit(&delta); + self.request_redraw(); } EventMsg::AgentReasoningDelta(AgentReasoningDeltaEvent { delta }) => { - // Buffer only – disable incremental reasoning streaming so we - // avoid truncated intermediate lines. Full text emitted on - // AgentReasoning. + // Stream CoT into the live pane; keep input visible and commit + // overflow rows incrementally to scrollback. + self.begin_stream(StreamKind::Reasoning); self.reasoning_buffer.push_str(&delta); + self.stream_push_and_maybe_commit(&delta); + self.request_redraw(); } - EventMsg::AgentReasoning(AgentReasoningEvent { text }) => { - // Emit full reasoning text once. Some providers might send - // final event with empty text if only deltas were used. - let full = if text.is_empty() { - std::mem::take(&mut self.reasoning_buffer) - } else { - self.reasoning_buffer.clear(); - text - }; - if !full.is_empty() { - self.add_to_history(HistoryCell::new_agent_reasoning(&self.config, full)); - } + EventMsg::AgentReasoning(AgentReasoningEvent { text: _ }) => { + // Final reasoning: commit remaining rows and close with a blank. + self.finalize_stream(StreamKind::Reasoning); self.request_redraw(); } EventMsg::TaskStarted => { self.bottom_pane.clear_ctrl_c_quit_hint(); self.bottom_pane.set_task_running(true); + // Replace composer with single-line spinner while waiting. + self.bottom_pane + .update_status_text("waiting for model".to_string()); self.request_redraw(); } EventMsg::TaskComplete(TaskCompleteEvent { last_agent_message: _, }) => { self.bottom_pane.set_task_running(false); + self.bottom_pane.clear_live_ring(); self.request_redraw(); } EventMsg::TokenCount(token_usage) => { @@ -298,8 +301,8 @@ impl ChatWidget<'_> { self.bottom_pane.set_task_running(false); } EventMsg::PlanUpdate(update) => { + // Commit plan updates directly to history (no status-line preview). self.add_to_history(HistoryCell::new_plan_update(update)); - self.request_redraw(); } EventMsg::ExecApprovalRequest(ExecApprovalRequestEvent { call_id: _, @@ -307,8 +310,7 @@ impl ChatWidget<'_> { cwd, reason, }) => { - // Print the command to the history so it is visible in the - // transcript *before* the modal asks for approval. + // Log a background summary immediately so the history is chronological. let cmdline = strip_bash_lc_and_escape(&command); let text = format!( "command requires approval:\n$ {cmdline}{reason}", @@ -344,7 +346,6 @@ impl ChatWidget<'_> { // approval dialog) and avoids surprising the user with a modal // prompt before they have seen *what* is being requested. // ------------------------------------------------------------------ - self.add_to_history(HistoryCell::new_patch_event( PatchEventType::ApprovalRequest, changes, @@ -379,8 +380,6 @@ impl ChatWidget<'_> { auto_approved, changes, }) => { - // Even when a patch is auto‑approved we still display the - // summary so the user can follow along. self.add_to_history(HistoryCell::new_patch_event( PatchEventType::ApplyBegin { auto_approved }, changes, @@ -393,6 +392,7 @@ impl ChatWidget<'_> { stdout, stderr, }) => { + // Compute summary before moving stdout into the history cell. let cmd = self.running_commands.remove(&call_id); self.add_to_history(HistoryCell::new_completed_exec_command( cmd.map(|cmd| cmd.command).unwrap_or_else(|| vec![call_id]), @@ -442,14 +442,15 @@ impl ChatWidget<'_> { self.app_event_tx.send(AppEvent::ExitRequest); } event => { - self.add_to_history(HistoryCell::new_background_event(format!("{event:?}"))); + let text = format!("{event:?}"); + self.add_to_history(HistoryCell::new_background_event(text.clone())); + self.update_latest_log(text); } } } /// Update the live log preview while a task is running. pub(crate) fn update_latest_log(&mut self, line: String) { - // Forward only if we are currently showing the status indicator. self.bottom_pane.update_status_text(line); } @@ -515,6 +516,97 @@ impl ChatWidget<'_> { } } +impl ChatWidget<'_> { + fn begin_stream(&mut self, kind: StreamKind) { + if self.current_stream != Some(kind) { + self.current_stream = Some(kind); + self.stream_header_emitted = false; + // Clear any previous live content; we're starting a new stream. + self.live_builder = RowBuilder::new(self.live_builder.width()); + // Ensure the waiting status is visible (composer replaced). + self.bottom_pane + .update_status_text("waiting for model".to_string()); + } + } + + fn stream_push_and_maybe_commit(&mut self, delta: &str) { + self.live_builder.push_fragment(delta); + + // Commit overflow rows (small batches) while keeping the last N rows visible. + let drained = self + .live_builder + .drain_commit_ready(self.live_max_rows as usize); + if !drained.is_empty() { + let mut lines: Vec> = Vec::new(); + if !self.stream_header_emitted { + match self.current_stream { + Some(StreamKind::Reasoning) => { + lines.push(ratatui::text::Line::from("thinking".magenta().italic())); + } + Some(StreamKind::Answer) => { + lines.push(ratatui::text::Line::from("codex".magenta().bold())); + } + None => {} + } + self.stream_header_emitted = true; + } + for r in drained { + lines.push(ratatui::text::Line::from(r.text)); + } + self.app_event_tx.send(AppEvent::InsertHistory(lines)); + } + + // Update the live ring overlay lines (text-only, newest at bottom). + let rows = self + .live_builder + .display_rows() + .into_iter() + .map(|r| ratatui::text::Line::from(r.text)) + .collect::>(); + self.bottom_pane + .set_live_ring_rows(self.live_max_rows, rows); + } + + fn finalize_stream(&mut self, kind: StreamKind) { + if self.current_stream != Some(kind) { + // Nothing to do; either already finalized or not the active stream. + return; + } + // Flush any partial line as a full row, then drain all remaining rows. + self.live_builder.end_line(); + let remaining = self.live_builder.drain_rows(); + // TODO: Re-add markdown rendering for assistant answers and reasoning. + // When finalizing, pass the accumulated text through `markdown::append_markdown` + // to build styled `Line<'static>` entries instead of raw plain text lines. + if !remaining.is_empty() || !self.stream_header_emitted { + let mut lines: Vec> = Vec::new(); + if !self.stream_header_emitted { + match kind { + StreamKind::Reasoning => { + lines.push(ratatui::text::Line::from("thinking".magenta().italic())); + } + StreamKind::Answer => { + lines.push(ratatui::text::Line::from("codex".magenta().bold())); + } + } + self.stream_header_emitted = true; + } + for r in remaining { + lines.push(ratatui::text::Line::from(r.text)); + } + // Close the block with a blank line for readability. + lines.push(ratatui::text::Line::from("")); + self.app_event_tx.send(AppEvent::InsertHistory(lines)); + } + + // Clear the live overlay and reset state for the next stream. + self.live_builder = RowBuilder::new(self.live_builder.width()); + self.bottom_pane.clear_live_ring(); + self.current_stream = None; + self.stream_header_emitted = false; + } +} + impl WidgetRef for &ChatWidget<'_> { fn render_ref(&self, area: Rect, buf: &mut Buffer) { // In the hybrid inline viewport mode we only draw the interactive diff --git a/codex-rs/tui/src/history_cell.rs b/codex-rs/tui/src/history_cell.rs index c2aafdd522..17f0e683c0 100644 --- a/codex-rs/tui/src/history_cell.rs +++ b/codex-rs/tui/src/history_cell.rs @@ -1,5 +1,4 @@ use crate::exec_command::strip_bash_lc_and_escape; -use crate::markdown::append_markdown; use crate::text_block::TextBlock; use crate::text_formatting::format_and_truncate_tool_result; use base64::Engine; @@ -68,12 +67,7 @@ pub(crate) enum HistoryCell { /// Message from the user. UserPrompt { view: TextBlock }, - /// Message from the agent. - AgentMessage { view: TextBlock }, - - /// Reasoning event from the agent. - AgentReasoning { view: TextBlock }, - + // AgentMessage and AgentReasoning variants were unused and have been removed. /// An exec tool call that has not finished yet. ActiveExecCommand { view: TextBlock }, @@ -128,8 +122,6 @@ impl HistoryCell { match self { HistoryCell::WelcomeMessage { view } | HistoryCell::UserPrompt { view } - | HistoryCell::AgentMessage { view } - | HistoryCell::AgentReasoning { view } | HistoryCell::BackgroundEvent { view } | HistoryCell::GitDiffOutput { view } | HistoryCell::ErrorEvent { view } @@ -231,28 +223,6 @@ impl HistoryCell { } } - pub(crate) fn new_agent_message(config: &Config, message: String) -> Self { - let mut lines: Vec> = Vec::new(); - lines.push(Line::from("codex".magenta().bold())); - append_markdown(&message, &mut lines, config); - lines.push(Line::from("")); - - HistoryCell::AgentMessage { - view: TextBlock::new(lines), - } - } - - pub(crate) fn new_agent_reasoning(config: &Config, text: String) -> Self { - let mut lines: Vec> = Vec::new(); - lines.push(Line::from("thinking".magenta().italic())); - append_markdown(&text, &mut lines, config); - lines.push(Line::from("")); - - HistoryCell::AgentReasoning { - view: TextBlock::new(lines), - } - } - pub(crate) fn new_active_exec_command(command: Vec) -> Self { let command_escaped = strip_bash_lc_and_escape(&command); diff --git a/codex-rs/tui/src/insert_history.rs b/codex-rs/tui/src/insert_history.rs index 87d88b7f03..5c316637b1 100644 --- a/codex-rs/tui/src/insert_history.rs +++ b/codex-rs/tui/src/insert_history.rs @@ -14,7 +14,6 @@ use crossterm::style::SetBackgroundColor; use crossterm::style::SetColors; use crossterm::style::SetForegroundColor; use ratatui::layout::Size; -use ratatui::prelude::Backend; use ratatui::style::Color; use ratatui::style::Modifier; use ratatui::text::Line; @@ -22,6 +21,20 @@ use ratatui::text::Span; /// Insert `lines` above the viewport. pub(crate) fn insert_history_lines(terminal: &mut tui::Tui, lines: Vec) { + let mut out = std::io::stdout(); + insert_history_lines_to_writer(terminal, &mut out, lines); +} + +/// Like `insert_history_lines`, but writes ANSI to the provided writer. This +/// is intended for testing where a capture buffer is used instead of stdout. +pub fn insert_history_lines_to_writer( + terminal: &mut crate::custom_terminal::Terminal, + writer: &mut W, + lines: Vec, +) where + B: ratatui::backend::Backend, + W: Write, +{ let screen_size = terminal.backend().size().unwrap_or(Size::new(0, 0)); let cursor_pos = terminal.get_cursor_position().ok(); @@ -32,10 +45,22 @@ pub(crate) fn insert_history_lines(terminal: &mut tui::Tui, lines: Vec) { // If the viewport is not at the bottom of the screen, scroll it down to make room. // Don't scroll it past the bottom of the screen. let scroll_amount = wrapped_lines.min(screen_size.height - area.bottom()); - terminal - .backend_mut() - .scroll_region_down(area.top()..screen_size.height, scroll_amount) - .ok(); + + // Emit ANSI to scroll the lower region (from the top of the viewport to the bottom + // of the screen) downward by `scroll_amount` lines. We do this by: + // 1) Limiting the scroll region to [area.top()+1 .. screen_height] (1-based bounds) + // 2) Placing the cursor at the top margin of that region + // 3) Emitting Reverse Index (RI, ESC M) `scroll_amount` times + // 4) Resetting the scroll region back to full screen + let top_1based = area.top() + 1; // Convert 0-based row to 1-based for DECSTBM + queue!(writer, SetScrollRegion(top_1based..screen_size.height)).ok(); + queue!(writer, MoveTo(0, area.top())).ok(); + for _ in 0..scroll_amount { + // Reverse Index (RI): ESC M + queue!(writer, Print("\x1bM")).ok(); + } + queue!(writer, ResetScrollRegion).ok(); + let cursor_top = area.top().saturating_sub(1); area.y += scroll_amount; terminal.set_viewport_area(area); @@ -59,23 +84,23 @@ pub(crate) fn insert_history_lines(terminal: &mut tui::Tui, lines: Vec) { // ││ ││ // │╰────────────────────────────╯│ // └──────────────────────────────┘ - queue!(std::io::stdout(), SetScrollRegion(1..area.top())).ok(); + queue!(writer, SetScrollRegion(1..area.top())).ok(); // NB: we are using MoveTo instead of set_cursor_position here to avoid messing with the // terminal's last_known_cursor_position, which hopefully will still be accurate after we // fetch/restore the cursor position. insert_history_lines should be cursor-position-neutral :) - queue!(std::io::stdout(), MoveTo(0, cursor_top)).ok(); + queue!(writer, MoveTo(0, cursor_top)).ok(); for line in lines { - queue!(std::io::stdout(), Print("\r\n")).ok(); - write_spans(&mut std::io::stdout(), line.iter()).ok(); + queue!(writer, Print("\r\n")).ok(); + write_spans(writer, line.iter()).ok(); } - queue!(std::io::stdout(), ResetScrollRegion).ok(); + queue!(writer, ResetScrollRegion).ok(); // Restore the cursor position to where it was before we started. if let Some(cursor_pos) = cursor_pos { - queue!(std::io::stdout(), MoveTo(cursor_pos.x, cursor_pos.y)).ok(); + queue!(writer, MoveTo(cursor_pos.x, cursor_pos.y)).ok(); } } @@ -88,19 +113,25 @@ fn wrapped_line_count(lines: &[Line], width: u16) -> u16 { } fn line_height(line: &Line, width: u16) -> u16 { - use unicode_width::UnicodeWidthStr; - // get the total display width of the line, accounting for double-width chars - let total_width = line + // Use the same visible-width slicing semantics as the live row builder so + // our pre-scroll estimation matches how rows will actually wrap. + let w = width.max(1) as usize; + let mut rows = 0u16; + let mut remaining = line .spans .iter() - .map(|span| span.content.width()) - .sum::(); - // divide by width to get the number of lines, rounding up - if width == 0 { - 1 - } else { - (total_width as u16).div_ceil(width).max(1) + .map(|s| s.content.as_ref()) + .collect::>() + .join(""); + while !remaining.is_empty() { + let (_prefix, suffix, taken) = crate::live_wrap::take_prefix_by_width(&remaining, w); + rows = rows.saturating_add(1); + if taken >= remaining.len() { + break; + } + remaining = suffix.to_string(); } + rows.max(1) } #[derive(Debug, Clone, PartialEq, Eq)] @@ -283,4 +314,12 @@ mod tests { String::from_utf8(expected).unwrap() ); } + + #[test] + fn line_height_counts_double_width_emoji() { + let line = Line::from("😀😀😀"); // each emoji ~ width 2 + assert_eq!(line_height(&line, 4), 2); + assert_eq!(line_height(&line, 2), 3); + assert_eq!(line_height(&line, 6), 1); + } } diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index 0ec9be6153..c619ce8ff0 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -25,13 +25,14 @@ mod bottom_pane; mod chatwidget; mod citation_regex; mod cli; -mod custom_terminal; +pub mod custom_terminal; mod exec_command; mod file_search; mod get_git_diff; mod git_warning_screen; mod history_cell; -mod insert_history; +pub mod insert_history; +pub mod live_wrap; mod log_layer; mod markdown; mod slash_command; diff --git a/codex-rs/tui/src/live_wrap.rs b/codex-rs/tui/src/live_wrap.rs new file mode 100644 index 0000000000..e78710dc6c --- /dev/null +++ b/codex-rs/tui/src/live_wrap.rs @@ -0,0 +1,290 @@ +use unicode_width::UnicodeWidthChar; +use unicode_width::UnicodeWidthStr; + +/// A single visual row produced by RowBuilder. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Row { + pub text: String, + /// True if this row ends with an explicit line break (as opposed to a hard wrap). + pub explicit_break: bool, +} + +impl Row { + pub fn width(&self) -> usize { + self.text.width() + } +} + +/// Incrementally wraps input text into visual rows of at most `width` cells. +/// +/// Step 1: plain-text only. ANSI-carry and styled spans will be added later. +pub struct RowBuilder { + target_width: usize, + /// Buffer for the current logical line (until a '\n' is seen). + current_line: String, + /// Output rows built so far for the current logical line and previous ones. + rows: Vec, +} + +impl RowBuilder { + pub fn new(target_width: usize) -> Self { + Self { + target_width: target_width.max(1), + current_line: String::new(), + rows: Vec::new(), + } + } + + pub fn width(&self) -> usize { + self.target_width + } + + pub fn set_width(&mut self, width: usize) { + self.target_width = width.max(1); + // Rewrap everything we have (simple approach for Step 1). + let mut all = String::new(); + for row in self.rows.drain(..) { + all.push_str(&row.text); + if row.explicit_break { + all.push('\n'); + } + } + all.push_str(&self.current_line); + self.current_line.clear(); + self.push_fragment(&all); + } + + /// Push an input fragment. May contain newlines. + pub fn push_fragment(&mut self, fragment: &str) { + if fragment.is_empty() { + return; + } + let mut start = 0usize; + for (i, ch) in fragment.char_indices() { + if ch == '\n' { + // Flush anything pending before the newline. + if start < i { + self.current_line.push_str(&fragment[start..i]); + } + self.flush_current_line(true); + start = i + ch.len_utf8(); + } + } + if start < fragment.len() { + self.current_line.push_str(&fragment[start..]); + self.wrap_current_line(); + } + } + + /// Mark the end of the current logical line (equivalent to pushing a '\n'). + pub fn end_line(&mut self) { + self.flush_current_line(true); + } + + /// Drain and return all produced rows. + pub fn drain_rows(&mut self) -> Vec { + std::mem::take(&mut self.rows) + } + + /// Return a snapshot of produced rows (non-draining). + pub fn rows(&self) -> &[Row] { + &self.rows + } + + /// Rows suitable for display, including the current partial line if any. + pub fn display_rows(&self) -> Vec { + let mut out = self.rows.clone(); + if !self.current_line.is_empty() { + out.push(Row { + text: self.current_line.clone(), + explicit_break: false, + }); + } + out + } + + /// Drain the oldest rows that exceed `max_keep` display rows (including the + /// current partial line, if any). Returns the drained rows in order. + pub fn drain_commit_ready(&mut self, max_keep: usize) -> Vec { + let display_count = self.rows.len() + if self.current_line.is_empty() { 0 } else { 1 }; + if display_count <= max_keep { + return Vec::new(); + } + let to_commit = display_count - max_keep; + let commit_count = to_commit.min(self.rows.len()); + let mut drained = Vec::with_capacity(commit_count); + for _ in 0..commit_count { + drained.push(self.rows.remove(0)); + } + drained + } + + fn flush_current_line(&mut self, explicit_break: bool) { + // Wrap any remaining content in the current line and then finalize with explicit_break. + self.wrap_current_line(); + // If the current line ended exactly on a width boundary and is non-empty, represent + // the explicit break as an empty explicit row so that fragmentation invariance holds. + if explicit_break { + if self.current_line.is_empty() { + // We ended on a boundary previously; add an empty explicit row. + self.rows.push(Row { + text: String::new(), + explicit_break: true, + }); + } else { + // There is leftover content that did not wrap yet; push it now with the explicit flag. + let mut s = String::new(); + std::mem::swap(&mut s, &mut self.current_line); + self.rows.push(Row { + text: s, + explicit_break: true, + }); + } + } + // Reset current line buffer for next logical line. + self.current_line.clear(); + } + + fn wrap_current_line(&mut self) { + // While the current_line exceeds width, cut a prefix. + loop { + if self.current_line.is_empty() { + break; + } + let (prefix, suffix, taken) = + take_prefix_by_width(&self.current_line, self.target_width); + if taken == 0 { + // Avoid infinite loop on pathological inputs; take one scalar and continue. + if let Some((i, ch)) = self.current_line.char_indices().next() { + let len = i + ch.len_utf8(); + let p = self.current_line[..len].to_string(); + self.rows.push(Row { + text: p, + explicit_break: false, + }); + self.current_line = self.current_line[len..].to_string(); + continue; + } + break; + } + if suffix.is_empty() { + // Fits entirely; keep in buffer (do not push yet) so we can append more later. + break; + } else { + // Emit wrapped prefix as a non-explicit row and continue with the remainder. + self.rows.push(Row { + text: prefix, + explicit_break: false, + }); + self.current_line = suffix.to_string(); + } + } + } +} + +/// Take a prefix of `text` whose visible width is at most `max_cols`. +/// Returns (prefix, suffix, prefix_width). +pub fn take_prefix_by_width(text: &str, max_cols: usize) -> (String, &str, usize) { + if max_cols == 0 || text.is_empty() { + return (String::new(), text, 0); + } + let mut cols = 0usize; + let mut end_idx = 0usize; + for (i, ch) in text.char_indices() { + let ch_width = UnicodeWidthChar::width(ch).unwrap_or(0); + if cols.saturating_add(ch_width) > max_cols { + break; + } + cols += ch_width; + end_idx = i + ch.len_utf8(); + if cols == max_cols { + break; + } + } + let prefix = text[..end_idx].to_string(); + let suffix = &text[end_idx..]; + (prefix, suffix, cols) +} + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + + #[test] + fn rows_do_not_exceed_width_ascii() { + let mut rb = RowBuilder::new(10); + rb.push_fragment("hello whirl this is a test"); + let rows = rb.rows().to_vec(); + assert_eq!( + rows, + vec![ + Row { + text: "hello whir".to_string(), + explicit_break: false + }, + Row { + text: "l this is ".to_string(), + explicit_break: false + } + ] + ); + } + + #[test] + fn rows_do_not_exceed_width_emoji_cjk() { + // 😀 is width 2; 你/好 are width 2. + let mut rb = RowBuilder::new(6); + rb.push_fragment("😀😀 你好"); + let rows = rb.rows().to_vec(); + // At width 6, we expect the first row to fit exactly two emojis and a space + // (2 + 2 + 1 = 5) plus one more column for the first CJK char (2 would overflow), + // so only the two emojis and the space fit; the rest remains buffered. + assert_eq!( + rows, + vec![Row { + text: "😀😀 ".to_string(), + explicit_break: false + }] + ); + } + + #[test] + fn fragmentation_invariance_long_token() { + let s = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; // 26 chars + let mut rb_all = RowBuilder::new(7); + rb_all.push_fragment(s); + let all_rows = rb_all.rows().to_vec(); + + let mut rb_chunks = RowBuilder::new(7); + for i in (0..s.len()).step_by(3) { + let end = (i + 3).min(s.len()); + rb_chunks.push_fragment(&s[i..end]); + } + let chunk_rows = rb_chunks.rows().to_vec(); + + assert_eq!(all_rows, chunk_rows); + } + + #[test] + fn newline_splits_rows() { + let mut rb = RowBuilder::new(10); + rb.push_fragment("hello\nworld"); + let rows = rb.display_rows(); + assert!(rows.iter().any(|r| r.explicit_break)); + assert_eq!(rows[0].text, "hello"); + // Second row should begin with 'world' + assert!(rows.iter().any(|r| r.text.starts_with("world"))); + } + + #[test] + fn rewrap_on_width_change() { + let mut rb = RowBuilder::new(10); + rb.push_fragment("abcdefghijK"); + assert!(!rb.rows().is_empty()); + rb.set_width(5); + for r in rb.rows() { + assert!(r.width() <= 5); + } + } +} diff --git a/codex-rs/tui/src/markdown.rs b/codex-rs/tui/src/markdown.rs index ab20138298..910a6869ec 100644 --- a/codex-rs/tui/src/markdown.rs +++ b/codex-rs/tui/src/markdown.rs @@ -1,3 +1,4 @@ +use crate::citation_regex::CITATION_REGEX; use codex_core::config::Config; use codex_core::config_types::UriBasedFileOpener; use ratatui::text::Line; @@ -5,8 +6,7 @@ use ratatui::text::Span; use std::borrow::Cow; use std::path::Path; -use crate::citation_regex::CITATION_REGEX; - +#[allow(dead_code)] pub(crate) fn append_markdown( markdown_source: &str, lines: &mut Vec>, @@ -15,6 +15,7 @@ pub(crate) fn append_markdown( append_markdown_with_opener_and_cwd(markdown_source, lines, config.file_opener, &config.cwd); } +#[allow(dead_code)] fn append_markdown_with_opener_and_cwd( markdown_source: &str, lines: &mut Vec>, @@ -60,6 +61,7 @@ fn append_markdown_with_opener_and_cwd( /// ```text /// ://file: /// ``` +#[allow(dead_code)] fn rewrite_file_citations<'a>( src: &'a str, file_opener: UriBasedFileOpener, diff --git a/codex-rs/tui/src/status_indicator_widget.rs b/codex-rs/tui/src/status_indicator_widget.rs index aa18ac6fa5..fad7e41a39 100644 --- a/codex-rs/tui/src/status_indicator_widget.rs +++ b/codex-rs/tui/src/status_indicator_widget.rs @@ -9,24 +9,22 @@ use std::thread; use std::time::Duration; use ratatui::buffer::Buffer; -use ratatui::layout::Alignment; use ratatui::layout::Rect; use ratatui::style::Color; use ratatui::style::Modifier; use ratatui::style::Style; -use ratatui::style::Stylize; use ratatui::text::Line; use ratatui::text::Span; -use ratatui::widgets::Block; -use ratatui::widgets::BorderType; -use ratatui::widgets::Borders; -use ratatui::widgets::Padding; use ratatui::widgets::Paragraph; use ratatui::widgets::WidgetRef; +use unicode_width::UnicodeWidthStr; use crate::app_event::AppEvent; use crate::app_event_sender::AppEventSender; +// We render the live text using markdown so it visually matches the history +// cells. Before rendering we strip any ANSI escape sequences to avoid writing +// raw control bytes into the back buffer. use codex_ansi_escape::ansi_escape_line; pub(crate) struct StatusIndicatorWidget { @@ -34,6 +32,14 @@ pub(crate) struct StatusIndicatorWidget { /// time). text: String, + /// Animation state: reveal target `text` progressively like a typewriter. + /// We compute the currently visible prefix length based on the current + /// frame index and a constant typing speed. The `base_frame` and + /// `reveal_len_at_base` form the anchor from which we advance. + last_target_len: usize, + base_frame: usize, + reveal_len_at_base: usize, + frame_idx: Arc, running: Arc, // Keep one sender alive to prevent the channel from closing while the @@ -66,9 +72,13 @@ impl StatusIndicatorWidget { } Self { - text: String::from("waiting for logs…"), + text: String::from("waiting for model"), + last_target_len: 0, + base_frame: 0, + reveal_len_at_base: 0, frame_idx, running, + _app_event_tx: app_event_tx, } } @@ -79,7 +89,67 @@ impl StatusIndicatorWidget { /// Update the line that is displayed in the widget. pub(crate) fn update_text(&mut self, text: String) { - self.text = text.replace(['\n', '\r'], " "); + // If the text hasn't changed, don't reset the baseline; let the + // animation continue advancing naturally. + if text == self.text { + return; + } + // Update the target text, preserving newlines so wrapping matches history cells. + // Strip ANSI escapes for the character count so the typewriter animation speed is stable. + let stripped = { + let line = ansi_escape_line(&text); + line.spans + .iter() + .map(|s| s.content.as_ref()) + .collect::>() + .join("") + }; + let new_len = stripped.chars().count(); + + // Compute how many characters are currently revealed so we can carry + // this forward as the new baseline when target text changes. + let current_frame = self.frame_idx.load(std::sync::atomic::Ordering::Relaxed); + let shown_now = self.current_shown_len(current_frame); + + self.text = text; + self.last_target_len = new_len; + self.base_frame = current_frame; + self.reveal_len_at_base = shown_now.min(new_len); + } + + /// Reset the animation and start revealing `text` from the beginning. + #[cfg(test)] + pub(crate) fn restart_with_text(&mut self, text: String) { + let sanitized = text.replace(['\n', '\r'], " "); + let stripped = { + let line = ansi_escape_line(&sanitized); + line.spans + .iter() + .map(|s| s.content.as_ref()) + .collect::>() + .join("") + }; + + let new_len = stripped.chars().count(); + let current_frame = self.frame_idx.load(std::sync::atomic::Ordering::Relaxed); + + self.text = sanitized; + self.last_target_len = new_len; + self.base_frame = current_frame; + // Start from zero revealed characters for a fresh typewriter cycle. + self.reveal_len_at_base = 0; + } + + /// Calculate how many characters should currently be visible given the + /// animation baseline and frame counter. + fn current_shown_len(&self, current_frame: usize) -> usize { + // Increase typewriter speed (~5x): reveal more characters per frame. + const TYPING_CHARS_PER_FRAME: usize = 7; + let frames = current_frame.saturating_sub(self.base_frame); + let advanced = self + .reveal_len_at_base + .saturating_add(frames.saturating_mul(TYPING_CHARS_PER_FRAME)); + advanced.min(self.last_target_len) } } @@ -92,26 +162,22 @@ impl Drop for StatusIndicatorWidget { impl WidgetRef for StatusIndicatorWidget { fn render_ref(&self, area: Rect, buf: &mut Buffer) { - let widget_style = Style::default(); - let block = Block::default() - .padding(Padding::new(1, 0, 0, 0)) - .borders(Borders::LEFT) - .border_type(BorderType::QuadrantOutside) - .border_style(widget_style.dim()); + // Ensure minimal height + if area.height == 0 || area.width == 0 { + return; + } + + // Build animated gradient header for the word "Working". let idx = self.frame_idx.load(std::sync::atomic::Ordering::Relaxed); let header_text = "Working"; let header_chars: Vec = header_text.chars().collect(); - let padding = 4usize; // virtual padding around the word for smoother loop let period = header_chars.len() + padding * 2; let pos = idx % period; - let has_true_color = supports_color::on_cached(supports_color::Stream::Stdout) .map(|level| level.has_16m) .unwrap_or(false); - - // Width of the bright band (in characters). - let band_half_width = 2.0; + let band_half_width = 2.0; // width of the bright band in characters let mut header_spans: Vec> = Vec::new(); for (i, ch) in header_chars.iter().enumerate() { @@ -133,64 +199,46 @@ impl WidgetRef for StatusIndicatorWidget { .fg(Color::Rgb(level, level, level)) .add_modifier(Modifier::BOLD) } else { - // Bold makes dark gray and gray look the same, so don't use it - // when true color is not supported. + // Bold makes dark gray and gray look the same, so don't use it when true color is not supported. Style::default().fg(color_for_level(level)) }; header_spans.push(Span::styled(ch.to_string(), style)); } - header_spans.push(Span::styled( + // Plain rendering: no borders or padding so the live cell is visually indistinguishable from terminal scrollback. + let inner_width = area.width as usize; + + // Compose a single status line like: "▌ Working [•] waiting for model" + let mut spans: Vec> = Vec::new(); + spans.push(Span::styled("▌ ", Style::default().fg(Color::Cyan))); + // Gradient header + spans.extend(header_spans); + // Space after header + spans.push(Span::styled( " ", Style::default() .fg(Color::White) .add_modifier(Modifier::BOLD), )); - // Ensure we do not overflow width. - let inner_width = block.inner(area).width as usize; - - // Sanitize and colour‑strip the potentially colourful log text. This - // ensures that **no** raw ANSI escape sequences leak into the - // back‑buffer which would otherwise cause cursor jumps or stray - // artefacts when the terminal is resized. - let line = ansi_escape_line(&self.text); - let mut sanitized_tail: String = line - .spans - .iter() - .map(|s| s.content.as_ref()) - .collect::>() - .join(""); - - // Truncate *after* stripping escape codes so width calculation is - // accurate. See UTF‑8 boundary comments above. - let header_len: usize = header_spans.iter().map(|s| s.content.len()).sum(); - - if header_len + sanitized_tail.len() > inner_width { - let available_bytes = inner_width.saturating_sub(header_len); - - if sanitized_tail.is_char_boundary(available_bytes) { - sanitized_tail.truncate(available_bytes); + // Truncate spans to fit the width. + let mut acc: Vec> = Vec::new(); + let mut used = 0usize; + for s in spans { + let w = s.content.width(); + if used + w <= inner_width { + acc.push(s); + used += w; } else { - let mut idx = available_bytes; - while idx < sanitized_tail.len() && !sanitized_tail.is_char_boundary(idx) { - idx += 1; - } - sanitized_tail.truncate(idx); + break; } } + let lines = vec![Line::from(acc)]; - let mut spans = header_spans; + // No-op once full text is revealed; the app no longer reacts to a completion event. - // Re‑apply the DIM modifier so the tail appears visually subdued - // irrespective of the colour information preserved by - // `ansi_escape_line`. - spans.push(Span::styled(sanitized_tail, Style::default().dim())); - - let paragraph = Paragraph::new(Line::from(spans)) - .block(block) - .alignment(Alignment::Left); + let paragraph = Paragraph::new(lines); paragraph.render_ref(area, buf); } } @@ -204,3 +252,50 @@ fn color_for_level(level: u8) -> Color { Color::White } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::app_event::AppEvent; + use crate::app_event_sender::AppEventSender; + use std::sync::mpsc::channel; + + #[test] + fn renders_without_left_border_or_padding() { + let (tx_raw, _rx) = channel::(); + let tx = AppEventSender::new(tx_raw); + let mut w = StatusIndicatorWidget::new(tx); + w.restart_with_text("Hello".to_string()); + + let area = ratatui::layout::Rect::new(0, 0, 30, 1); + // Allow a short delay so the typewriter reveals the first character. + std::thread::sleep(std::time::Duration::from_millis(120)); + let mut buf = ratatui::buffer::Buffer::empty(area); + w.render_ref(area, &mut buf); + + // Leftmost column has the left bar + let ch0 = buf[(0, 0)].symbol().chars().next().unwrap_or(' '); + assert_eq!(ch0, '▌', "expected left bar at col 0: {ch0:?}"); + } + + #[test] + fn working_header_is_present_on_last_line() { + let (tx_raw, _rx) = channel::(); + let tx = AppEventSender::new(tx_raw); + let mut w = StatusIndicatorWidget::new(tx); + w.restart_with_text("Hi".to_string()); + // Ensure some frames elapse so we get a stable state. + std::thread::sleep(std::time::Duration::from_millis(120)); + + let area = ratatui::layout::Rect::new(0, 0, 30, 1); + let mut buf = ratatui::buffer::Buffer::empty(area); + w.render_ref(area, &mut buf); + + // Single line; it should contain the animated "Working" header. + let mut row = String::new(); + for x in 0..area.width { + row.push(buf[(x, 0)].symbol().chars().next().unwrap_or(' ')); + } + assert!(row.contains("Working"), "expected Working header: {row:?}"); + } +} diff --git a/codex-rs/tui/tests/vt100_history.rs b/codex-rs/tui/tests/vt100_history.rs new file mode 100644 index 0000000000..11ee044041 --- /dev/null +++ b/codex-rs/tui/tests/vt100_history.rs @@ -0,0 +1,214 @@ +#![cfg(feature = "vt100-tests")] +#![expect(clippy::expect_used)] + +use ratatui::backend::TestBackend; +use ratatui::layout::Rect; +use ratatui::style::Color; +use ratatui::style::Style; +use ratatui::text::Line; +use ratatui::text::Span; + +// Small helper macro to assert a collection contains an item with a clearer +// failure message. +macro_rules! assert_contains { + ($collection:expr, $item:expr $(,)?) => { + assert!( + $collection.contains(&$item), + "Expected {:?} to contain {:?}", + $collection, + $item + ); + }; + ($collection:expr, $item:expr, $($arg:tt)+) => { + assert!($collection.contains(&$item), $($arg)+); + }; +} + +struct TestScenario { + width: u16, + height: u16, + term: codex_tui::custom_terminal::Terminal, +} + +impl TestScenario { + fn new(width: u16, height: u16, viewport: Rect) -> Self { + let backend = TestBackend::new(width, height); + let mut term = codex_tui::custom_terminal::Terminal::with_options(backend) + .expect("failed to construct terminal"); + term.set_viewport_area(viewport); + Self { + width, + height, + term, + } + } + + fn run_insert(&mut self, lines: Vec>) -> Vec { + let mut buf: Vec = Vec::new(); + codex_tui::insert_history::insert_history_lines_to_writer(&mut self.term, &mut buf, lines); + buf + } + + fn screen_rows_from_bytes(&self, bytes: &[u8]) -> Vec { + let mut parser = vt100::Parser::new(self.height, self.width, 0); + parser.process(bytes); + let screen = parser.screen(); + + let mut rows: Vec = Vec::with_capacity(self.height as usize); + for row in 0..self.height { + let mut s = String::with_capacity(self.width as usize); + for col in 0..self.width { + if let Some(cell) = screen.cell(row, col) { + if let Some(ch) = cell.contents().chars().next() { + s.push(ch); + } else { + s.push(' '); + } + } else { + s.push(' '); + } + } + rows.push(s.trim_end().to_string()); + } + rows + } +} + +#[test] +fn hist_001_basic_insertion_no_wrap() { + // Screen of 20x6; viewport is the last row (height=1 at y=5) + let area = Rect::new(0, 5, 20, 1); + let mut scenario = TestScenario::new(20, 6, area); + + let lines = vec![Line::from("first"), Line::from("second")]; + let buf = scenario.run_insert(lines); + let rows = scenario.screen_rows_from_bytes(&buf); + assert_contains!(rows, String::from("first")); + assert_contains!(rows, String::from("second")); + let first_idx = rows + .iter() + .position(|r| r == "first") + .expect("expected 'first' row to be present"); + let second_idx = rows + .iter() + .position(|r| r == "second") + .expect("expected 'second' row to be present"); + assert_eq!(second_idx, first_idx + 1, "rows should be adjacent"); +} + +#[test] +fn hist_002_long_token_wraps() { + let area = Rect::new(0, 5, 20, 1); + let mut scenario = TestScenario::new(20, 6, area); + + let long = "A".repeat(45); // > 2 lines at width 20 + let lines = vec![Line::from(long.clone())]; + let buf = scenario.run_insert(lines); + let mut parser = vt100::Parser::new(6, 20, 0); + parser.process(&buf); + let screen = parser.screen(); + + // Count total A's on the screen + let mut count_a = 0usize; + for row in 0..6 { + for col in 0..20 { + if let Some(cell) = screen.cell(row, col) { + if let Some(ch) = cell.contents().chars().next() { + if ch == 'A' { + count_a += 1; + } + } + } + } + } + + assert_eq!( + count_a, + long.len(), + "wrapped content did not preserve all characters" + ); +} + +#[test] +fn hist_003_emoji_and_cjk() { + let area = Rect::new(0, 5, 20, 1); + let mut scenario = TestScenario::new(20, 6, area); + + let text = String::from("😀😀😀😀😀 你好世界"); + let lines = vec![Line::from(text.clone())]; + let buf = scenario.run_insert(lines); + let rows = scenario.screen_rows_from_bytes(&buf); + let reconstructed: String = rows.join("").chars().filter(|c| *c != ' ').collect(); + for ch in text.chars().filter(|c| !c.is_whitespace()) { + assert!( + reconstructed.contains(ch), + "missing character {ch:?} in reconstructed screen" + ); + } +} + +#[test] +fn hist_004_mixed_ansi_spans() { + let area = Rect::new(0, 5, 20, 1); + let mut scenario = TestScenario::new(20, 6, area); + + let line = Line::from(vec![ + Span::styled("red", Style::default().fg(Color::Red)), + Span::raw("+plain"), + ]); + let buf = scenario.run_insert(vec![line]); + let rows = scenario.screen_rows_from_bytes(&buf); + assert_contains!(rows, String::from("red+plain")); +} + +#[test] +fn hist_006_cursor_restoration() { + let area = Rect::new(0, 5, 20, 1); + let mut scenario = TestScenario::new(20, 6, area); + + let lines = vec![Line::from("x")]; + let buf = scenario.run_insert(lines); + let s = String::from_utf8_lossy(&buf); + // CUP to 1;1 (ANSI: ESC[1;1H) + assert!( + s.contains("\u{1b}[1;1H"), + "expected final CUP to 1;1 in output, got: {s:?}" + ); + // Reset scroll region + assert!( + s.contains("\u{1b}[r"), + "expected reset scroll region in output, got: {s:?}" + ); +} + +#[test] +fn hist_005_pre_scroll_region_down() { + // Viewport not at bottom: y=3 (0-based), height=1 + let area = Rect::new(0, 3, 20, 1); + let mut scenario = TestScenario::new(20, 6, area); + + let lines = vec![Line::from("first"), Line::from("second")]; + let buf = scenario.run_insert(lines); + let s = String::from_utf8_lossy(&buf); + // Expect we limited scroll region to [top+1 .. screen_height] => [4 .. 6] (1-based) + assert!( + s.contains("\u{1b}[4;6r"), + "expected pre-scroll SetScrollRegion 4..6, got: {s:?}" + ); + // Expect we moved cursor to top of that region: row 3 (0-based) => CUP 4;1H + assert!( + s.contains("\u{1b}[4;1H"), + "expected cursor at top of pre-scroll region, got: {s:?}" + ); + // Expect at least two Reverse Index commands (ESC M) for two inserted lines + let ri_count = s.matches("\u{1b}M").count(); + assert!( + ri_count >= 1, + "expected at least one RI (ESC M), got: {s:?}" + ); + // After pre-scroll, we set insertion scroll region to [1 .. new_top] => [1 .. 5] + assert!( + s.contains("\u{1b}[1;5r"), + "expected insertion SetScrollRegion 1..5, got: {s:?}" + ); +} diff --git a/codex-rs/tui/tests/vt100_live_commit.rs b/codex-rs/tui/tests/vt100_live_commit.rs new file mode 100644 index 0000000000..c0cfb3211a --- /dev/null +++ b/codex-rs/tui/tests/vt100_live_commit.rs @@ -0,0 +1,101 @@ +#![cfg(feature = "vt100-tests")] + +use ratatui::backend::TestBackend; +use ratatui::layout::Rect; +use ratatui::text::Line; + +#[test] +fn live_001_commit_on_overflow() { + let backend = TestBackend::new(20, 6); + let mut term = match codex_tui::custom_terminal::Terminal::with_options(backend) { + Ok(t) => t, + Err(e) => panic!("failed to construct terminal: {e}"), + }; + let area = Rect::new(0, 5, 20, 1); + term.set_viewport_area(area); + + // Build 5 explicit rows at width 20. + let mut rb = codex_tui::live_wrap::RowBuilder::new(20); + rb.push_fragment("one\n"); + rb.push_fragment("two\n"); + rb.push_fragment("three\n"); + rb.push_fragment("four\n"); + rb.push_fragment("five\n"); + + // Keep the last 3 in the live ring; commit the first 2. + let commit_rows = rb.drain_commit_ready(3); + let lines: Vec> = commit_rows + .into_iter() + .map(|r| Line::from(r.text)) + .collect(); + + let mut buf: Vec = Vec::new(); + codex_tui::insert_history::insert_history_lines_to_writer(&mut term, &mut buf, lines); + + let mut parser = vt100::Parser::new(6, 20, 0); + parser.process(&buf); + let screen = parser.screen(); + + // The words "one" and "two" should appear above the viewport. + let mut joined = String::new(); + for row in 0..6 { + for col in 0..20 { + if let Some(cell) = screen.cell(row, col) { + if let Some(ch) = cell.contents().chars().next() { + joined.push(ch); + } else { + joined.push(' '); + } + } + } + joined.push('\n'); + } + assert!( + joined.contains("one"), + "expected committed 'one' to be visible\n{joined}" + ); + assert!( + joined.contains("two"), + "expected committed 'two' to be visible\n{joined}" + ); + // The last three (three,four,five) remain in the live ring, not committed here. +} + +#[test] +fn live_002_pre_scroll_and_commit() { + let backend = TestBackend::new(20, 6); + let mut term = match codex_tui::custom_terminal::Terminal::with_options(backend) { + Ok(t) => t, + Err(e) => panic!("failed to construct terminal: {e}"), + }; + // Viewport not at bottom: y=3 + let area = Rect::new(0, 3, 20, 1); + term.set_viewport_area(area); + + let mut rb = codex_tui::live_wrap::RowBuilder::new(20); + rb.push_fragment("alpha\n"); + rb.push_fragment("beta\n"); + rb.push_fragment("gamma\n"); + rb.push_fragment("delta\n"); + + // Keep 3, commit 1. + let commit_rows = rb.drain_commit_ready(3); + let lines: Vec> = commit_rows + .into_iter() + .map(|r| Line::from(r.text)) + .collect(); + + let mut buf: Vec = Vec::new(); + codex_tui::insert_history::insert_history_lines_to_writer(&mut term, &mut buf, lines); + let s = String::from_utf8_lossy(&buf); + + // Expect a SetScrollRegion to [area.top()+1 .. screen_height] and a cursor move to top of that region. + assert!( + s.contains("\u{1b}[4;6r"), + "expected pre-scroll region 4..6, got: {s:?}" + ); + assert!( + s.contains("\u{1b}[4;1H"), + "expected cursor CUP 4;1H, got: {s:?}" + ); +} From fcdb1c4b4da1fd774309abedeb257b1b060dc65d Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Mon, 4 Aug 2025 21:57:55 -0700 Subject: [PATCH 067/100] fix: disable reorderArrays in tamasfe.even-better-toml (#1837) The existing setting kept destroying my `~/.codex/config.toml` for the reasons mentioned in the comment. --- .vscode/settings.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 1712f5989b..aadeca0867 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -11,6 +11,8 @@ "editor.defaultFormatter": "tamasfe.even-better-toml", "editor.formatOnSave": true, }, - "evenBetterToml.formatter.reorderArrays": true, + // Array order for options in ~/.codex/config.toml such as `notify` and the + // `args` for an MCP server is significant, so we disable reordering. + "evenBetterToml.formatter.reorderArrays": false, "evenBetterToml.formatter.reorderKeys": true, } From 136b3ee5bf23982622e554e507eddae5287b5ffe Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Mon, 4 Aug 2025 23:50:03 -0700 Subject: [PATCH 068/100] chore: introduce ModelFamily abstraction (#1838) To date, we have a number of hardcoded OpenAI model slug checks spread throughout the codebase, which makes it hard to audit the various special cases for each model. To mitigate this issue, this PR introduces the idea of a `ModelFamily` that has fields to represent the existing special cases, such as `supports_reasoning_summaries` and `uses_local_shell_tool`. There is a `find_family_for_model()` function that maps the raw model slug to a `ModelFamily`. This function hardcodes all the knowledge about the special attributes for each model. This PR then replaces the hardcoded model name checks with checks against a `ModelFamily`. Note `ModelFamily` is now available as `Config::model_family`. We should ultimately remove `Config::model` in favor of `Config::model_family::slug`. --- codex-rs/core/src/chat_completions.rs | 10 +-- codex-rs/core/src/client.rs | 12 ++-- codex-rs/core/src/client_common.rs | 38 +++-------- codex-rs/core/src/config.rs | 33 +++++---- codex-rs/core/src/lib.rs | 2 +- codex-rs/core/src/model_family.rs | 93 ++++++++++++++++++++++++++ codex-rs/core/src/openai_model_info.rs | 6 +- codex-rs/core/src/openai_tools.rs | 36 +++++----- codex-rs/exec/src/event_processor.rs | 3 +- codex-rs/tui/src/history_cell.rs | 3 +- 10 files changed, 161 insertions(+), 75 deletions(-) create mode 100644 codex-rs/core/src/model_family.rs diff --git a/codex-rs/core/src/chat_completions.rs b/codex-rs/core/src/chat_completions.rs index e1804b191e..6aeccc5dfb 100644 --- a/codex-rs/core/src/chat_completions.rs +++ b/codex-rs/core/src/chat_completions.rs @@ -21,6 +21,7 @@ use crate::client_common::ResponseEvent; use crate::client_common::ResponseStream; use crate::error::CodexErr; use crate::error::Result; +use crate::model_family::ModelFamily; use crate::models::ContentItem; use crate::models::ResponseItem; use crate::openai_tools::create_tools_json_for_chat_completions_api; @@ -29,7 +30,7 @@ use crate::util::backoff; /// Implementation for the classic Chat Completions API. pub(crate) async fn stream_chat_completions( prompt: &Prompt, - model: &str, + model_family: &ModelFamily, include_plan_tool: bool, client: &reqwest::Client, provider: &ModelProviderInfo, @@ -37,7 +38,7 @@ pub(crate) async fn stream_chat_completions( // Build messages array let mut messages = Vec::::new(); - let full_instructions = prompt.get_full_instructions(model); + let full_instructions = prompt.get_full_instructions(model_family); messages.push(json!({"role": "system", "content": full_instructions})); if let Some(instr) = &prompt.get_formatted_user_instructions() { @@ -110,9 +111,10 @@ pub(crate) async fn stream_chat_completions( } } - let tools_json = create_tools_json_for_chat_completions_api(prompt, model, include_plan_tool)?; + let tools_json = + create_tools_json_for_chat_completions_api(prompt, model_family, include_plan_tool)?; let payload = json!({ - "model": model, + "model": model_family.slug, "messages": messages, "stream": true, "tools": tools_json, diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index 00762a8a67..38f390cb30 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -82,7 +82,7 @@ impl ModelClient { // Create the raw streaming connection first. let response_stream = stream_chat_completions( prompt, - &self.config.model, + &self.config.model_family, self.config.include_plan_tool, &self.client, &self.provider, @@ -127,13 +127,17 @@ impl ModelClient { let store = prompt.store && auth_mode != Some(AuthMode::ChatGPT); - let full_instructions = prompt.get_full_instructions(&self.config.model); + let full_instructions = prompt.get_full_instructions(&self.config.model_family); let tools_json = create_tools_json_for_responses_api( prompt, - &self.config.model, + &self.config.model_family, self.config.include_plan_tool, )?; - let reasoning = create_reasoning_param_for_request(&self.config, self.effort, self.summary); + let reasoning = create_reasoning_param_for_request( + &self.config.model_family, + self.effort, + self.summary, + ); // Request encrypted COT if we are not storing responses, // otherwise reasoning items will be referenced by ID diff --git a/codex-rs/core/src/client_common.rs b/codex-rs/core/src/client_common.rs index 6d9524cc92..58ec1c3f69 100644 --- a/codex-rs/core/src/client_common.rs +++ b/codex-rs/core/src/client_common.rs @@ -1,6 +1,7 @@ use crate::config_types::ReasoningEffort as ReasoningEffortConfig; use crate::config_types::ReasoningSummary as ReasoningSummaryConfig; use crate::error::Result; +use crate::model_family::ModelFamily; use crate::models::ResponseItem; use crate::protocol::TokenUsage; use codex_apply_patch::APPLY_PATCH_TOOL_INSTRUCTIONS; @@ -42,13 +43,13 @@ pub struct Prompt { } impl Prompt { - pub(crate) fn get_full_instructions(&self, model: &str) -> Cow<'_, str> { + pub(crate) fn get_full_instructions(&self, model: &ModelFamily) -> Cow<'_, str> { let base = self .base_instructions_override .as_deref() .unwrap_or(BASE_INSTRUCTIONS); let mut sections: Vec<&str> = vec![base]; - if model.starts_with("gpt-4.1") { + if model.needs_special_apply_patch_instructions { sections.push(APPLY_PATCH_TOOL_INSTRUCTIONS); } Cow::Owned(sections.join("\n")) @@ -144,14 +145,12 @@ pub(crate) struct ResponsesApiRequest<'a> { pub(crate) include: Vec, } -use crate::config::Config; - pub(crate) fn create_reasoning_param_for_request( - config: &Config, + model_family: &ModelFamily, effort: ReasoningEffortConfig, summary: ReasoningSummaryConfig, ) -> Option { - if model_supports_reasoning_summaries(config) { + if model_family.supports_reasoning_summaries { let effort: Option = effort.into(); let effort = effort?; Some(Reasoning { @@ -163,27 +162,6 @@ pub(crate) fn create_reasoning_param_for_request( } } -pub fn model_supports_reasoning_summaries(config: &Config) -> bool { - // Currently, we hardcode this rule to decide whether to enable reasoning. - // We expect reasoning to apply only to OpenAI models, but we do not want - // users to have to mess with their config to disable reasoning for models - // that do not support it, such as `gpt-4.1`. - // - // Though if a user is using Codex with non-OpenAI models that, say, happen - // to start with "o", then they can set `model_reasoning_effort = "none"` in - // config.toml to disable reasoning. - // - // Converseley, if a user has a non-OpenAI provider that supports reasoning, - // they can set the top-level `model_supports_reasoning_summaries = true` - // config option to enable reasoning. - if config.model_supports_reasoning_summaries { - return true; - } - - let model = &config.model; - model.starts_with("o") || model.starts_with("codex") -} - pub(crate) struct ResponseStream { pub(crate) rx_event: mpsc::Receiver>, } @@ -198,6 +176,9 @@ impl Stream for ResponseStream { #[cfg(test)] mod tests { + #![allow(clippy::expect_used)] + use crate::model_family::find_family_for_model; + use super::*; #[test] @@ -207,7 +188,8 @@ mod tests { ..Default::default() }; let expected = format!("{BASE_INSTRUCTIONS}\n{APPLY_PATCH_TOOL_INSTRUCTIONS}"); - let full = prompt.get_full_instructions("gpt-4.1"); + let model_family = find_family_for_model("gpt-4.1").expect("known model slug"); + let full = prompt.get_full_instructions(&model_family); assert_eq!(full, expected); } } diff --git a/codex-rs/core/src/config.rs b/codex-rs/core/src/config.rs index b43dc56ba0..a0f36f4587 100644 --- a/codex-rs/core/src/config.rs +++ b/codex-rs/core/src/config.rs @@ -10,6 +10,8 @@ use crate::config_types::ShellEnvironmentPolicyToml; use crate::config_types::Tui; use crate::config_types::UriBasedFileOpener; use crate::flags::OPENAI_DEFAULT_MODEL; +use crate::model_family::ModelFamily; +use crate::model_family::find_family_for_model; use crate::model_provider_info::ModelProviderInfo; use crate::model_provider_info::built_in_model_providers; use crate::openai_model_info::get_model_info; @@ -33,6 +35,8 @@ pub struct Config { /// Optional override of model selection. pub model: String, + pub model_family: ModelFamily, + /// Size of the context window for the model, in tokens. pub model_context_window: Option, @@ -134,10 +138,6 @@ pub struct Config { /// request using the Responses API. pub model_reasoning_summary: ReasoningSummary, - /// When set to `true`, overrides the default heuristic and forces - /// `model_supports_reasoning_summaries()` to return `true`. - pub model_supports_reasoning_summaries: bool, - /// Base URL for requests to ChatGPT (as opposed to the OpenAI API). pub chatgpt_base_url: String, @@ -465,7 +465,19 @@ impl Config { .or(config_profile.model) .or(cfg.model) .unwrap_or_else(default_model); - let openai_model_info = get_model_info(&model); + let model_family = find_family_for_model(&model).unwrap_or_else(|| { + let supports_reasoning_summaries = + cfg.model_supports_reasoning_summaries.unwrap_or(false); + ModelFamily { + slug: model.clone(), + family: model.clone(), + needs_special_apply_patch_instructions: false, + supports_reasoning_summaries, + uses_local_shell_tool: false, + } + }); + + let openai_model_info = get_model_info(&model_family); let model_context_window = cfg .model_context_window .or_else(|| openai_model_info.as_ref().map(|info| info.context_window)); @@ -490,6 +502,7 @@ impl Config { let config = Self { model, + model_family, model_context_window, model_max_output_tokens, model_provider_id, @@ -527,10 +540,6 @@ impl Config { .or(cfg.model_reasoning_summary) .unwrap_or_default(), - model_supports_reasoning_summaries: cfg - .model_supports_reasoning_summaries - .unwrap_or(false), - chatgpt_base_url: config_profile .chatgpt_base_url .or(cfg.chatgpt_base_url) @@ -871,6 +880,7 @@ disable_response_storage = true assert_eq!( Config { model: "o3".to_string(), + model_family: find_family_for_model("o3").expect("known model slug"), model_context_window: Some(200_000), model_max_output_tokens: Some(100_000), model_provider_id: "openai".to_string(), @@ -893,7 +903,6 @@ disable_response_storage = true hide_agent_reasoning: false, model_reasoning_effort: ReasoningEffort::High, model_reasoning_summary: ReasoningSummary::Detailed, - model_supports_reasoning_summaries: false, chatgpt_base_url: "https://chatgpt.com/backend-api/".to_string(), experimental_resume: None, base_instructions: None, @@ -921,6 +930,7 @@ disable_response_storage = true )?; let expected_gpt3_profile_config = Config { model: "gpt-3.5-turbo".to_string(), + model_family: find_family_for_model("gpt-3.5-turbo").expect("known model slug"), model_context_window: Some(16_385), model_max_output_tokens: Some(4_096), model_provider_id: "openai-chat-completions".to_string(), @@ -943,7 +953,6 @@ disable_response_storage = true hide_agent_reasoning: false, model_reasoning_effort: ReasoningEffort::default(), model_reasoning_summary: ReasoningSummary::default(), - model_supports_reasoning_summaries: false, chatgpt_base_url: "https://chatgpt.com/backend-api/".to_string(), experimental_resume: None, base_instructions: None, @@ -986,6 +995,7 @@ disable_response_storage = true )?; let expected_zdr_profile_config = Config { model: "o3".to_string(), + model_family: find_family_for_model("o3").expect("known model slug"), model_context_window: Some(200_000), model_max_output_tokens: Some(100_000), model_provider_id: "openai".to_string(), @@ -1008,7 +1018,6 @@ disable_response_storage = true hide_agent_reasoning: false, model_reasoning_effort: ReasoningEffort::default(), model_reasoning_summary: ReasoningSummary::default(), - model_supports_reasoning_summaries: false, chatgpt_base_url: "https://chatgpt.com/backend-api/".to_string(), experimental_resume: None, base_instructions: None, diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index 4f083d9e56..f9c608b554 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -31,6 +31,7 @@ mod model_provider_info; pub use model_provider_info::ModelProviderInfo; pub use model_provider_info::WireApi; pub use model_provider_info::built_in_model_providers; +pub mod model_family; mod models; mod openai_model_info; mod openai_tools; @@ -47,5 +48,4 @@ mod user_notification; pub mod util; pub use apply_patch::CODEX_APPLY_PATCH_ARG1; -pub use client_common::model_supports_reasoning_summaries; pub use safety::get_platform_sandbox; diff --git a/codex-rs/core/src/model_family.rs b/codex-rs/core/src/model_family.rs new file mode 100644 index 0000000000..9bc61270ce --- /dev/null +++ b/codex-rs/core/src/model_family.rs @@ -0,0 +1,93 @@ +/// A model family is a group of models that share certain characteristics. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct ModelFamily { + /// The full model slug used to derive this model family, e.g. + /// "gpt-4.1-2025-04-14". + pub slug: String, + + /// The model family name, e.g. "gpt-4.1". Note this should able to be used + /// with [`crate::openai_model_info::get_model_info`]. + pub family: String, + + /// True if the model needs additional instructions on how to use the + /// "virtual" `apply_patch` CLI. + pub needs_special_apply_patch_instructions: bool, + + // Whether the `reasoning` field can be set when making a request to this + // model family. Note it has `effort` and `summary` subfields (though + // `summary` is optional). + pub supports_reasoning_summaries: bool, + + // This should be set to true when the model expects a tool named + // "local_shell" to be provided. Its contract must be understood natively by + // the model such that its description can be omitted. + // See https://platform.openai.com/docs/guides/tools-local-shell + pub uses_local_shell_tool: bool, +} + +macro_rules! model_family { + ( + $slug:expr, $family:expr $(, $key:ident : $value:expr )* $(,)? + ) => {{ + // defaults + let mut mf = ModelFamily { + slug: $slug.to_string(), + family: $family.to_string(), + needs_special_apply_patch_instructions: false, + supports_reasoning_summaries: false, + uses_local_shell_tool: false, + }; + // apply overrides + $( + mf.$key = $value; + )* + Some(mf) + }}; +} + +macro_rules! simple_model_family { + ( + $slug:expr, $family:expr + ) => {{ + Some(ModelFamily { + slug: $slug.to_string(), + family: $family.to_string(), + needs_special_apply_patch_instructions: false, + supports_reasoning_summaries: false, + uses_local_shell_tool: false, + }) + }}; +} + +/// Returns a `ModelFamily` for the given model slug, or `None` if the slug +/// does not match any known model family. +pub fn find_family_for_model(slug: &str) -> Option { + if slug.starts_with("o3") { + model_family!( + slug, "o3", + supports_reasoning_summaries: true, + ) + } else if slug.starts_with("o4-mini") { + model_family!( + slug, "o4-mini", + supports_reasoning_summaries: true, + ) + } else if slug.starts_with("codex-mini-latest") { + model_family!( + slug, "codex-mini-latest", + supports_reasoning_summaries: true, + uses_local_shell_tool: true, + ) + } else if slug.starts_with("gpt-4.1") { + model_family!( + slug, "gpt-4.1", + needs_special_apply_patch_instructions: true, + ) + } else if slug.starts_with("gpt-4o") { + simple_model_family!(slug, "gpt-4o") + } else if slug.starts_with("gpt-3.5") { + simple_model_family!(slug, "gpt-3.5") + } else { + None + } +} diff --git a/codex-rs/core/src/openai_model_info.rs b/codex-rs/core/src/openai_model_info.rs index 9ffd831a91..51f028cbdd 100644 --- a/codex-rs/core/src/openai_model_info.rs +++ b/codex-rs/core/src/openai_model_info.rs @@ -1,3 +1,5 @@ +use crate::model_family::ModelFamily; + /// Metadata about a model, particularly OpenAI models. /// We may want to consider including details like the pricing for /// input tokens, output tokens, etc., though users will need to be able to @@ -14,8 +16,8 @@ pub(crate) struct ModelInfo { /// Note details such as what a model like gpt-4o is aliased to may be out of /// date. -pub(crate) fn get_model_info(name: &str) -> Option { - match name { +pub(crate) fn get_model_info(model_family: &ModelFamily) -> Option { + match model_family.slug.as_str() { // https://platform.openai.com/docs/models/o3 "o3" => Some(ModelInfo { context_window: 200_000, diff --git a/codex-rs/core/src/openai_tools.rs b/codex-rs/core/src/openai_tools.rs index 0f1e7d9ca7..305fa523bb 100644 --- a/codex-rs/core/src/openai_tools.rs +++ b/codex-rs/core/src/openai_tools.rs @@ -1,9 +1,9 @@ use serde::Serialize; use serde_json::json; use std::collections::BTreeMap; -use std::sync::LazyLock; use crate::client_common::Prompt; +use crate::model_family::ModelFamily; use crate::plan_tool::PLAN_TOOL; #[derive(Debug, Clone, Serialize)] @@ -42,8 +42,7 @@ pub(crate) enum JsonSchema { }, } -/// Tool usage specification -static DEFAULT_TOOLS: LazyLock> = LazyLock::new(|| { +fn create_shell_tool() -> OpenAiTool { let mut properties = BTreeMap::new(); properties.insert( "command".to_string(), @@ -54,7 +53,7 @@ static DEFAULT_TOOLS: LazyLock> = LazyLock::new(|| { properties.insert("workdir".to_string(), JsonSchema::String); properties.insert("timeout".to_string(), JsonSchema::Number); - vec![OpenAiTool::Function(ResponsesApiTool { + OpenAiTool::Function(ResponsesApiTool { name: "shell", description: "Runs a shell command, and returns its output.", strict: false, @@ -63,29 +62,26 @@ static DEFAULT_TOOLS: LazyLock> = LazyLock::new(|| { required: &["command"], additional_properties: false, }, - })] -}); - -static DEFAULT_CODEX_MODEL_TOOLS: LazyLock> = - LazyLock::new(|| vec![OpenAiTool::LocalShell {}]); + }) +} /// Returns JSON values that are compatible with Function Calling in the /// Responses API: /// https://platform.openai.com/docs/guides/function-calling?api-mode=responses pub(crate) fn create_tools_json_for_responses_api( prompt: &Prompt, - model: &str, + model_family: &ModelFamily, include_plan_tool: bool, ) -> crate::error::Result> { // Assemble tool list: built-in tools + any extra tools from the prompt. - let default_tools = if model.starts_with("codex") { - &DEFAULT_CODEX_MODEL_TOOLS - } else { - &DEFAULT_TOOLS - }; - let mut tools_json = Vec::with_capacity(default_tools.len() + prompt.extra_tools.len()); - for t in default_tools.iter() { - tools_json.push(serde_json::to_value(t)?); + let mut openai_tools = vec![create_shell_tool()]; + if model_family.uses_local_shell_tool { + openai_tools.push(OpenAiTool::LocalShell {}); + } + + let mut tools_json = Vec::with_capacity(openai_tools.len() + prompt.extra_tools.len() + 1); + for tool in openai_tools.iter() { + tools_json.push(serde_json::to_value(tool)?); } tools_json.extend( prompt @@ -107,13 +103,13 @@ pub(crate) fn create_tools_json_for_responses_api( /// https://platform.openai.com/docs/guides/function-calling?api-mode=chat pub(crate) fn create_tools_json_for_chat_completions_api( prompt: &Prompt, - model: &str, + model_family: &ModelFamily, include_plan_tool: bool, ) -> crate::error::Result> { // We start with the JSON for the Responses API and than rewrite it to match // the chat completions tool call format. let responses_api_tools_json = - create_tools_json_for_responses_api(prompt, model, include_plan_tool)?; + create_tools_json_for_responses_api(prompt, model_family, include_plan_tool)?; let tools_json = responses_api_tools_json .into_iter() .filter_map(|mut tool| { diff --git a/codex-rs/exec/src/event_processor.rs b/codex-rs/exec/src/event_processor.rs index 0f189f3fa2..0a2a141eca 100644 --- a/codex-rs/exec/src/event_processor.rs +++ b/codex-rs/exec/src/event_processor.rs @@ -3,7 +3,6 @@ use std::path::Path; use codex_common::summarize_sandbox_policy; use codex_core::WireApi; use codex_core::config::Config; -use codex_core::model_supports_reasoning_summaries; use codex_core::protocol::Event; pub(crate) enum CodexStatus { @@ -29,7 +28,7 @@ pub(crate) fn create_config_summary_entries(config: &Config) -> Vec<(&'static st ("sandbox", summarize_sandbox_policy(&config.sandbox_policy)), ]; if config.model_provider.wire_api == WireApi::Responses - && model_supports_reasoning_summaries(config) + && config.model_family.supports_reasoning_summaries { entries.push(( "reasoning effort", diff --git a/codex-rs/tui/src/history_cell.rs b/codex-rs/tui/src/history_cell.rs index 17f0e683c0..2fb0eecb28 100644 --- a/codex-rs/tui/src/history_cell.rs +++ b/codex-rs/tui/src/history_cell.rs @@ -7,7 +7,6 @@ use codex_common::elapsed::format_duration; use codex_common::summarize_sandbox_policy; use codex_core::WireApi; use codex_core::config::Config; -use codex_core::model_supports_reasoning_summaries; use codex_core::plan_tool::PlanItemArg; use codex_core::plan_tool::StepStatus; use codex_core::plan_tool::UpdatePlanArgs; @@ -177,7 +176,7 @@ impl HistoryCell { ("sandbox", summarize_sandbox_policy(&config.sandbox_policy)), ]; if config.model_provider.wire_api == WireApi::Responses - && model_supports_reasoning_summaries(config) + && config.model_family.supports_reasoning_summaries { entries.push(( "reasoning effort", From d31e149cb1b4439f47393115d7a85b3c8ab8c90d Mon Sep 17 00:00:00 2001 From: Dylan Date: Tue, 5 Aug 2025 00:43:23 -0700 Subject: [PATCH 069/100] [prompt] Update prompt.md (#1839) ## Summary Additional clarifications to our prompt. Still very concise, but we'll continue to add more here. --- codex-rs/core/prompt.md | 49 ++++++++++++++++++++++++------- codex-rs/core/src/openai_tools.rs | 2 +- 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/codex-rs/core/prompt.md b/codex-rs/core/prompt.md index f194eba4e2..d5d96a89b4 100644 --- a/codex-rs/core/prompt.md +++ b/codex-rs/core/prompt.md @@ -1,8 +1,21 @@ -Please resolve the user's task by editing and testing the code files in your current code execution session. -You are a deployed coding agent. -Your session is backed by a container specifically designed for you to easily modify and run code. -The repo(s) are already cloned in your working directory, and you must fully solve the problem for your answer to be considered correct. +You are operating as and within the Codex CLI, an open-source, terminal-based agentic coding assistant built by OpenAI. It wraps OpenAI models to enable natural language interaction with a local codebase. You are expected to be precise, safe, and helpful. +Your capabilities: +- Receive user prompts, project context, and files. +- Stream responses and emit function calls (e.g., shell commands, code edits). +- Run commands, like apply_patch, and manage user approvals based on policy. +- Work inside a workspace with sandboxing instructions specified by the policy described in (## Sandbox environment and approval instructions) + +Within this context, Codex refers to the open-source agentic coding interface (not the old Codex language model built by OpenAI). + +## General guidelines +As a deployed coding agent, please continue working on the user's task until their query is resolved, before ending your turn and yielding back to the user. Only terminate your turn when you are sure that the task is solved. If you are not sure about file content or codebase structure pertaining to the user's request, use your tools to read files and gather the relevant information. Do NOT guess or make up an answer. + +After a user sends their first message, you should immediately provide a brief message acknowledging their request to set the tone and expectation of future work to be done (no more than 8-10 words). This should be done before performing work like exploring the codebase, writing or reading files, or other tool calls needed to complete the task. Use a natural, collaborative tone similar to how a teammate would receive a task during a pair programming session. + +Please resolve the user's task by editing the code files in your current code execution session. Your session allows for you to modify and run code. The repo(s) are already cloned in your working directory, and you must fully solve the problem for your answer to be considered correct. + +### Task execution You MUST adhere to the following criteria when executing the task: - Working on the repo(s) in the current environment is allowed, even if they are proprietary. @@ -12,7 +25,7 @@ You MUST adhere to the following criteria when executing the task: - `user_instructions` are not part of the user's request, but guidance for how to complete the task. - Do not cite `user_instructions` back to the user unless a specific piece is relevant. - Do not use \`ls -R\`, \`find\`, or \`grep\` - these are slow in large repos. Use \`rg\` and \`rg --files\`. -- Use \`apply_patch\` to edit files: {"command":["apply_patch","*** Begin Patch\\n*** Update File: path/to/file.py\\n@@ def example():\\n- pass\\n+ return 123\\n*** End Patch"]} +- Use the \`apply_patch\` shell command to edit files: {"command":["apply_patch","*** Begin Patch\\n*** Update File: path/to/file.py\\n@@ def example():\\n- pass\\n+ return 123\\n*** End Patch"]} - If completing the user's task requires writing or modifying files: - Your code and final answer should follow these _CODING GUIDELINES_: - Fix the problem at the root cause rather than applying surface-level patches, when possible. @@ -35,12 +48,11 @@ You MUST adhere to the following criteria when executing the task: - If completing the user's task DOES NOT require writing or modifying files (e.g., the user asks a question about the code base): - Respond in a friendly tune as a remote teammate, who is knowledgeable, capable and eager to help with coding. - When your task involves writing or modifying files: - - Do NOT tell the user to "save the file" or "copy the code into a file" if you already created or modified the file using \`apply_patch\`. Instead, reference the file as already saved. + - Do NOT tell the user to "save the file" or "copy the code into a file" if you already created or modified the file using the `apply_patch` shell command. Instead, reference the file as already saved. - Do NOT show the full contents of large files you have already written, unless the user explicitly asks for them. -§ `apply-patch` Specification - -Your patch language is a stripped‑down, file‑oriented diff format designed to be easy to parse and safe to apply. You can think of it as a high‑level envelope: +## Using the shell command `apply_patch` to edit files +`apply_patch` is a shell command for editing files. Your patch language is a stripped‑down, file‑oriented diff format designed to be easy to parse and safe to apply. You can think of it as a high‑level envelope: *** Begin Patch [ one or more file sections ] @@ -92,14 +104,28 @@ It is important to remember: - You must include a header with your intended action (Add/Delete/Update) - You must prefix new lines with `+` even when creating a new file +- You must follow this schema exactly when providing a patch -You can invoke apply_patch like: +You can invoke apply_patch with the following shell command: ``` shell {"command":["apply_patch","*** Begin Patch\n*** Add File: hello.txt\n+Hello, world!\n*** End Patch\n"]} ``` -Plan updates +## Sandbox environment and approval instructions + +You are running in a sandboxed workspace backed by version control. The sandbox might be configured by the user to restrict certain behaviors, like accessing the internet or writing to files outside the current directory. + +Commands that are blocked by sandbox settings will be automatically sent to the user for approval. The result of the request will be returned (i.e. the command result, or the request denial). +The user also has an opportunity to approve the same command for the rest of the session. + +Guidance on running within the sandbox: +- When running commands that will likely require approval, attempt to use simple, precise commands, to reduce frequency of approval requests. +- When approval is denied or a command fails due to a permission error, do not retry the exact command in a different way. Move on and continue trying to address the user's request. + + +## Tools available +### Plan updates A tool named `update_plan` is available. Use it to keep an up‑to‑date, step‑by‑step plan for the task so you can follow your progress. When making your plans, keep in mind that you are a deployed coding agent - `update_plan` calls should not involve doing anything that you aren't capable of doing. For example, `update_plan` calls should NEVER contain tasks to merge your own pull requests. Only stop to ask the user if you genuinely need their feedback on a change. @@ -107,3 +133,4 @@ A tool named `update_plan` is available. Use it to keep an up‑to‑date, step - Whenever you finish a step, call `update_plan` again, marking the finished step as `completed` and the next step as `in_progress`. - If your plan needs to change, call `update_plan` with the revised steps and include an `explanation` describing the change. - When all steps are complete, make a final `update_plan` call with all steps marked `completed`. + diff --git a/codex-rs/core/src/openai_tools.rs b/codex-rs/core/src/openai_tools.rs index 305fa523bb..7d4bf4aa1a 100644 --- a/codex-rs/core/src/openai_tools.rs +++ b/codex-rs/core/src/openai_tools.rs @@ -55,7 +55,7 @@ fn create_shell_tool() -> OpenAiTool { OpenAiTool::Function(ResponsesApiTool { name: "shell", - description: "Runs a shell command, and returns its output.", + description: "Runs a shell command and returns its output", strict: false, parameters: JsonSchema::Object { properties, From e0303dbac06b6ead612107fa2144cf5b809bbe98 Mon Sep 17 00:00:00 2001 From: easong-openai Date: Tue, 5 Aug 2025 01:56:13 -0700 Subject: [PATCH 070/100] Rescue chat completion changes (#1846) https://github.com/openai/codex/pull/1835 has some messed up history. This adds support for streaming chat completions, which is useful for ollama. We should probably take a very skeptical eye to the code introduced in this PR. --------- Co-authored-by: Ahmed Ibrahim --- codex-rs/config.md | 13 + codex-rs/core/src/chat_completions.rs | 241 ++++++++++++++---- codex-rs/core/src/client.rs | 14 +- codex-rs/core/src/client_common.rs | 1 + codex-rs/core/src/codex.rs | 40 ++- codex-rs/core/src/config.rs | 12 + codex-rs/core/src/models.rs | 8 + codex-rs/core/src/protocol.rs | 16 ++ .../src/event_processor_with_human_output.rs | 34 +++ codex-rs/mcp-server/src/codex_tool_runner.rs | 4 +- codex-rs/mcp-server/src/conversation_loop.rs | 4 +- codex-rs/tui/src/bottom_pane/mod.rs | 148 ++++++++++- codex-rs/tui/src/chatwidget.rs | 69 ++++- 13 files changed, 547 insertions(+), 57 deletions(-) diff --git a/codex-rs/config.md b/codex-rs/config.md index c7dfe42a75..992fe1aacc 100644 --- a/codex-rs/config.md +++ b/codex-rs/config.md @@ -483,6 +483,19 @@ Setting `hide_agent_reasoning` to `true` suppresses these events in **both** the hide_agent_reasoning = true # defaults to false ``` +## show_raw_agent_reasoning + +Surfaces the model’s raw chain-of-thought ("raw reasoning content") when available. + +Notes: +- Only takes effect if the selected model/provider actually emits raw reasoning content. Many models do not. When unsupported, this option has no visible effect. +- Raw reasoning may include intermediate thoughts or sensitive context. Enable only if acceptable for your workflow. + +Example: +```toml +show_raw_agent_reasoning = true # defaults to false +``` + ## model_context_window The size of the context window for the model, in tokens. diff --git a/codex-rs/core/src/chat_completions.rs b/codex-rs/core/src/chat_completions.rs index 6aeccc5dfb..956dcebda9 100644 --- a/codex-rs/core/src/chat_completions.rs +++ b/codex-rs/core/src/chat_completions.rs @@ -23,6 +23,7 @@ use crate::error::CodexErr; use crate::error::Result; use crate::model_family::ModelFamily; use crate::models::ContentItem; +use crate::models::ReasoningItemContent; use crate::models::ResponseItem; use crate::openai_tools::create_tools_json_for_chat_completions_api; use crate::util::backoff; @@ -209,6 +210,8 @@ async fn process_chat_sse( } let mut fn_call_state = FunctionCallState::default(); + let mut assistant_text = String::new(); + let mut reasoning_text = String::new(); loop { let sse = match timeout(idle_timeout, stream.next()).await { @@ -237,6 +240,31 @@ async fn process_chat_sse( // OpenAI Chat streaming sends a literal string "[DONE]" when finished. if sse.data.trim() == "[DONE]" { + // Emit any finalized items before closing so downstream consumers receive + // terminal events for both assistant content and raw reasoning. + if !assistant_text.is_empty() { + let item = ResponseItem::Message { + role: "assistant".to_string(), + content: vec![ContentItem::OutputText { + text: std::mem::take(&mut assistant_text), + }], + id: None, + }; + let _ = tx_event.send(Ok(ResponseEvent::OutputItemDone(item))).await; + } + + if !reasoning_text.is_empty() { + let item = ResponseItem::Reasoning { + id: String::new(), + summary: Vec::new(), + content: Some(vec![ReasoningItemContent::ReasoningText { + text: std::mem::take(&mut reasoning_text), + }]), + encrypted_content: None, + }; + let _ = tx_event.send(Ok(ResponseEvent::OutputItemDone(item))).await; + } + let _ = tx_event .send(Ok(ResponseEvent::Completed { response_id: String::new(), @@ -256,26 +284,47 @@ async fn process_chat_sse( let choice_opt = chunk.get("choices").and_then(|c| c.get(0)); if let Some(choice) = choice_opt { - // Handle assistant content tokens. + // Handle assistant content tokens as streaming deltas. if let Some(content) = choice .get("delta") .and_then(|d| d.get("content")) .and_then(|c| c.as_str()) { - // Emit a delta so downstream consumers can stream text live. - let _ = tx_event - .send(Ok(ResponseEvent::OutputTextDelta(content.to_string()))) - .await; + if !content.is_empty() { + assistant_text.push_str(content); + let _ = tx_event + .send(Ok(ResponseEvent::OutputTextDelta(content.to_string()))) + .await; + } + } - let item = ResponseItem::Message { - role: "assistant".to_string(), - content: vec![ContentItem::OutputText { - text: content.to_string(), - }], - id: None, - }; + // Forward any reasoning/thinking deltas if present. + // Some providers stream `reasoning` as a plain string while others + // nest the text under an object (e.g. `{ "reasoning": { "text": "…" } }`). + if let Some(reasoning_val) = choice.get("delta").and_then(|d| d.get("reasoning")) { + let mut maybe_text = reasoning_val.as_str().map(|s| s.to_string()); - let _ = tx_event.send(Ok(ResponseEvent::OutputItemDone(item))).await; + if maybe_text.is_none() && reasoning_val.is_object() { + if let Some(s) = reasoning_val + .get("text") + .and_then(|t| t.as_str()) + .filter(|s| !s.is_empty()) + { + maybe_text = Some(s.to_string()); + } else if let Some(s) = reasoning_val + .get("content") + .and_then(|t| t.as_str()) + .filter(|s| !s.is_empty()) + { + maybe_text = Some(s.to_string()); + } + } + + if let Some(reasoning) = maybe_text { + let _ = tx_event + .send(Ok(ResponseEvent::ReasoningContentDelta(reasoning))) + .await; + } } // Handle streaming function / tool calls. @@ -312,7 +361,21 @@ async fn process_chat_sse( if let Some(finish_reason) = choice.get("finish_reason").and_then(|v| v.as_str()) { match finish_reason { "tool_calls" if fn_call_state.active => { - // Build the FunctionCall response item. + // First, flush the terminal raw reasoning so UIs can finalize + // the reasoning stream before any exec/tool events begin. + if !reasoning_text.is_empty() { + let item = ResponseItem::Reasoning { + id: String::new(), + summary: Vec::new(), + content: Some(vec![ReasoningItemContent::ReasoningText { + text: std::mem::take(&mut reasoning_text), + }]), + encrypted_content: None, + }; + let _ = tx_event.send(Ok(ResponseEvent::OutputItemDone(item))).await; + } + + // Then emit the FunctionCall response item. let item = ResponseItem::FunctionCall { id: None, name: fn_call_state.name.clone().unwrap_or_else(|| "".to_string()), @@ -320,11 +383,33 @@ async fn process_chat_sse( call_id: fn_call_state.call_id.clone().unwrap_or_else(String::new), }; - // Emit it downstream. let _ = tx_event.send(Ok(ResponseEvent::OutputItemDone(item))).await; } "stop" => { - // Regular turn without tool-call. + // Regular turn without tool-call. Emit the final assistant message + // as a single OutputItemDone so non-delta consumers see the result. + if !assistant_text.is_empty() { + let item = ResponseItem::Message { + role: "assistant".to_string(), + content: vec![ContentItem::OutputText { + text: std::mem::take(&mut assistant_text), + }], + id: None, + }; + let _ = tx_event.send(Ok(ResponseEvent::OutputItemDone(item))).await; + } + // Also emit a terminal Reasoning item so UIs can finalize raw reasoning. + if !reasoning_text.is_empty() { + let item = ResponseItem::Reasoning { + id: String::new(), + summary: Vec::new(), + content: Some(vec![ReasoningItemContent::ReasoningText { + text: std::mem::take(&mut reasoning_text), + }]), + encrypted_content: None, + }; + let _ = tx_event.send(Ok(ResponseEvent::OutputItemDone(item))).await; + } } _ => {} } @@ -362,10 +447,17 @@ async fn process_chat_sse( /// The adapter is intentionally *lossless*: callers who do **not** opt in via /// [`AggregateStreamExt::aggregate()`] keep receiving the original unmodified /// events. +#[derive(Copy, Clone, Eq, PartialEq)] +enum AggregateMode { + AggregatedOnly, + Streaming, +} pub(crate) struct AggregatedChatStream { inner: S, cumulative: String, - pending_completed: Option, + cumulative_reasoning: String, + pending: std::collections::VecDeque, + mode: AggregateMode, } impl Stream for AggregatedChatStream @@ -377,8 +469,8 @@ where fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { let this = self.get_mut(); - // First, flush any buffered Completed event from the previous call. - if let Some(ev) = this.pending_completed.take() { + // First, flush any buffered events from the previous call. + if let Some(ev) = this.pending.pop_front() { return Poll::Ready(Some(Ok(ev))); } @@ -395,16 +487,21 @@ where let is_assistant_delta = matches!(&item, crate::models::ResponseItem::Message { role, .. } if role == "assistant"); if is_assistant_delta { - if let crate::models::ResponseItem::Message { content, .. } = &item { - if let Some(text) = content.iter().find_map(|c| match c { - crate::models::ContentItem::OutputText { text } => Some(text), - _ => None, - }) { - this.cumulative.push_str(text); + // Only use the final assistant message if we have not + // seen any deltas; otherwise, deltas already built the + // cumulative text and this would duplicate it. + if this.cumulative.is_empty() { + if let crate::models::ResponseItem::Message { content, .. } = &item { + if let Some(text) = content.iter().find_map(|c| match c { + crate::models::ContentItem::OutputText { text } => Some(text), + _ => None, + }) { + this.cumulative.push_str(text); + } } } - // Swallow partial assistant chunk; keep polling. + // Swallow assistant message here; emit on Completed. continue; } @@ -415,24 +512,50 @@ where response_id, token_usage, }))) => { + // Build any aggregated items in the correct order: Reasoning first, then Message. + let mut emitted_any = false; + + if !this.cumulative_reasoning.is_empty() + && matches!(this.mode, AggregateMode::AggregatedOnly) + { + let aggregated_reasoning = crate::models::ResponseItem::Reasoning { + id: String::new(), + summary: Vec::new(), + content: Some(vec![ + crate::models::ReasoningItemContent::ReasoningText { + text: std::mem::take(&mut this.cumulative_reasoning), + }, + ]), + encrypted_content: None, + }; + this.pending + .push_back(ResponseEvent::OutputItemDone(aggregated_reasoning)); + emitted_any = true; + } + if !this.cumulative.is_empty() { - let aggregated_item = crate::models::ResponseItem::Message { + let aggregated_message = crate::models::ResponseItem::Message { id: None, role: "assistant".to_string(), content: vec![crate::models::ContentItem::OutputText { text: std::mem::take(&mut this.cumulative), }], }; + this.pending + .push_back(ResponseEvent::OutputItemDone(aggregated_message)); + emitted_any = true; + } - // Buffer Completed so it is returned *after* the aggregated message. - this.pending_completed = Some(ResponseEvent::Completed { - response_id, - token_usage, + // Always emit Completed last when anything was aggregated. + if emitted_any { + this.pending.push_back(ResponseEvent::Completed { + response_id: response_id.clone(), + token_usage: token_usage.clone(), }); - - return Poll::Ready(Some(Ok(ResponseEvent::OutputItemDone( - aggregated_item, - )))); + // Return the first pending event now. + if let Some(ev) = this.pending.pop_front() { + return Poll::Ready(Some(Ok(ev))); + } } // Nothing aggregated – forward Completed directly. @@ -447,13 +570,27 @@ where continue; } Poll::Ready(Some(Ok(ResponseEvent::OutputTextDelta(delta)))) => { - // Forward deltas unchanged so callers can stream text - // live while still receiving a single aggregated - // OutputItemDone at the end of the turn. - return Poll::Ready(Some(Ok(ResponseEvent::OutputTextDelta(delta)))); + // Always accumulate deltas so we can emit a final OutputItemDone at Completed. + this.cumulative.push_str(&delta); + if matches!(this.mode, AggregateMode::Streaming) { + // In streaming mode, also forward the delta immediately. + return Poll::Ready(Some(Ok(ResponseEvent::OutputTextDelta(delta)))); + } else { + continue; + } } - Poll::Ready(Some(Ok(ResponseEvent::ReasoningSummaryDelta(delta)))) => { - return Poll::Ready(Some(Ok(ResponseEvent::ReasoningSummaryDelta(delta)))); + Poll::Ready(Some(Ok(ResponseEvent::ReasoningContentDelta(delta)))) => { + // Always accumulate reasoning deltas so we can emit a final Reasoning item at Completed. + this.cumulative_reasoning.push_str(&delta); + if matches!(this.mode, AggregateMode::Streaming) { + // In streaming mode, also forward the delta immediately. + return Poll::Ready(Some(Ok(ResponseEvent::ReasoningContentDelta(delta)))); + } else { + continue; + } + } + Poll::Ready(Some(Ok(ResponseEvent::ReasoningSummaryDelta(_)))) => { + continue; } } } @@ -482,12 +619,24 @@ pub(crate) trait AggregateStreamExt: Stream> + Size /// } /// ``` fn aggregate(self) -> AggregatedChatStream { - AggregatedChatStream { - inner: self, - cumulative: String::new(), - pending_completed: None, - } + AggregatedChatStream::new(self, AggregateMode::AggregatedOnly) } } impl AggregateStreamExt for T where T: Stream> + Sized {} + +impl AggregatedChatStream { + fn new(inner: S, mode: AggregateMode) -> Self { + AggregatedChatStream { + inner, + cumulative: String::new(), + cumulative_reasoning: String::new(), + pending: std::collections::VecDeque::new(), + mode, + } + } + + pub(crate) fn streaming_mode(inner: S) -> Self { + Self::new(inner, AggregateMode::Streaming) + } +} diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index 38f390cb30..514e683e53 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -92,7 +92,11 @@ impl ModelClient { // Wrap it with the aggregation adapter so callers see *only* // the final assistant message per turn (matching the // behaviour of the Responses API). - let mut aggregated = response_stream.aggregate(); + let mut aggregated = if self.config.show_raw_agent_reasoning { + crate::chat_completions::AggregatedChatStream::streaming_mode(response_stream) + } else { + response_stream.aggregate() + }; // Bridge the aggregated stream back into a standard // `ResponseStream` by forwarding events through a channel. @@ -437,6 +441,14 @@ async fn process_sse( } } } + "response.reasoning_text.delta" => { + if let Some(delta) = event.delta { + let event = ResponseEvent::ReasoningContentDelta(delta); + if tx_event.send(Ok(event)).await.is_err() { + return; + } + } + } "response.created" => { if event.response.is_some() { let _ = tx_event.send(Ok(ResponseEvent::Created {})).await; diff --git a/codex-rs/core/src/client_common.rs b/codex-rs/core/src/client_common.rs index 58ec1c3f69..8b845a52e6 100644 --- a/codex-rs/core/src/client_common.rs +++ b/codex-rs/core/src/client_common.rs @@ -72,6 +72,7 @@ pub enum ResponseEvent { }, OutputTextDelta(String), ReasoningSummaryDelta(String), + ReasoningContentDelta(String), } #[derive(Debug, Serialize)] diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 8d24356460..0ce0c4ea2b 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -56,6 +56,7 @@ use crate::mcp_tool_call::handle_mcp_tool_call; use crate::models::ContentItem; use crate::models::FunctionCallOutputPayload; use crate::models::LocalShellAction; +use crate::models::ReasoningItemContent; use crate::models::ReasoningItemReasoningSummary; use crate::models::ResponseInputItem; use crate::models::ResponseItem; @@ -66,6 +67,8 @@ use crate::protocol::AgentMessageDeltaEvent; use crate::protocol::AgentMessageEvent; use crate::protocol::AgentReasoningDeltaEvent; use crate::protocol::AgentReasoningEvent; +use crate::protocol::AgentReasoningRawContentDeltaEvent; +use crate::protocol::AgentReasoningRawContentEvent; use crate::protocol::ApplyPatchApprovalRequestEvent; use crate::protocol::AskForApproval; use crate::protocol::BackgroundEventEvent; @@ -227,6 +230,7 @@ pub(crate) struct Session { state: Mutex, codex_linux_sandbox_exe: Option, user_shell: shell::Shell, + show_raw_agent_reasoning: bool, } impl Session { @@ -822,6 +826,7 @@ async fn submission_loop( codex_linux_sandbox_exe: config.codex_linux_sandbox_exe.clone(), disable_response_storage, user_shell: default_shell, + show_raw_agent_reasoning: config.show_raw_agent_reasoning, })); // Patch restored state into the newly created session. @@ -1132,6 +1137,7 @@ async fn run_task(sess: Arc, sub_id: String, input: Vec) { ResponseItem::Reasoning { id, summary, + content, encrypted_content, }, None, @@ -1139,6 +1145,7 @@ async fn run_task(sess: Arc, sub_id: String, input: Vec) { items_to_record_in_conversation_history.push(ResponseItem::Reasoning { id: id.clone(), summary: summary.clone(), + content: content.clone(), encrypted_content: encrypted_content.clone(), }); } @@ -1392,6 +1399,17 @@ async fn try_run_turn( }; sess.tx_event.send(event).await.ok(); } + ResponseEvent::ReasoningContentDelta(delta) => { + if sess.show_raw_agent_reasoning { + let event = Event { + id: sub_id.to_string(), + msg: EventMsg::AgentReasoningRawContentDelta( + AgentReasoningRawContentDeltaEvent { delta }, + ), + }; + sess.tx_event.send(event).await.ok(); + } + } } } } @@ -1498,7 +1516,12 @@ async fn handle_response_item( } None } - ResponseItem::Reasoning { summary, .. } => { + ResponseItem::Reasoning { + id: _, + summary, + content, + encrypted_content: _, + } => { for item in summary { let text = match item { ReasoningItemReasoningSummary::SummaryText { text } => text, @@ -1509,6 +1532,21 @@ async fn handle_response_item( }; sess.tx_event.send(event).await.ok(); } + if sess.show_raw_agent_reasoning && content.is_some() { + let content = content.unwrap(); + for item in content { + let text = match item { + ReasoningItemContent::ReasoningText { text } => text, + }; + let event = Event { + id: sub_id.to_string(), + msg: EventMsg::AgentReasoningRawContent(AgentReasoningRawContentEvent { + text, + }), + }; + sess.tx_event.send(event).await.ok(); + } + } None } ResponseItem::FunctionCall { diff --git a/codex-rs/core/src/config.rs b/codex-rs/core/src/config.rs index a0f36f4587..d97d5ec13d 100644 --- a/codex-rs/core/src/config.rs +++ b/codex-rs/core/src/config.rs @@ -61,6 +61,10 @@ pub struct Config { /// users are only interested in the final agent responses. pub hide_agent_reasoning: bool, + /// When set to `true`, `AgentReasoningRawContentEvent` events will be shown in the UI/output. + /// Defaults to `false`. + pub show_raw_agent_reasoning: bool, + /// Disable server-side response storage (sends the full conversation /// context with every request). Currently necessary for OpenAI customers /// who have opted into Zero Data Retention (ZDR). @@ -325,6 +329,10 @@ pub struct ConfigToml { /// UI/output. Defaults to `false`. pub hide_agent_reasoning: Option, + /// When set to `true`, `AgentReasoningRawContentEvent` events will be shown in the UI/output. + /// Defaults to `false`. + pub show_raw_agent_reasoning: Option, + pub model_reasoning_effort: Option, pub model_reasoning_summary: Option, @@ -531,6 +539,7 @@ impl Config { codex_linux_sandbox_exe, hide_agent_reasoning: cfg.hide_agent_reasoning.unwrap_or(false), + show_raw_agent_reasoning: cfg.show_raw_agent_reasoning.unwrap_or(false), model_reasoning_effort: config_profile .model_reasoning_effort .or(cfg.model_reasoning_effort) @@ -901,6 +910,7 @@ disable_response_storage = true tui: Tui::default(), codex_linux_sandbox_exe: None, hide_agent_reasoning: false, + show_raw_agent_reasoning: false, model_reasoning_effort: ReasoningEffort::High, model_reasoning_summary: ReasoningSummary::Detailed, chatgpt_base_url: "https://chatgpt.com/backend-api/".to_string(), @@ -951,6 +961,7 @@ disable_response_storage = true tui: Tui::default(), codex_linux_sandbox_exe: None, hide_agent_reasoning: false, + show_raw_agent_reasoning: false, model_reasoning_effort: ReasoningEffort::default(), model_reasoning_summary: ReasoningSummary::default(), chatgpt_base_url: "https://chatgpt.com/backend-api/".to_string(), @@ -1016,6 +1027,7 @@ disable_response_storage = true tui: Tui::default(), codex_linux_sandbox_exe: None, hide_agent_reasoning: false, + show_raw_agent_reasoning: false, model_reasoning_effort: ReasoningEffort::default(), model_reasoning_summary: ReasoningSummary::default(), chatgpt_base_url: "https://chatgpt.com/backend-api/".to_string(), diff --git a/codex-rs/core/src/models.rs b/codex-rs/core/src/models.rs index 91bfb3bc8c..fb48b53070 100644 --- a/codex-rs/core/src/models.rs +++ b/codex-rs/core/src/models.rs @@ -45,6 +45,8 @@ pub enum ResponseItem { Reasoning { id: String, summary: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + content: Option>, encrypted_content: Option, }, LocalShellCall { @@ -136,6 +138,12 @@ pub enum ReasoningItemReasoningSummary { SummaryText { text: String }, } +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ReasoningItemContent { + ReasoningText { text: String }, +} + impl From> for ResponseInputItem { fn from(items: Vec) -> Self { Self::Message { diff --git a/codex-rs/core/src/protocol.rs b/codex-rs/core/src/protocol.rs index 82591a2c78..aa330f6bae 100644 --- a/codex-rs/core/src/protocol.rs +++ b/codex-rs/core/src/protocol.rs @@ -359,6 +359,12 @@ pub enum EventMsg { /// Agent reasoning delta event from agent. AgentReasoningDelta(AgentReasoningDeltaEvent), + /// Raw chain-of-thought from agent. + AgentReasoningRawContent(AgentReasoningRawContentEvent), + + /// Agent reasoning content delta event from agent. + AgentReasoningRawContentDelta(AgentReasoningRawContentDeltaEvent), + /// Ack the client's configure message. SessionConfigured(SessionConfiguredEvent), @@ -464,6 +470,16 @@ pub struct AgentReasoningEvent { pub text: String, } +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct AgentReasoningRawContentEvent { + pub text: String, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct AgentReasoningRawContentDeltaEvent { + pub delta: String, +} + #[derive(Debug, Clone, Deserialize, Serialize)] pub struct AgentReasoningDeltaEvent { pub delta: String, diff --git a/codex-rs/exec/src/event_processor_with_human_output.rs b/codex-rs/exec/src/event_processor_with_human_output.rs index 7703c138fc..393ef4ab1b 100644 --- a/codex-rs/exec/src/event_processor_with_human_output.rs +++ b/codex-rs/exec/src/event_processor_with_human_output.rs @@ -5,6 +5,8 @@ use codex_core::plan_tool::UpdatePlanArgs; use codex_core::protocol::AgentMessageDeltaEvent; use codex_core::protocol::AgentMessageEvent; use codex_core::protocol::AgentReasoningDeltaEvent; +use codex_core::protocol::AgentReasoningRawContentDeltaEvent; +use codex_core::protocol::AgentReasoningRawContentEvent; use codex_core::protocol::BackgroundEventEvent; use codex_core::protocol::ErrorEvent; use codex_core::protocol::Event; @@ -55,8 +57,10 @@ pub(crate) struct EventProcessorWithHumanOutput { /// Whether to include `AgentReasoning` events in the output. show_agent_reasoning: bool, + show_raw_agent_reasoning: bool, answer_started: bool, reasoning_started: bool, + raw_reasoning_started: bool, last_message_path: Option, } @@ -81,8 +85,10 @@ impl EventProcessorWithHumanOutput { green: Style::new().green(), cyan: Style::new().cyan(), show_agent_reasoning: !config.hide_agent_reasoning, + show_raw_agent_reasoning: config.show_raw_agent_reasoning, answer_started: false, reasoning_started: false, + raw_reasoning_started: false, last_message_path, } } else { @@ -97,8 +103,10 @@ impl EventProcessorWithHumanOutput { green: Style::new(), cyan: Style::new(), show_agent_reasoning: !config.hide_agent_reasoning, + show_raw_agent_reasoning: config.show_raw_agent_reasoning, answer_started: false, reasoning_started: false, + raw_reasoning_started: false, last_message_path, } } @@ -203,6 +211,32 @@ impl EventProcessor for EventProcessorWithHumanOutput { #[allow(clippy::expect_used)] std::io::stdout().flush().expect("could not flush stdout"); } + EventMsg::AgentReasoningRawContent(AgentReasoningRawContentEvent { text }) => { + if !self.show_raw_agent_reasoning { + return CodexStatus::Running; + } + if !self.raw_reasoning_started { + print!("{text}"); + #[allow(clippy::expect_used)] + std::io::stdout().flush().expect("could not flush stdout"); + } else { + println!(); + self.raw_reasoning_started = false; + } + } + EventMsg::AgentReasoningRawContentDelta(AgentReasoningRawContentDeltaEvent { + delta, + }) => { + if !self.show_raw_agent_reasoning { + return CodexStatus::Running; + } + if !self.raw_reasoning_started { + self.raw_reasoning_started = true; + } + print!("{delta}"); + #[allow(clippy::expect_used)] + std::io::stdout().flush().expect("could not flush stdout"); + } EventMsg::AgentMessage(AgentMessageEvent { message }) => { // if answer_started is false, this means we haven't received any // delta. Thus, we need to print the message as a new answer. diff --git a/codex-rs/mcp-server/src/codex_tool_runner.rs b/codex-rs/mcp-server/src/codex_tool_runner.rs index 205dfa4631..b91c4a7609 100644 --- a/codex-rs/mcp-server/src/codex_tool_runner.rs +++ b/codex-rs/mcp-server/src/codex_tool_runner.rs @@ -252,7 +252,9 @@ async fn run_codex_tool_session_inner( EventMsg::AgentMessage(AgentMessageEvent { .. }) => { // TODO: think how we want to support this in the MCP } - EventMsg::TaskStarted + EventMsg::AgentReasoningRawContent(_) + | EventMsg::AgentReasoningRawContentDelta(_) + | EventMsg::TaskStarted | EventMsg::TokenCount(_) | EventMsg::AgentReasoning(_) | EventMsg::McpToolCallBegin(_) diff --git a/codex-rs/mcp-server/src/conversation_loop.rs b/codex-rs/mcp-server/src/conversation_loop.rs index 1db39a2306..80c34760c5 100644 --- a/codex-rs/mcp-server/src/conversation_loop.rs +++ b/codex-rs/mcp-server/src/conversation_loop.rs @@ -90,7 +90,9 @@ pub async fn run_conversation_loop( EventMsg::AgentMessage(AgentMessageEvent { .. }) => { // TODO: think how we want to support this in the MCP } - EventMsg::TaskStarted + EventMsg::AgentReasoningRawContent(_) + | EventMsg::AgentReasoningRawContentDelta(_) + | EventMsg::TaskStarted | EventMsg::TokenCount(_) | EventMsg::AgentReasoning(_) | EventMsg::McpToolCallBegin(_) diff --git a/codex-rs/tui/src/bottom_pane/mod.rs b/codex-rs/tui/src/bottom_pane/mod.rs index fde0b3bde8..cdb01ba06a 100644 --- a/codex-rs/tui/src/bottom_pane/mod.rs +++ b/codex-rs/tui/src/bottom_pane/mod.rs @@ -138,6 +138,11 @@ impl BottomPane<'_> { view.handle_key_event(self, key_event); if !view.is_complete() { self.active_view = Some(view); + } else if self.is_task_running { + let mut v = StatusIndicatorView::new(self.app_event_tx.clone()); + v.update_text("waiting for model".to_string()); + self.active_view = Some(Box::new(v)); + self.status_view_active = true; } self.request_redraw(); InputResult::None @@ -163,6 +168,12 @@ impl BottomPane<'_> { CancellationEvent::Handled => { if !view.is_complete() { self.active_view = Some(view); + } else if self.is_task_running { + // Modal aborted but task still running – restore status indicator. + let mut v = StatusIndicatorView::new(self.app_event_tx.clone()); + v.update_text("waiting for model".to_string()); + self.active_view = Some(Box::new(v)); + self.status_view_active = true; } self.show_ctrl_c_quit_hint(); } @@ -202,15 +213,20 @@ impl BottomPane<'_> { handled_by_view = true; } - // Fallback: if the current active view did not consume status updates, - // present an overlay above the composer. - if !handled_by_view { + // Fallback: if the current active view did not consume status updates + // and no modal view is active, present an overlay above the composer. + // If a modal is active, do NOT render the overlay to avoid drawing + // over the dialog. + if !handled_by_view && self.active_view.is_none() { if self.live_status.is_none() { self.live_status = Some(StatusIndicatorWidget::new(self.app_event_tx.clone())); } if let Some(status) = &mut self.live_status { status.update_text(text); } + } else if !handled_by_view { + // Ensure any previous overlay is cleared when a modal becomes active. + self.live_status = None; } self.request_redraw(); } @@ -296,6 +312,8 @@ impl BottomPane<'_> { // Otherwise create a new approval modal overlay. let modal = ApprovalModalView::new(request, self.app_event_tx.clone()); self.active_view = Some(Box::new(modal)); + // Hide any overlay status while a modal is visible. + self.live_status = None; self.status_view_active = false; self.request_redraw() } @@ -368,16 +386,18 @@ impl WidgetRef for &BottomPane<'_> { y_offset = y_offset.saturating_add(1); } if let Some(status) = &self.live_status { - let live_h = status.desired_height(area.width).min(area.height); + let live_h = status + .desired_height(area.width) + .min(area.height.saturating_sub(y_offset)); if live_h > 0 { let live_rect = Rect { x: area.x, - y: area.y, + y: area.y + y_offset, width: area.width, height: live_h, }; status.render_ref(live_rect, buf); - y_offset = live_h; + y_offset = y_offset.saturating_add(live_h); } } @@ -540,6 +560,122 @@ mod tests { ); } + #[test] + fn overlay_not_shown_above_approval_modal() { + let (tx_raw, _rx) = channel::(); + let tx = AppEventSender::new(tx_raw); + let mut pane = BottomPane::new(BottomPaneParams { + app_event_tx: tx, + has_input_focus: true, + enhanced_keys_supported: false, + }); + + // Create an approval modal (active view). + pane.push_approval_request(exec_request()); + // Attempt to update status; this should NOT create an overlay while modal is visible. + pane.update_status_text("running command".to_string()); + + // Render and verify the top row does not include the Working header overlay. + let area = Rect::new(0, 0, 60, 6); + let mut buf = Buffer::empty(area); + (&pane).render_ref(area, &mut buf); + + let mut r0 = String::new(); + for x in 0..area.width { + r0.push(buf[(x, 0)].symbol().chars().next().unwrap_or(' ')); + } + assert!( + !r0.contains("Working"), + "overlay Working header should not render above modal" + ); + } + + #[test] + fn composer_not_shown_after_denied_if_task_running() { + let (tx_raw, rx) = channel::(); + let tx = AppEventSender::new(tx_raw); + let mut pane = BottomPane::new(BottomPaneParams { + app_event_tx: tx.clone(), + has_input_focus: true, + enhanced_keys_supported: false, + }); + + // Start a running task so the status indicator replaces the composer. + pane.set_task_running(true); + pane.update_status_text("waiting for model".to_string()); + + // Push an approval modal (e.g., command approval) which should hide the status view. + pane.push_approval_request(exec_request()); + + // Simulate pressing 'n' (deny) on the modal. + use crossterm::event::KeyCode; + use crossterm::event::KeyEvent; + use crossterm::event::KeyModifiers; + pane.handle_key_event(KeyEvent::new(KeyCode::Char('n'), KeyModifiers::NONE)); + + // After denial, since the task is still running, the status indicator + // should be restored as the active view; the composer should NOT be visible. + assert!( + pane.status_view_active, + "status view should be active after denial" + ); + assert!(pane.active_view.is_some(), "active view should be present"); + + // Render and ensure the top row includes the Working header instead of the composer. + // Give the animation thread a moment to tick. + std::thread::sleep(std::time::Duration::from_millis(120)); + let area = Rect::new(0, 0, 40, 3); + let mut buf = Buffer::empty(area); + (&pane).render_ref(area, &mut buf); + let mut row0 = String::new(); + for x in 0..area.width { + row0.push(buf[(x, 0)].symbol().chars().next().unwrap_or(' ')); + } + assert!( + row0.contains("Working"), + "expected Working header after denial: {row0:?}" + ); + + // Drain the channel to avoid unused warnings. + drop(rx); + } + + #[test] + fn status_indicator_visible_during_command_execution() { + let (tx_raw, _rx) = channel::(); + let tx = AppEventSender::new(tx_raw); + let mut pane = BottomPane::new(BottomPaneParams { + app_event_tx: tx, + has_input_focus: true, + enhanced_keys_supported: false, + }); + + // Begin a task: show initial status. + pane.set_task_running(true); + pane.update_status_text("waiting for model".to_string()); + + // As a long-running command begins (post-approval), ensure the status + // indicator is visible while we wait for the command to run. + pane.update_status_text("running command".to_string()); + + // Allow some frames so the animation thread ticks. + std::thread::sleep(std::time::Duration::from_millis(120)); + + // Render and confirm the line contains the "Working" header. + let area = Rect::new(0, 0, 40, 3); + let mut buf = Buffer::empty(area); + (&pane).render_ref(area, &mut buf); + + let mut row0 = String::new(); + for x in 0..area.width { + row0.push(buf[(x, 0)].symbol().chars().next().unwrap_or(' ')); + } + assert!( + row0.contains("Working"), + "expected Working header: {row0:?}" + ); + } + #[test] fn bottom_padding_present_for_status_view() { let (tx_raw, _rx) = channel::(); diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index f63810b62a..94bd2f121b 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -9,6 +9,8 @@ use codex_core::protocol::AgentMessageDeltaEvent; use codex_core::protocol::AgentMessageEvent; use codex_core::protocol::AgentReasoningDeltaEvent; use codex_core::protocol::AgentReasoningEvent; +use codex_core::protocol::AgentReasoningRawContentDeltaEvent; +use codex_core::protocol::AgentReasoningRawContentEvent; use codex_core::protocol::ApplyPatchApprovalRequestEvent; use codex_core::protocol::ErrorEvent; use codex_core::protocol::Event; @@ -61,6 +63,7 @@ pub(crate) struct ChatWidget<'a> { initial_user_message: Option, token_usage: TokenUsage, reasoning_buffer: String, + content_buffer: String, // Buffer for streaming assistant answer text; we do not surface partial // We wait for the final AgentMessage event and then emit the full text // at once into scrollback so the history contains a single message. @@ -101,6 +104,24 @@ fn create_initial_user_message(text: String, image_paths: Vec) -> Optio } impl ChatWidget<'_> { + fn emit_stream_header(&mut self, kind: StreamKind) { + use ratatui::text::Line as RLine; + if self.stream_header_emitted { + return; + } + let header = match kind { + StreamKind::Reasoning => RLine::from("thinking".magenta().italic()), + StreamKind::Answer => RLine::from("codex".magenta().bold()), + }; + self.app_event_tx + .send(AppEvent::InsertHistory(vec![header])); + self.stream_header_emitted = true; + } + fn finalize_active_stream(&mut self) { + if let Some(kind) = self.current_stream { + self.finalize_stream(kind); + } + } pub(crate) fn new( config: Config, app_event_tx: AppEventSender, @@ -161,6 +182,7 @@ impl ChatWidget<'_> { ), token_usage: TokenUsage::default(), reasoning_buffer: String::new(), + content_buffer: String::new(), answer_buffer: String::new(), running_commands: HashMap::new(), live_builder: RowBuilder::new(80), @@ -276,6 +298,20 @@ impl ChatWidget<'_> { self.finalize_stream(StreamKind::Reasoning); self.request_redraw(); } + EventMsg::AgentReasoningRawContentDelta(AgentReasoningRawContentDeltaEvent { + delta, + }) => { + // Treat raw reasoning content the same as summarized reasoning for UI flow. + self.begin_stream(StreamKind::Reasoning); + self.reasoning_buffer.push_str(&delta); + self.stream_push_and_maybe_commit(&delta); + self.request_redraw(); + } + EventMsg::AgentReasoningRawContent(AgentReasoningRawContentEvent { text: _ }) => { + // Finalize the raw reasoning stream just like the summarized reasoning event. + self.finalize_stream(StreamKind::Reasoning); + self.request_redraw(); + } EventMsg::TaskStarted => { self.bottom_pane.clear_ctrl_c_quit_hint(); self.bottom_pane.set_task_running(true); @@ -299,6 +335,14 @@ impl ChatWidget<'_> { EventMsg::Error(ErrorEvent { message }) => { self.add_to_history(HistoryCell::new_error_event(message.clone())); self.bottom_pane.set_task_running(false); + self.bottom_pane.clear_live_ring(); + self.live_builder = RowBuilder::new(self.live_builder.width()); + self.current_stream = None; + self.stream_header_emitted = false; + self.answer_buffer.clear(); + self.reasoning_buffer.clear(); + self.content_buffer.clear(); + self.request_redraw(); } EventMsg::PlanUpdate(update) => { // Commit plan updates directly to history (no status-line preview). @@ -310,6 +354,7 @@ impl ChatWidget<'_> { cwd, reason, }) => { + self.finalize_active_stream(); // Log a background summary immediately so the history is chronological. let cmdline = strip_bash_lc_and_escape(&command); let text = format!( @@ -336,6 +381,7 @@ impl ChatWidget<'_> { reason, grant_root, }) => { + self.finalize_active_stream(); // ------------------------------------------------------------------ // Before we even prompt the user for approval we surface the patch // summary in the main conversation so that the dialog appears in a @@ -365,6 +411,10 @@ impl ChatWidget<'_> { command, cwd, }) => { + self.finalize_active_stream(); + // Ensure the status indicator is visible while the command runs. + self.bottom_pane + .update_status_text("running command".to_string()); self.running_commands.insert( call_id, RunningCommand { @@ -408,6 +458,7 @@ impl ChatWidget<'_> { call_id: _, invocation, }) => { + self.finalize_active_stream(); self.add_to_history(HistoryCell::new_active_mcp_tool_call(invocation)); } EventMsg::McpToolCallEnd(McpToolCallEndEvent { @@ -451,7 +502,9 @@ impl ChatWidget<'_> { /// Update the live log preview while a task is running. pub(crate) fn update_latest_log(&mut self, line: String) { - self.bottom_pane.update_status_text(line); + if self.bottom_pane.is_task_running() { + self.bottom_pane.update_status_text(line); + } } fn request_redraw(&mut self) { @@ -478,8 +531,15 @@ impl ChatWidget<'_> { if self.bottom_pane.is_task_running() { self.bottom_pane.clear_ctrl_c_quit_hint(); self.submit_op(Op::Interrupt); + self.bottom_pane.set_task_running(false); + self.bottom_pane.clear_live_ring(); + self.live_builder = RowBuilder::new(self.live_builder.width()); + self.current_stream = None; + self.stream_header_emitted = false; self.answer_buffer.clear(); self.reasoning_buffer.clear(); + self.content_buffer.clear(); + self.request_redraw(); CancellationEvent::Ignored } else if self.bottom_pane.ctrl_c_quit_hint_visible() { self.submit_op(Op::Shutdown); @@ -518,6 +578,12 @@ impl ChatWidget<'_> { impl ChatWidget<'_> { fn begin_stream(&mut self, kind: StreamKind) { + if let Some(current) = self.current_stream { + if current != kind { + self.finalize_stream(current); + } + } + if self.current_stream != Some(kind) { self.current_stream = Some(kind); self.stream_header_emitted = false; @@ -526,6 +592,7 @@ impl ChatWidget<'_> { // Ensure the waiting status is visible (composer replaced). self.bottom_pane .update_status_text("waiting for model".to_string()); + self.emit_stream_header(kind); } } From 9285350842fc87b4bf84f4788151707153270b6f Mon Sep 17 00:00:00 2001 From: easong-openai Date: Tue, 5 Aug 2025 11:31:11 -0700 Subject: [PATCH 071/100] Introduce `--oss` flag to use gpt-oss models (#1848) This adds support for easily running Codex backed by a local Ollama instance running our new open source models. See https://github.com/openai/gpt-oss for details. If you pass in `--oss` you'll be prompted to install/launch ollama, and it will automatically download the 20b model and attempt to use it. We'll likely want to expand this with some options later to make the experience smoother for users who can't run the 20b or want to run the 120b. Co-authored-by: Michael Bolin --- README.md | 36 ++ codex-rs/Cargo.lock | 19 + codex-rs/Cargo.toml | 1 + codex-rs/core/src/config.rs | 10 +- codex-rs/core/src/lib.rs | 1 + codex-rs/core/src/model_family.rs | 2 + codex-rs/core/src/model_provider_info.rs | 120 ++++-- codex-rs/exec/Cargo.toml | 1 + codex-rs/exec/src/cli.rs | 3 + codex-rs/exec/src/lib.rs | 24 +- codex-rs/mcp-server/src/codex_tool_config.rs | 2 + .../src/tool_handlers/create_conversation.rs | 2 + codex-rs/ollama/Cargo.toml | 32 ++ codex-rs/ollama/src/client.rs | 366 ++++++++++++++++++ codex-rs/ollama/src/lib.rs | 52 +++ codex-rs/ollama/src/parser.rs | 82 ++++ codex-rs/ollama/src/pull.rs | 147 +++++++ codex-rs/ollama/src/url.rs | 39 ++ codex-rs/tui/Cargo.toml | 1 + codex-rs/tui/src/cli.rs | 6 + codex-rs/tui/src/lib.rs | 22 +- 21 files changed, 924 insertions(+), 44 deletions(-) create mode 100644 codex-rs/ollama/Cargo.toml create mode 100644 codex-rs/ollama/src/client.rs create mode 100644 codex-rs/ollama/src/lib.rs create mode 100644 codex-rs/ollama/src/parser.rs create mode 100644 codex-rs/ollama/src/pull.rs create mode 100644 codex-rs/ollama/src/url.rs diff --git a/README.md b/README.md index c7f6a1d595..dd5e466252 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ This is the home of the **Codex CLI**, which is a coding agent from OpenAI that - [Quickstart](#quickstart) - [OpenAI API Users](#openai-api-users) - [OpenAI Plus/Pro Users](#openai-pluspro-users) + - [Using OpenAI Open Source Models](#using-open-source-models) - [Why Codex?](#why-codex) - [Security model & permissions](#security-model--permissions) - [Platform sandboxing details](#platform-sandboxing-details) @@ -186,6 +187,41 @@ they'll be committed to your working directory. --- +## Using Open Source Models + +Codex can run fully locally against an OpenAI‑compatible OSS host (like Ollama) using the `--oss` flag: + +- Interactive UI: + - codex --oss +- Non‑interactive (programmatic) mode: + - echo "Refactor utils" | codex exec --oss + +Model selection when using `--oss`: + +- If you omit `-m/--model`, Codex defaults to -m gpt-oss:20b and will verify it exists locally (downloading if needed). +- To pick a different size, pass one of: + - -m "gpt-oss:20b" + - -m "gpt-oss:120b" + +Point Codex at your own OSS host: + +- By default, `--oss` talks to http://localhost:11434/v1. +- To use a different host, set one of these environment variables before running Codex: + - CODEX_OSS_BASE_URL, for example: + - CODEX_OSS_BASE_URL="http://my-ollama.example.com:11434/v1" codex --oss -m gpt-oss:20b + - or CODEX_OSS_PORT (when the host is localhost): + - CODEX_OSS_PORT=11434 codex --oss + +Advanced: you can persist this in your config instead of environment variables by overriding the built‑in `oss` provider in `~/.codex/config.toml`: + +```toml +[model_providers.oss] +name = "Open Source" +base_url = "http://my-ollama.example.com:11434/v1" +``` + +--- + ## Why Codex? Codex CLI is built for developers who already **live in the terminal** and want diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 2e20a7d624..4e21baf7f5 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -729,6 +729,7 @@ dependencies = [ "codex-arg0", "codex-common", "codex-core", + "codex-ollama", "owo-colors", "predicates", "serde_json", @@ -838,6 +839,23 @@ dependencies = [ "wiremock", ] +[[package]] +name = "codex-ollama" +version = "0.0.0" +dependencies = [ + "async-stream", + "bytes", + "codex-core", + "futures", + "reqwest", + "serde_json", + "tempfile", + "tokio", + "toml 0.9.4", + "tracing", + "wiremock", +] + [[package]] name = "codex-tui" version = "0.0.0" @@ -852,6 +870,7 @@ dependencies = [ "codex-core", "codex-file-search", "codex-login", + "codex-ollama", "color-eyre", "crossterm", "image", diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index 0f8085c7e5..0ed8852228 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -14,6 +14,7 @@ members = [ "mcp-client", "mcp-server", "mcp-types", + "ollama", "tui", ] resolver = "2" diff --git a/codex-rs/core/src/config.rs b/codex-rs/core/src/config.rs index d97d5ec13d..e62fcc39e2 100644 --- a/codex-rs/core/src/config.rs +++ b/codex-rs/core/src/config.rs @@ -385,6 +385,8 @@ pub struct ConfigOverrides { pub codex_linux_sandbox_exe: Option, pub base_instructions: Option, pub include_plan_tool: Option, + pub default_disable_response_storage: Option, + pub default_show_raw_agent_reasoning: Option, } impl Config { @@ -408,6 +410,8 @@ impl Config { codex_linux_sandbox_exe, base_instructions, include_plan_tool, + default_disable_response_storage, + default_show_raw_agent_reasoning, } = overrides; let config_profile = match config_profile_key.as_ref().or(cfg.profile.as_ref()) { @@ -525,6 +529,7 @@ impl Config { disable_response_storage: config_profile .disable_response_storage .or(cfg.disable_response_storage) + .or(default_disable_response_storage) .unwrap_or(false), notify: cfg.notify, user_instructions, @@ -539,7 +544,10 @@ impl Config { codex_linux_sandbox_exe, hide_agent_reasoning: cfg.hide_agent_reasoning.unwrap_or(false), - show_raw_agent_reasoning: cfg.show_raw_agent_reasoning.unwrap_or(false), + show_raw_agent_reasoning: cfg + .show_raw_agent_reasoning + .or(default_show_raw_agent_reasoning) + .unwrap_or(false), model_reasoning_effort: config_profile .model_reasoning_effort .or(cfg.model_reasoning_effort) diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index f9c608b554..965cb77bf1 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -28,6 +28,7 @@ mod mcp_connection_manager; mod mcp_tool_call; mod message_history; mod model_provider_info; +pub use model_provider_info::BUILT_IN_OSS_MODEL_PROVIDER_ID; pub use model_provider_info::ModelProviderInfo; pub use model_provider_info::WireApi; pub use model_provider_info::built_in_model_providers; diff --git a/codex-rs/core/src/model_family.rs b/codex-rs/core/src/model_family.rs index 9bc61270ce..7c4a9de6c2 100644 --- a/codex-rs/core/src/model_family.rs +++ b/codex-rs/core/src/model_family.rs @@ -85,6 +85,8 @@ pub fn find_family_for_model(slug: &str) -> Option { ) } else if slug.starts_with("gpt-4o") { simple_model_family!(slug, "gpt-4o") + } else if slug.starts_with("gpt-oss") { + simple_model_family!(slug, "gpt-oss") } else if slug.starts_with("gpt-3.5") { simple_model_family!(slug, "gpt-3.5") } else { diff --git a/codex-rs/core/src/model_provider_info.rs b/codex-rs/core/src/model_provider_info.rs index 49478660f4..595f05ef75 100644 --- a/codex-rs/core/src/model_provider_info.rs +++ b/codex-rs/core/src/model_provider_info.rs @@ -226,53 +226,93 @@ impl ModelProviderInfo { } } +const DEFAULT_OLLAMA_PORT: u32 = 11434; + +pub const BUILT_IN_OSS_MODEL_PROVIDER_ID: &str = "oss"; + /// Built-in default provider list. pub fn built_in_model_providers() -> HashMap { use ModelProviderInfo as P; - // We do not want to be in the business of adjucating which third-party - // providers are bundled with Codex CLI, so we only include the OpenAI - // provider by default. Users are encouraged to add to `model_providers` - // in config.toml to add their own providers. - [( - "openai", - P { - name: "OpenAI".into(), - // Allow users to override the default OpenAI endpoint by - // exporting `OPENAI_BASE_URL`. This is useful when pointing - // Codex at a proxy, mock server, or Azure-style deployment - // without requiring a full TOML override for the built-in - // OpenAI provider. - base_url: std::env::var("OPENAI_BASE_URL") + // These CODEX_OSS_ environment variables are experimental: we may + // switch to reading values from config.toml instead. + let codex_oss_base_url = match std::env::var("CODEX_OSS_BASE_URL") + .ok() + .filter(|v| !v.trim().is_empty()) + { + Some(url) => url, + None => format!( + "http://localhost:{port}/v1", + port = std::env::var("CODEX_OSS_PORT") .ok() - .filter(|v| !v.trim().is_empty()), - env_key: None, - env_key_instructions: None, - wire_api: WireApi::Responses, - query_params: None, - http_headers: Some( - [("version".to_string(), env!("CARGO_PKG_VERSION").to_string())] + .filter(|v| !v.trim().is_empty()) + .and_then(|v| v.parse::().ok()) + .unwrap_or(DEFAULT_OLLAMA_PORT) + ), + }; + + // We do not want to be in the business of adjucating which third-party + // providers are bundled with Codex CLI, so we only include the OpenAI and + // open source ("oss") providers by default. Users are encouraged to add to + // `model_providers` in config.toml to add their own providers. + [ + ( + "openai", + P { + name: "OpenAI".into(), + // Allow users to override the default OpenAI endpoint by + // exporting `OPENAI_BASE_URL`. This is useful when pointing + // Codex at a proxy, mock server, or Azure-style deployment + // without requiring a full TOML override for the built-in + // OpenAI provider. + base_url: std::env::var("OPENAI_BASE_URL") + .ok() + .filter(|v| !v.trim().is_empty()), + env_key: None, + env_key_instructions: None, + wire_api: WireApi::Responses, + query_params: None, + http_headers: Some( + [("version".to_string(), env!("CARGO_PKG_VERSION").to_string())] + .into_iter() + .collect(), + ), + env_http_headers: Some( + [ + ( + "OpenAI-Organization".to_string(), + "OPENAI_ORGANIZATION".to_string(), + ), + ("OpenAI-Project".to_string(), "OPENAI_PROJECT".to_string()), + ] .into_iter() .collect(), - ), - env_http_headers: Some( - [ - ( - "OpenAI-Organization".to_string(), - "OPENAI_ORGANIZATION".to_string(), - ), - ("OpenAI-Project".to_string(), "OPENAI_PROJECT".to_string()), - ] - .into_iter() - .collect(), - ), - // Use global defaults for retry/timeout unless overridden in config.toml. - request_max_retries: None, - stream_max_retries: None, - stream_idle_timeout_ms: None, - requires_auth: true, - }, - )] + ), + // Use global defaults for retry/timeout unless overridden in config.toml. + request_max_retries: None, + stream_max_retries: None, + stream_idle_timeout_ms: None, + requires_auth: true, + }, + ), + ( + BUILT_IN_OSS_MODEL_PROVIDER_ID, + P { + name: "Open Source".into(), + base_url: Some(codex_oss_base_url), + env_key: None, + env_key_instructions: None, + wire_api: WireApi::Chat, + query_params: None, + http_headers: None, + env_http_headers: None, + request_max_retries: None, + stream_max_retries: None, + stream_idle_timeout_ms: None, + requires_auth: false, + }, + ), + ] .into_iter() .map(|(k, v)| (k.to_string(), v)) .collect() diff --git a/codex-rs/exec/Cargo.toml b/codex-rs/exec/Cargo.toml index cd521410b1..aee480d7b4 100644 --- a/codex-rs/exec/Cargo.toml +++ b/codex-rs/exec/Cargo.toml @@ -25,6 +25,7 @@ codex-common = { path = "../common", features = [ "sandbox_summary", ] } codex-core = { path = "../core" } +codex-ollama = { path = "../ollama" } owo-colors = "4.2.0" serde_json = "1" shlex = "1.3.0" diff --git a/codex-rs/exec/src/cli.rs b/codex-rs/exec/src/cli.rs index 53af25c7e9..ea659e3252 100644 --- a/codex-rs/exec/src/cli.rs +++ b/codex-rs/exec/src/cli.rs @@ -14,6 +14,9 @@ pub struct Cli { #[arg(long, short = 'm')] pub model: Option, + #[arg(long = "oss", default_value_t = false)] + pub oss: bool, + /// Select the sandbox policy to use when executing model-generated shell /// commands. #[arg(long = "sandbox", short = 's')] diff --git a/codex-rs/exec/src/lib.rs b/codex-rs/exec/src/lib.rs index ce4d7f65cc..c1af4f5b45 100644 --- a/codex-rs/exec/src/lib.rs +++ b/codex-rs/exec/src/lib.rs @@ -9,6 +9,7 @@ use std::path::PathBuf; use std::sync::Arc; pub use cli::Cli; +use codex_core::BUILT_IN_OSS_MODEL_PROVIDER_ID; use codex_core::codex_wrapper::CodexConversation; use codex_core::codex_wrapper::{self}; use codex_core::config::Config; @@ -35,6 +36,7 @@ pub async fn run_main(cli: Cli, codex_linux_sandbox_exe: Option) -> any let Cli { images, model, + oss, config_profile, full_auto, dangerously_bypass_approvals_and_sandbox, @@ -114,6 +116,24 @@ pub async fn run_main(cli: Cli, codex_linux_sandbox_exe: Option) -> any sandbox_mode_cli_arg.map(Into::::into) }; + // When using `--oss`, let the bootstrapper pick the model (defaulting to + // gpt-oss:20b) and ensure it is present locally. Also, force the built‑in + // `oss` model provider. + let model_provider_override = if oss { + Some(BUILT_IN_OSS_MODEL_PROVIDER_ID.to_owned()) + } else { + None + }; + let model = if oss { + Some( + codex_ollama::ensure_oss_ready(model.clone()) + .await + .map_err(|e| anyhow::anyhow!("OSS setup failed: {e}"))?, + ) + } else { + model + }; + // Load configuration and determine approval policy let overrides = ConfigOverrides { model, @@ -123,10 +143,12 @@ pub async fn run_main(cli: Cli, codex_linux_sandbox_exe: Option) -> any approval_policy: Some(AskForApproval::Never), sandbox_mode, cwd: cwd.map(|p| p.canonicalize().unwrap_or(p)), - model_provider: None, + model_provider: model_provider_override, codex_linux_sandbox_exe, base_instructions: None, include_plan_tool: None, + default_disable_response_storage: oss.then_some(true), + default_show_raw_agent_reasoning: oss.then_some(true), }; // Parse `-c` overrides. let cli_kv_overrides = match config_overrides.parse_overrides() { diff --git a/codex-rs/mcp-server/src/codex_tool_config.rs b/codex-rs/mcp-server/src/codex_tool_config.rs index 877d0e05f7..f1a502bbb3 100644 --- a/codex-rs/mcp-server/src/codex_tool_config.rs +++ b/codex-rs/mcp-server/src/codex_tool_config.rs @@ -158,6 +158,8 @@ impl CodexToolCallParam { codex_linux_sandbox_exe, base_instructions, include_plan_tool, + default_disable_response_storage: None, + default_show_raw_agent_reasoning: None, }; let cli_overrides = cli_overrides diff --git a/codex-rs/mcp-server/src/tool_handlers/create_conversation.rs b/codex-rs/mcp-server/src/tool_handlers/create_conversation.rs index 28a8965115..c1f4035663 100644 --- a/codex-rs/mcp-server/src/tool_handlers/create_conversation.rs +++ b/codex-rs/mcp-server/src/tool_handlers/create_conversation.rs @@ -59,6 +59,8 @@ pub(crate) async fn handle_create_conversation( codex_linux_sandbox_exe: None, base_instructions, include_plan_tool: None, + default_disable_response_storage: None, + default_show_raw_agent_reasoning: None, }; let cfg: CodexConfig = match CodexConfig::load_with_cli_overrides(cli_overrides, overrides) { diff --git a/codex-rs/ollama/Cargo.toml b/codex-rs/ollama/Cargo.toml new file mode 100644 index 0000000000..ead9a06494 --- /dev/null +++ b/codex-rs/ollama/Cargo.toml @@ -0,0 +1,32 @@ +[package] +edition = "2024" +name = "codex-ollama" +version = { workspace = true } + +[lib] +name = "codex_ollama" +path = "src/lib.rs" + +[lints] +workspace = true + +[dependencies] +async-stream = "0.3" +bytes = "1.10.1" +codex-core = { path = "../core" } +futures = "0.3" +reqwest = { version = "0.12", features = ["json", "stream"] } +serde_json = "1" +tokio = { version = "1", features = [ + "io-std", + "macros", + "process", + "rt-multi-thread", + "signal", +] } +toml = "0.9.2" +tracing = { version = "0.1.41", features = ["log"] } +wiremock = "0.6" + +[dev-dependencies] +tempfile = "3" diff --git a/codex-rs/ollama/src/client.rs b/codex-rs/ollama/src/client.rs new file mode 100644 index 0000000000..8a15039fad --- /dev/null +++ b/codex-rs/ollama/src/client.rs @@ -0,0 +1,366 @@ +use bytes::BytesMut; +use futures::StreamExt; +use futures::stream::BoxStream; +use serde_json::Value as JsonValue; +use std::collections::VecDeque; +use std::io; + +use codex_core::WireApi; + +use crate::parser::pull_events_from_value; +use crate::pull::PullEvent; +use crate::pull::PullProgressReporter; +use crate::url::base_url_to_host_root; +use crate::url::is_openai_compatible_base_url; + +/// Client for interacting with a local Ollama instance. +pub struct OllamaClient { + client: reqwest::Client, + host_root: String, + uses_openai_compat: bool, +} + +impl OllamaClient { + pub fn from_oss_provider() -> Self { + #![allow(clippy::expect_used)] + // Use the built-in OSS provider's base URL. + let built_in_model_providers = codex_core::built_in_model_providers(); + let provider = built_in_model_providers + .get(codex_core::BUILT_IN_OSS_MODEL_PROVIDER_ID) + .expect("oss provider must exist"); + let base_url = provider + .base_url + .as_ref() + .expect("oss provider must have a base_url"); + Self::from_provider(base_url, provider.wire_api) + } + + /// Construct a client for the built‑in open‑source ("oss") model provider + /// and verify that a local Ollama server is reachable. If no server is + /// detected, returns an error with helpful installation/run instructions. + pub async fn try_from_oss_provider() -> io::Result { + let client = Self::from_oss_provider(); + if client.probe_server().await? { + Ok(client) + } else { + Err(io::Error::other( + "No running Ollama server detected. Start it with: `ollama serve` (after installing). Install instructions: https://github.com/ollama/ollama?tab=readme-ov-file#ollama", + )) + } + } + + /// Build a client from a provider definition. Falls back to the default + /// local URL if no base_url is configured. + fn from_provider(base_url: &str, wire_api: WireApi) -> Self { + let uses_openai_compat = is_openai_compatible_base_url(base_url) + || matches!(wire_api, WireApi::Chat) && is_openai_compatible_base_url(base_url); + let host_root = base_url_to_host_root(base_url); + let client = reqwest::Client::builder() + .connect_timeout(std::time::Duration::from_secs(5)) + .build() + .unwrap_or_else(|_| reqwest::Client::new()); + Self { + client, + host_root, + uses_openai_compat, + } + } + + /// Low-level constructor given a raw host root, e.g. "http://localhost:11434". + #[cfg(test)] + fn from_host_root(host_root: impl Into) -> Self { + let client = reqwest::Client::builder() + .connect_timeout(std::time::Duration::from_secs(5)) + .build() + .unwrap_or_else(|_| reqwest::Client::new()); + Self { + client, + host_root: host_root.into(), + uses_openai_compat: false, + } + } + + /// Probe whether the server is reachable by hitting the appropriate health endpoint. + pub async fn probe_server(&self) -> io::Result { + let url = if self.uses_openai_compat { + format!("{}/v1/models", self.host_root.trim_end_matches('/')) + } else { + format!("{}/api/tags", self.host_root.trim_end_matches('/')) + }; + let resp = self.client.get(url).send().await; + Ok(matches!(resp, Ok(r) if r.status().is_success())) + } + + /// Return the list of model names known to the local Ollama instance. + pub async fn fetch_models(&self) -> io::Result> { + let tags_url = format!("{}/api/tags", self.host_root.trim_end_matches('/')); + let resp = self + .client + .get(tags_url) + .send() + .await + .map_err(io::Error::other)?; + if !resp.status().is_success() { + return Ok(Vec::new()); + } + let val = resp.json::().await.map_err(io::Error::other)?; + let names = val + .get("models") + .and_then(|m| m.as_array()) + .map(|arr| { + arr.iter() + .filter_map(|v| v.get("name").and_then(|n| n.as_str())) + .map(|s| s.to_string()) + .collect::>() + }) + .unwrap_or_default(); + Ok(names) + } + + /// Start a model pull and emit streaming events. The returned stream ends when + /// a Success event is observed or the server closes the connection. + pub async fn pull_model_stream( + &self, + model: &str, + ) -> io::Result> { + let url = format!("{}/api/pull", self.host_root.trim_end_matches('/')); + let resp = self + .client + .post(url) + .json(&serde_json::json!({"model": model, "stream": true})) + .send() + .await + .map_err(io::Error::other)?; + if !resp.status().is_success() { + return Err(io::Error::other(format!( + "failed to start pull: HTTP {}", + resp.status() + ))); + } + + let mut stream = resp.bytes_stream(); + let mut buf = BytesMut::new(); + let _pending: VecDeque = VecDeque::new(); + + // Using an async stream adaptor backed by unfold-like manual loop. + let s = async_stream::stream! { + while let Some(chunk) = stream.next().await { + match chunk { + Ok(bytes) => { + buf.extend_from_slice(&bytes); + while let Some(pos) = buf.iter().position(|b| *b == b'\n') { + let line = buf.split_to(pos + 1); + if let Ok(text) = std::str::from_utf8(&line) { + let text = text.trim(); + if text.is_empty() { continue; } + if let Ok(value) = serde_json::from_str::(text) { + for ev in pull_events_from_value(&value) { yield ev; } + if let Some(err_msg) = value.get("error").and_then(|e| e.as_str()) { + yield PullEvent::Error(err_msg.to_string()); + return; + } + if let Some(status) = value.get("status").and_then(|s| s.as_str()) { + if status == "success" { yield PullEvent::Success; return; } + } + } + } + } + } + Err(_) => { + // Connection error: end the stream. + return; + } + } + } + }; + + Ok(Box::pin(s)) + } + + /// High-level helper to pull a model and drive a progress reporter. + pub async fn pull_with_reporter( + &self, + model: &str, + reporter: &mut dyn PullProgressReporter, + ) -> io::Result<()> { + reporter.on_event(&PullEvent::Status(format!("Pulling model {model}...")))?; + let mut stream = self.pull_model_stream(model).await?; + while let Some(event) = stream.next().await { + reporter.on_event(&event)?; + match event { + PullEvent::Success => { + return Ok(()); + } + PullEvent::Error(err) => { + // Emperically, ollama returns a 200 OK response even when + // the output stream includes an error message. Verify with: + // + // `curl -i http://localhost:11434/api/pull -d '{ "model": "foobarbaz" }'` + // + // As such, we have to check the event stream, not the + // HTTP response status, to determine whether to return Err. + return Err(io::Error::other(format!("Pull failed: {err}"))); + } + PullEvent::ChunkProgress { .. } | PullEvent::Status(_) => { + continue; + } + } + } + Err(io::Error::other( + "Pull stream ended unexpectedly without success.", + )) + } +} + +#[cfg(test)] +mod tests { + #![allow(clippy::expect_used, clippy::unwrap_used)] + use super::*; + + /// Simple RAII guard to set an environment variable for the duration of a test + /// and restore the previous value (or remove it) on drop to avoid cross-test + /// interference. + struct EnvVarGuard { + key: String, + prev: Option, + } + impl EnvVarGuard { + fn set(key: &str, value: String) -> Self { + let prev = std::env::var(key).ok(); + // set_var is safe but we mirror existing tests that use an unsafe block + // to silence edition lints around global mutation during tests. + unsafe { std::env::set_var(key, value) }; + Self { + key: key.to_string(), + prev, + } + } + } + impl Drop for EnvVarGuard { + fn drop(&mut self) { + match &self.prev { + Some(v) => unsafe { std::env::set_var(&self.key, v) }, + None => unsafe { std::env::remove_var(&self.key) }, + } + } + } + + // Happy-path tests using a mock HTTP server; skip if sandbox network is disabled. + #[tokio::test] + async fn test_fetch_models_happy_path() { + if std::env::var(codex_core::spawn::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR).is_ok() { + tracing::info!( + "{} is set; skipping test_fetch_models_happy_path", + codex_core::spawn::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR + ); + return; + } + + let server = wiremock::MockServer::start().await; + wiremock::Mock::given(wiremock::matchers::method("GET")) + .and(wiremock::matchers::path("/api/tags")) + .respond_with( + wiremock::ResponseTemplate::new(200).set_body_raw( + serde_json::json!({ + "models": [ {"name": "llama3.2:3b"}, {"name":"mistral"} ] + }) + .to_string(), + "application/json", + ), + ) + .mount(&server) + .await; + + let client = OllamaClient::from_host_root(server.uri()); + let models = client.fetch_models().await.expect("fetch models"); + assert!(models.contains(&"llama3.2:3b".to_string())); + assert!(models.contains(&"mistral".to_string())); + } + + #[tokio::test] + async fn test_probe_server_happy_path_openai_compat_and_native() { + if std::env::var(codex_core::spawn::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR).is_ok() { + tracing::info!( + "{} set; skipping test_probe_server_happy_path_openai_compat_and_native", + codex_core::spawn::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR + ); + return; + } + + let server = wiremock::MockServer::start().await; + + // Native endpoint + wiremock::Mock::given(wiremock::matchers::method("GET")) + .and(wiremock::matchers::path("/api/tags")) + .respond_with(wiremock::ResponseTemplate::new(200)) + .mount(&server) + .await; + let native = OllamaClient::from_host_root(server.uri()); + assert!(native.probe_server().await.expect("probe native")); + + // OpenAI compatibility endpoint + wiremock::Mock::given(wiremock::matchers::method("GET")) + .and(wiremock::matchers::path("/v1/models")) + .respond_with(wiremock::ResponseTemplate::new(200)) + .mount(&server) + .await; + // Ensure the built-in OSS provider points at our mock server for this test + // to avoid depending on any globally configured environment from other tests. + let _guard = EnvVarGuard::set("CODEX_OSS_BASE_URL", format!("{}/v1", server.uri())); + let ollama_client = OllamaClient::from_oss_provider(); + assert!(ollama_client.probe_server().await.expect("probe compat")); + } + + #[tokio::test] + async fn test_try_from_oss_provider_ok_when_server_running() { + if std::env::var(codex_core::spawn::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR).is_ok() { + tracing::info!( + "{} set; skipping test_try_from_oss_provider_ok_when_server_running", + codex_core::spawn::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR + ); + return; + } + + let server = wiremock::MockServer::start().await; + // Configure built‑in `oss` provider to point at this mock server. + // set_var is unsafe on Rust 2024 edition; use unsafe block in tests. + let _guard = EnvVarGuard::set("CODEX_OSS_BASE_URL", format!("{}/v1", server.uri())); + + // OpenAI‑compat models endpoint responds OK. + wiremock::Mock::given(wiremock::matchers::method("GET")) + .and(wiremock::matchers::path("/v1/models")) + .respond_with(wiremock::ResponseTemplate::new(200)) + .mount(&server) + .await; + + let _client = OllamaClient::try_from_oss_provider() + .await + .expect("client should be created when probe succeeds"); + } + + #[tokio::test] + async fn test_try_from_oss_provider_err_when_server_missing() { + if std::env::var(codex_core::spawn::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR).is_ok() { + tracing::info!( + "{} set; skipping test_try_from_oss_provider_err_when_server_missing", + codex_core::spawn::CODEX_SANDBOX_NETWORK_DISABLED_ENV_VAR + ); + return; + } + + let server = wiremock::MockServer::start().await; + // Point oss provider at our mock server but do NOT set up a handler + // for /v1/models so the request returns a non‑success status. + unsafe { std::env::set_var("CODEX_OSS_BASE_URL", format!("{}/v1", server.uri())) }; + + let err = OllamaClient::try_from_oss_provider() + .await + .err() + .expect("expected error"); + let msg = err.to_string(); + assert!( + msg.contains("No running Ollama server detected."), + "msg = {msg}" + ); + } +} diff --git a/codex-rs/ollama/src/lib.rs b/codex-rs/ollama/src/lib.rs new file mode 100644 index 0000000000..d6f1e04d1f --- /dev/null +++ b/codex-rs/ollama/src/lib.rs @@ -0,0 +1,52 @@ +mod client; +mod parser; +mod pull; +mod url; + +pub use client::OllamaClient; +pub use pull::CliProgressReporter; +pub use pull::PullEvent; +pub use pull::PullProgressReporter; +pub use pull::TuiProgressReporter; + +/// Default OSS model to use when `--oss` is passed without an explicit `-m`. +pub const DEFAULT_OSS_MODEL: &str = "gpt-oss:20b"; + +/// Prepare the local OSS environment when `--oss` is selected. +/// +/// - Ensures a local Ollama server is reachable. +/// - Selects the final model name (CLI override or default). +/// - Checks if the model exists locally and pulls it if missing. +/// +/// Returns the final model name that should be used by the caller. +pub async fn ensure_oss_ready(cli_model: Option) -> std::io::Result { + // Only download when the requested model is the default OSS model (or when -m is not provided). + let should_download = cli_model + .as_deref() + .map(|name| name == DEFAULT_OSS_MODEL) + .unwrap_or(true); + let model = cli_model.unwrap_or_else(|| DEFAULT_OSS_MODEL.to_string()); + + // Verify local Ollama is reachable. + let ollama_client = crate::OllamaClient::try_from_oss_provider().await?; + + if should_download { + // If the model is not present locally, pull it. + match ollama_client.fetch_models().await { + Ok(models) => { + if !models.iter().any(|m| m == &model) { + let mut reporter = crate::CliProgressReporter::new(); + ollama_client + .pull_with_reporter(&model, &mut reporter) + .await?; + } + } + Err(err) => { + // Not fatal; higher layers may still proceed and surface errors later. + tracing::warn!("Failed to query local models from Ollama: {}.", err); + } + } + } + + Ok(model) +} diff --git a/codex-rs/ollama/src/parser.rs b/codex-rs/ollama/src/parser.rs new file mode 100644 index 0000000000..b3ed2ca8c3 --- /dev/null +++ b/codex-rs/ollama/src/parser.rs @@ -0,0 +1,82 @@ +use serde_json::Value as JsonValue; + +use crate::pull::PullEvent; + +// Convert a single JSON object representing a pull update into one or more events. +pub(crate) fn pull_events_from_value(value: &JsonValue) -> Vec { + let mut events = Vec::new(); + if let Some(status) = value.get("status").and_then(|s| s.as_str()) { + events.push(PullEvent::Status(status.to_string())); + if status == "success" { + events.push(PullEvent::Success); + } + } + let digest = value + .get("digest") + .and_then(|d| d.as_str()) + .unwrap_or("") + .to_string(); + let total = value.get("total").and_then(|t| t.as_u64()); + let completed = value.get("completed").and_then(|t| t.as_u64()); + if total.is_some() || completed.is_some() { + events.push(PullEvent::ChunkProgress { + digest, + total, + completed, + }); + } + events +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_pull_events_decoder_status_and_success() { + let v: JsonValue = serde_json::json!({"status":"verifying"}); + let events = pull_events_from_value(&v); + assert!(matches!(events.as_slice(), [PullEvent::Status(s)] if s == "verifying")); + + let v2: JsonValue = serde_json::json!({"status":"success"}); + let events2 = pull_events_from_value(&v2); + assert_eq!(events2.len(), 2); + assert!(matches!(events2[0], PullEvent::Status(ref s) if s == "success")); + assert!(matches!(events2[1], PullEvent::Success)); + } + + #[test] + fn test_pull_events_decoder_progress() { + let v: JsonValue = serde_json::json!({"digest":"sha256:abc","total":100}); + let events = pull_events_from_value(&v); + assert_eq!(events.len(), 1); + match &events[0] { + PullEvent::ChunkProgress { + digest, + total, + completed, + } => { + assert_eq!(digest, "sha256:abc"); + assert_eq!(*total, Some(100)); + assert_eq!(*completed, None); + } + _ => panic!("expected ChunkProgress"), + } + + let v2: JsonValue = serde_json::json!({"digest":"sha256:def","completed":42}); + let events2 = pull_events_from_value(&v2); + assert_eq!(events2.len(), 1); + match &events2[0] { + PullEvent::ChunkProgress { + digest, + total, + completed, + } => { + assert_eq!(digest, "sha256:def"); + assert_eq!(*total, None); + assert_eq!(*completed, Some(42)); + } + _ => panic!("expected ChunkProgress"), + } + } +} diff --git a/codex-rs/ollama/src/pull.rs b/codex-rs/ollama/src/pull.rs new file mode 100644 index 0000000000..0dd35cd786 --- /dev/null +++ b/codex-rs/ollama/src/pull.rs @@ -0,0 +1,147 @@ +use std::collections::HashMap; +use std::io; +use std::io::Write; + +/// Events emitted while pulling a model from Ollama. +#[derive(Debug, Clone)] +pub enum PullEvent { + /// A human-readable status message (e.g., "verifying", "writing"). + Status(String), + /// Byte-level progress update for a specific layer digest. + ChunkProgress { + digest: String, + total: Option, + completed: Option, + }, + /// The pull finished successfully. + Success, + + /// Error event with a message. + Error(String), +} + +/// A simple observer for pull progress events. Implementations decide how to +/// render progress (CLI, TUI, logs, ...). +pub trait PullProgressReporter { + fn on_event(&mut self, event: &PullEvent) -> io::Result<()>; +} + +/// A minimal CLI reporter that writes inline progress to stderr. +pub struct CliProgressReporter { + printed_header: bool, + last_line_len: usize, + last_completed_sum: u64, + last_instant: std::time::Instant, + totals_by_digest: HashMap, +} + +impl Default for CliProgressReporter { + fn default() -> Self { + Self::new() + } +} + +impl CliProgressReporter { + pub fn new() -> Self { + Self { + printed_header: false, + last_line_len: 0, + last_completed_sum: 0, + last_instant: std::time::Instant::now(), + totals_by_digest: HashMap::new(), + } + } +} + +impl PullProgressReporter for CliProgressReporter { + fn on_event(&mut self, event: &PullEvent) -> io::Result<()> { + let mut out = std::io::stderr(); + match event { + PullEvent::Status(status) => { + // Avoid noisy manifest messages; otherwise show status inline. + if status.eq_ignore_ascii_case("pulling manifest") { + return Ok(()); + } + let pad = self.last_line_len.saturating_sub(status.len()); + let line = format!("\r{status}{}", " ".repeat(pad)); + self.last_line_len = status.len(); + out.write_all(line.as_bytes())?; + out.flush() + } + PullEvent::ChunkProgress { + digest, + total, + completed, + } => { + if let Some(t) = *total { + self.totals_by_digest + .entry(digest.clone()) + .or_insert((0, 0)) + .0 = t; + } + if let Some(c) = *completed { + self.totals_by_digest + .entry(digest.clone()) + .or_insert((0, 0)) + .1 = c; + } + + let (sum_total, sum_completed) = self + .totals_by_digest + .values() + .fold((0u64, 0u64), |acc, (t, c)| (acc.0 + *t, acc.1 + *c)); + if sum_total > 0 { + if !self.printed_header { + let gb = (sum_total as f64) / (1024.0 * 1024.0 * 1024.0); + let header = format!("Downloading model: total {gb:.2} GB\n"); + out.write_all(b"\r\x1b[2K")?; + out.write_all(header.as_bytes())?; + self.printed_header = true; + } + let now = std::time::Instant::now(); + let dt = now + .duration_since(self.last_instant) + .as_secs_f64() + .max(0.001); + let dbytes = sum_completed.saturating_sub(self.last_completed_sum) as f64; + let speed_mb_s = dbytes / (1024.0 * 1024.0) / dt; + self.last_completed_sum = sum_completed; + self.last_instant = now; + + let done_gb = (sum_completed as f64) / (1024.0 * 1024.0 * 1024.0); + let total_gb = (sum_total as f64) / (1024.0 * 1024.0 * 1024.0); + let pct = (sum_completed as f64) * 100.0 / (sum_total as f64); + let text = + format!("{done_gb:.2}/{total_gb:.2} GB ({pct:.1}%) {speed_mb_s:.1} MB/s"); + let pad = self.last_line_len.saturating_sub(text.len()); + let line = format!("\r{text}{}", " ".repeat(pad)); + self.last_line_len = text.len(); + out.write_all(line.as_bytes())?; + out.flush() + } else { + Ok(()) + } + } + PullEvent::Error(_) => { + // This will be handled by the caller, so we don't do anything + // here or the error will be printed twice. + Ok(()) + } + PullEvent::Success => { + out.write_all(b"\n")?; + out.flush() + } + } + } +} + +/// For now the TUI reporter delegates to the CLI reporter. This keeps UI and +/// CLI behavior aligned until a dedicated TUI integration is implemented. +#[derive(Default)] +pub struct TuiProgressReporter(CliProgressReporter); + +impl PullProgressReporter for TuiProgressReporter { + fn on_event(&mut self, event: &PullEvent) -> io::Result<()> { + self.0.on_event(event) + } +} diff --git a/codex-rs/ollama/src/url.rs b/codex-rs/ollama/src/url.rs new file mode 100644 index 0000000000..7c143ce426 --- /dev/null +++ b/codex-rs/ollama/src/url.rs @@ -0,0 +1,39 @@ +/// Identify whether a base_url points at an OpenAI-compatible root (".../v1"). +pub(crate) fn is_openai_compatible_base_url(base_url: &str) -> bool { + base_url.trim_end_matches('/').ends_with("/v1") +} + +/// Convert a provider base_url into the native Ollama host root. +/// For example, "http://localhost:11434/v1" -> "http://localhost:11434". +pub fn base_url_to_host_root(base_url: &str) -> String { + let trimmed = base_url.trim_end_matches('/'); + if trimmed.ends_with("/v1") { + trimmed + .trim_end_matches("/v1") + .trim_end_matches('/') + .to_string() + } else { + trimmed.to_string() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_base_url_to_host_root() { + assert_eq!( + base_url_to_host_root("http://localhost:11434/v1"), + "http://localhost:11434" + ); + assert_eq!( + base_url_to_host_root("http://localhost:11434"), + "http://localhost:11434" + ); + assert_eq!( + base_url_to_host_root("http://localhost:11434/"), + "http://localhost:11434" + ); + } +} diff --git a/codex-rs/tui/Cargo.toml b/codex-rs/tui/Cargo.toml index 60af056a2d..49d843f046 100644 --- a/codex-rs/tui/Cargo.toml +++ b/codex-rs/tui/Cargo.toml @@ -33,6 +33,7 @@ codex-common = { path = "../common", features = [ codex-core = { path = "../core" } codex-file-search = { path = "../file-search" } codex-login = { path = "../login" } +codex-ollama = { path = "../ollama" } color-eyre = "0.6.3" crossterm = { version = "0.28.1", features = ["bracketed-paste"] } image = { version = "^0.25.6", default-features = false, features = ["jpeg"] } diff --git a/codex-rs/tui/src/cli.rs b/codex-rs/tui/src/cli.rs index cb1b725a64..85dffbebb3 100644 --- a/codex-rs/tui/src/cli.rs +++ b/codex-rs/tui/src/cli.rs @@ -17,6 +17,12 @@ pub struct Cli { #[arg(long, short = 'm')] pub model: Option, + /// Convenience flag to select the local open source model provider. + /// Equivalent to -c model_provider=oss; verifies a local Ollama server is + /// running. + #[arg(long = "oss", default_value_t = false)] + pub oss: bool, + /// Configuration profile from config.toml to specify default options. #[arg(long = "profile", short = 'p')] pub config_profile: Option, diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index c619ce8ff0..0b833b13ae 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -3,6 +3,7 @@ // alternate‑screen mode starts; that file opts‑out locally via `allow`. #![deny(clippy::print_stdout, clippy::print_stderr)] use app::App; +use codex_core::BUILT_IN_OSS_MODEL_PROVIDER_ID; use codex_core::config::Config; use codex_core::config::ConfigOverrides; use codex_core::config_types::SandboxMode; @@ -70,18 +71,35 @@ pub async fn run_main( ) }; + let model_provider_override = if cli.oss { + Some(BUILT_IN_OSS_MODEL_PROVIDER_ID.to_owned()) + } else { + None + }; let config = { // Load configuration and support CLI overrides. let overrides = ConfigOverrides { - model: cli.model.clone(), + // When using `--oss`, let the bootstrapper pick the model + // (defaulting to gpt-oss:20b) and ensure it is present locally. + model: if cli.oss { + Some( + codex_ollama::ensure_oss_ready(cli.model.clone()) + .await + .map_err(|e| std::io::Error::other(format!("OSS setup failed: {e}")))?, + ) + } else { + cli.model.clone() + }, approval_policy, sandbox_mode, cwd: cli.cwd.clone().map(|p| p.canonicalize().unwrap_or(p)), - model_provider: None, + model_provider: model_provider_override, config_profile: cli.config_profile.clone(), codex_linux_sandbox_exe, base_instructions: None, include_plan_tool: Some(true), + default_disable_response_storage: cli.oss.then_some(true), + default_show_raw_agent_reasoning: cli.oss.then_some(true), }; // Parse `-c` overrides from the CLI. let cli_kv_overrides = match cli.config_overrides.parse_overrides() { From 9f91b3da243fe05ab63b21db7f75848a7cc4a398 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Tue, 5 Aug 2025 11:39:30 -0700 Subject: [PATCH 072/100] fix: correct spelling error that sneaked through (#1855) I ended up force-pushing https://github.com/openai/codex/pull/1848 because CI jobs were not being triggered after updating the PR on GitHub, so this spelling error sneaked through. --- codex-rs/ollama/src/client.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codex-rs/ollama/src/client.rs b/codex-rs/ollama/src/client.rs index 8a15039fad..f86271dc5e 100644 --- a/codex-rs/ollama/src/client.rs +++ b/codex-rs/ollama/src/client.rs @@ -192,7 +192,7 @@ impl OllamaClient { return Ok(()); } PullEvent::Error(err) => { - // Emperically, ollama returns a 200 OK response even when + // Empirically, ollama returns a 200 OK response even when // the output stream includes an error message. Verify with: // // `curl -i http://localhost:11434/api/pull -d '{ "model": "foobarbaz" }'` From bd24bc320ecd18d214cb194af88555fbf6fbd264 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Tue, 5 Aug 2025 11:44:04 -0700 Subject: [PATCH 073/100] fix: clean out some ASCII (#1856) Similar to https://github.com/openai/codex/pull/1855, this got through. Fixed by running: ``` ./scripts/asciicheck.py README.md ``` --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index dd5e466252..b323d8978d 100644 --- a/README.md +++ b/README.md @@ -189,11 +189,11 @@ they'll be committed to your working directory. ## Using Open Source Models -Codex can run fully locally against an OpenAI‑compatible OSS host (like Ollama) using the `--oss` flag: +Codex can run fully locally against an OpenAI-compatible OSS host (like Ollama) using the `--oss` flag: - Interactive UI: - codex --oss -- Non‑interactive (programmatic) mode: +- Non-interactive (programmatic) mode: - echo "Refactor utils" | codex exec --oss Model selection when using `--oss`: @@ -212,7 +212,7 @@ Point Codex at your own OSS host: - or CODEX_OSS_PORT (when the host is localhost): - CODEX_OSS_PORT=11434 codex --oss -Advanced: you can persist this in your config instead of environment variables by overriding the built‑in `oss` provider in `~/.codex/config.toml`: +Advanced: you can persist this in your config instead of environment variables by overriding the built-in `oss` provider in `~/.codex/config.toml`: ```toml [model_providers.oss] From 0c5fa271bcce88748eb87210588ba3ef88c559dc Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Tue, 5 Aug 2025 11:48:28 -0700 Subject: [PATCH 074/100] fix: README ToC did not match contents (#1857) Similar to https://github.com/openai/codex/pull/1855, this got through. Fixed by running: ``` python3 scripts/readme_toc.py --fix README.md ``` --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b323d8978d..31e87f055a 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ This is the home of the **Codex CLI**, which is a coding agent from OpenAI that - [Quickstart](#quickstart) - [OpenAI API Users](#openai-api-users) - [OpenAI Plus/Pro Users](#openai-pluspro-users) - - [Using OpenAI Open Source Models](#using-open-source-models) +- [Using Open Source Models](#using-open-source-models) - [Why Codex?](#why-codex) - [Security model & permissions](#security-model--permissions) - [Platform sandboxing details](#platform-sandboxing-details) From d365cae0771855d2ac2bddc90ef04b60ca872e7e Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Tue, 5 Aug 2025 13:55:32 -0700 Subject: [PATCH 075/100] fix: when using `--oss`, ensure correct configuration is threaded through correctly (#1859) This PR started as an investigation with the goal of eliminating the use of `unsafe { std::env::set_var() }` in `ollama/src/client.rs`, as setting environment variables in a multithreaded context is indeed unsafe and these tests were observed to be flaky, as a result. Though as I dug deeper into the issue, I discovered that the logic for instantiating `OllamaClient` under test scenarios was not quite right. In this PR, I aimed to: - share more code between the two creation codepaths, `try_from_oss_provider()` and `try_from_provider_with_base_url()` - use the values from `Config` when setting up Ollama, as we have various mechanisms for overriding config values, so we should be sure that we are always using the ultimate `Config` for things such as the `ModelProviderInfo` associated with the `oss` id Once this was in place, `OllamaClient::try_from_provider_with_base_url()` could be used in unit tests for `OllamaClient` so it was possible to create a properly configured client without having to set environment variables. --- codex-rs/core/src/lib.rs | 1 + codex-rs/core/src/model_provider_info.rs | 73 +++++----- codex-rs/exec/src/lib.rs | 30 ++-- codex-rs/ollama/src/client.rs | 177 +++++++++++------------ codex-rs/ollama/src/lib.rs | 42 +++--- codex-rs/tui/src/lib.rs | 31 ++-- 6 files changed, 176 insertions(+), 178 deletions(-) diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index 965cb77bf1..d072613e10 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -32,6 +32,7 @@ pub use model_provider_info::BUILT_IN_OSS_MODEL_PROVIDER_ID; pub use model_provider_info::ModelProviderInfo; pub use model_provider_info::WireApi; pub use model_provider_info::built_in_model_providers; +pub use model_provider_info::create_oss_provider_with_base_url; pub mod model_family; mod models; mod openai_model_info; diff --git a/codex-rs/core/src/model_provider_info.rs b/codex-rs/core/src/model_provider_info.rs index 595f05ef75..db369df3b7 100644 --- a/codex-rs/core/src/model_provider_info.rs +++ b/codex-rs/core/src/model_provider_info.rs @@ -234,23 +234,6 @@ pub const BUILT_IN_OSS_MODEL_PROVIDER_ID: &str = "oss"; pub fn built_in_model_providers() -> HashMap { use ModelProviderInfo as P; - // These CODEX_OSS_ environment variables are experimental: we may - // switch to reading values from config.toml instead. - let codex_oss_base_url = match std::env::var("CODEX_OSS_BASE_URL") - .ok() - .filter(|v| !v.trim().is_empty()) - { - Some(url) => url, - None => format!( - "http://localhost:{port}/v1", - port = std::env::var("CODEX_OSS_PORT") - .ok() - .filter(|v| !v.trim().is_empty()) - .and_then(|v| v.parse::().ok()) - .unwrap_or(DEFAULT_OLLAMA_PORT) - ), - }; - // We do not want to be in the business of adjucating which third-party // providers are bundled with Codex CLI, so we only include the OpenAI and // open source ("oss") providers by default. Users are encouraged to add to @@ -295,29 +278,51 @@ pub fn built_in_model_providers() -> HashMap { requires_auth: true, }, ), - ( - BUILT_IN_OSS_MODEL_PROVIDER_ID, - P { - name: "Open Source".into(), - base_url: Some(codex_oss_base_url), - env_key: None, - env_key_instructions: None, - wire_api: WireApi::Chat, - query_params: None, - http_headers: None, - env_http_headers: None, - request_max_retries: None, - stream_max_retries: None, - stream_idle_timeout_ms: None, - requires_auth: false, - }, - ), + (BUILT_IN_OSS_MODEL_PROVIDER_ID, create_oss_provider()), ] .into_iter() .map(|(k, v)| (k.to_string(), v)) .collect() } +pub fn create_oss_provider() -> ModelProviderInfo { + // These CODEX_OSS_ environment variables are experimental: we may + // switch to reading values from config.toml instead. + let codex_oss_base_url = match std::env::var("CODEX_OSS_BASE_URL") + .ok() + .filter(|v| !v.trim().is_empty()) + { + Some(url) => url, + None => format!( + "http://localhost:{port}/v1", + port = std::env::var("CODEX_OSS_PORT") + .ok() + .filter(|v| !v.trim().is_empty()) + .and_then(|v| v.parse::().ok()) + .unwrap_or(DEFAULT_OLLAMA_PORT) + ), + }; + + create_oss_provider_with_base_url(&codex_oss_base_url) +} + +pub fn create_oss_provider_with_base_url(base_url: &str) -> ModelProviderInfo { + ModelProviderInfo { + name: "gpt-oss".into(), + base_url: Some(base_url.into()), + env_key: None, + env_key_instructions: None, + wire_api: WireApi::Chat, + query_params: None, + http_headers: None, + env_http_headers: None, + request_max_retries: None, + stream_max_retries: None, + stream_idle_timeout_ms: None, + requires_auth: false, + } +} + #[cfg(test)] mod tests { #![allow(clippy::unwrap_used)] diff --git a/codex-rs/exec/src/lib.rs b/codex-rs/exec/src/lib.rs index c1af4f5b45..a0360182b4 100644 --- a/codex-rs/exec/src/lib.rs +++ b/codex-rs/exec/src/lib.rs @@ -22,6 +22,7 @@ use codex_core::protocol::InputItem; use codex_core::protocol::Op; use codex_core::protocol::TaskCompleteEvent; use codex_core::util::is_inside_git_repo; +use codex_ollama::DEFAULT_OSS_MODEL; use event_processor_with_human_output::EventProcessorWithHumanOutput; use event_processor_with_json_output::EventProcessorWithJsonOutput; use tracing::debug; @@ -35,7 +36,7 @@ use crate::event_processor::EventProcessor; pub async fn run_main(cli: Cli, codex_linux_sandbox_exe: Option) -> anyhow::Result<()> { let Cli { images, - model, + model: model_cli_arg, oss, config_profile, full_auto, @@ -119,19 +120,18 @@ pub async fn run_main(cli: Cli, codex_linux_sandbox_exe: Option) -> any // When using `--oss`, let the bootstrapper pick the model (defaulting to // gpt-oss:20b) and ensure it is present locally. Also, force the built‑in // `oss` model provider. - let model_provider_override = if oss { - Some(BUILT_IN_OSS_MODEL_PROVIDER_ID.to_owned()) + let model = if let Some(model) = model_cli_arg { + Some(model) + } else if oss { + Some(DEFAULT_OSS_MODEL.to_owned()) } else { - None + None // No model specified, will use the default. }; - let model = if oss { - Some( - codex_ollama::ensure_oss_ready(model.clone()) - .await - .map_err(|e| anyhow::anyhow!("OSS setup failed: {e}"))?, - ) + + let model_provider = if oss { + Some(BUILT_IN_OSS_MODEL_PROVIDER_ID.to_string()) } else { - model + None // No specific model provider override. }; // Load configuration and determine approval policy @@ -143,7 +143,7 @@ pub async fn run_main(cli: Cli, codex_linux_sandbox_exe: Option) -> any approval_policy: Some(AskForApproval::Never), sandbox_mode, cwd: cwd.map(|p| p.canonicalize().unwrap_or(p)), - model_provider: model_provider_override, + model_provider, codex_linux_sandbox_exe, base_instructions: None, include_plan_tool: None, @@ -170,6 +170,12 @@ pub async fn run_main(cli: Cli, codex_linux_sandbox_exe: Option) -> any )) }; + if oss { + codex_ollama::ensure_oss_ready(&config) + .await + .map_err(|e| anyhow::anyhow!("OSS setup failed: {e}"))?; + } + // Print the effective configuration and prompt so users can see what Codex // is using. event_processor.print_config_summary(&config, &prompt); diff --git a/codex-rs/ollama/src/client.rs b/codex-rs/ollama/src/client.rs index f86271dc5e..6f4621135c 100644 --- a/codex-rs/ollama/src/client.rs +++ b/codex-rs/ollama/src/client.rs @@ -5,13 +5,17 @@ use serde_json::Value as JsonValue; use std::collections::VecDeque; use std::io; -use codex_core::WireApi; - use crate::parser::pull_events_from_value; use crate::pull::PullEvent; use crate::pull::PullProgressReporter; use crate::url::base_url_to_host_root; use crate::url::is_openai_compatible_base_url; +use codex_core::BUILT_IN_OSS_MODEL_PROVIDER_ID; +use codex_core::ModelProviderInfo; +use codex_core::WireApi; +use codex_core::config::Config; + +const OLLAMA_CONNECTION_ERROR: &str = "No running Ollama server detected. Start it with: `ollama serve` (after installing). Install instructions: https://github.com/ollama/ollama?tab=readme-ov-file#ollama"; /// Client for interacting with a local Ollama instance. pub struct OllamaClient { @@ -21,74 +25,77 @@ pub struct OllamaClient { } impl OllamaClient { - pub fn from_oss_provider() -> Self { + /// Construct a client for the built‑in open‑source ("oss") model provider + /// and verify that a local Ollama server is reachable. If no server is + /// detected, returns an error with helpful installation/run instructions. + pub async fn try_from_oss_provider(config: &Config) -> io::Result { + // Note that we must look up the provider from the Config to ensure that + // any overrides the user has in their config.toml are taken into + // account. + let provider = config + .model_providers + .get(BUILT_IN_OSS_MODEL_PROVIDER_ID) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::NotFound, + format!("Built-in provider {BUILT_IN_OSS_MODEL_PROVIDER_ID} not found",), + ) + })?; + + Self::try_from_provider(provider).await + } + + #[cfg(test)] + async fn try_from_provider_with_base_url(base_url: &str) -> io::Result { + let provider = codex_core::create_oss_provider_with_base_url(base_url); + Self::try_from_provider(&provider).await + } + + /// Build a client from a provider definition and verify the server is reachable. + async fn try_from_provider(provider: &ModelProviderInfo) -> io::Result { #![allow(clippy::expect_used)] - // Use the built-in OSS provider's base URL. - let built_in_model_providers = codex_core::built_in_model_providers(); - let provider = built_in_model_providers - .get(codex_core::BUILT_IN_OSS_MODEL_PROVIDER_ID) - .expect("oss provider must exist"); let base_url = provider .base_url .as_ref() .expect("oss provider must have a base_url"); - Self::from_provider(base_url, provider.wire_api) - } - - /// Construct a client for the built‑in open‑source ("oss") model provider - /// and verify that a local Ollama server is reachable. If no server is - /// detected, returns an error with helpful installation/run instructions. - pub async fn try_from_oss_provider() -> io::Result { - let client = Self::from_oss_provider(); - if client.probe_server().await? { - Ok(client) - } else { - Err(io::Error::other( - "No running Ollama server detected. Start it with: `ollama serve` (after installing). Install instructions: https://github.com/ollama/ollama?tab=readme-ov-file#ollama", - )) - } - } - - /// Build a client from a provider definition. Falls back to the default - /// local URL if no base_url is configured. - fn from_provider(base_url: &str, wire_api: WireApi) -> Self { let uses_openai_compat = is_openai_compatible_base_url(base_url) - || matches!(wire_api, WireApi::Chat) && is_openai_compatible_base_url(base_url); + || matches!(provider.wire_api, WireApi::Chat) + && is_openai_compatible_base_url(base_url); let host_root = base_url_to_host_root(base_url); let client = reqwest::Client::builder() .connect_timeout(std::time::Duration::from_secs(5)) .build() .unwrap_or_else(|_| reqwest::Client::new()); - Self { + let client = Self { client, host_root, uses_openai_compat, - } - } - - /// Low-level constructor given a raw host root, e.g. "http://localhost:11434". - #[cfg(test)] - fn from_host_root(host_root: impl Into) -> Self { - let client = reqwest::Client::builder() - .connect_timeout(std::time::Duration::from_secs(5)) - .build() - .unwrap_or_else(|_| reqwest::Client::new()); - Self { - client, - host_root: host_root.into(), - uses_openai_compat: false, - } + }; + client.probe_server().await?; + Ok(client) } /// Probe whether the server is reachable by hitting the appropriate health endpoint. - pub async fn probe_server(&self) -> io::Result { + async fn probe_server(&self) -> io::Result<()> { let url = if self.uses_openai_compat { format!("{}/v1/models", self.host_root.trim_end_matches('/')) } else { format!("{}/api/tags", self.host_root.trim_end_matches('/')) }; - let resp = self.client.get(url).send().await; - Ok(matches!(resp, Ok(r) if r.status().is_success())) + let resp = self.client.get(url).send().await.map_err(|err| { + tracing::warn!("Failed to connect to Ollama server: {err:?}"); + io::Error::other(OLLAMA_CONNECTION_ERROR) + })?; + if resp.status().is_success() { + Ok(()) + } else { + tracing::warn!( + "Failed to probe server at {}: HTTP {}", + self.host_root, + resp.status() + ); + Err(io::Error::other(OLLAMA_CONNECTION_ERROR)) + } } /// Return the list of model names known to the local Ollama instance. @@ -210,6 +217,20 @@ impl OllamaClient { "Pull stream ended unexpectedly without success.", )) } + + /// Low-level constructor given a raw host root, e.g. "http://localhost:11434". + #[cfg(test)] + fn from_host_root(host_root: impl Into) -> Self { + let client = reqwest::Client::builder() + .connect_timeout(std::time::Duration::from_secs(5)) + .build() + .unwrap_or_else(|_| reqwest::Client::new()); + Self { + client, + host_root: host_root.into(), + uses_openai_compat: false, + } + } } #[cfg(test)] @@ -217,34 +238,6 @@ mod tests { #![allow(clippy::expect_used, clippy::unwrap_used)] use super::*; - /// Simple RAII guard to set an environment variable for the duration of a test - /// and restore the previous value (or remove it) on drop to avoid cross-test - /// interference. - struct EnvVarGuard { - key: String, - prev: Option, - } - impl EnvVarGuard { - fn set(key: &str, value: String) -> Self { - let prev = std::env::var(key).ok(); - // set_var is safe but we mirror existing tests that use an unsafe block - // to silence edition lints around global mutation during tests. - unsafe { std::env::set_var(key, value) }; - Self { - key: key.to_string(), - prev, - } - } - } - impl Drop for EnvVarGuard { - fn drop(&mut self) { - match &self.prev { - Some(v) => unsafe { std::env::set_var(&self.key, v) }, - None => unsafe { std::env::remove_var(&self.key) }, - } - } - } - // Happy-path tests using a mock HTTP server; skip if sandbox network is disabled. #[tokio::test] async fn test_fetch_models_happy_path() { @@ -296,7 +289,7 @@ mod tests { .mount(&server) .await; let native = OllamaClient::from_host_root(server.uri()); - assert!(native.probe_server().await.expect("probe native")); + native.probe_server().await.expect("probe native"); // OpenAI compatibility endpoint wiremock::Mock::given(wiremock::matchers::method("GET")) @@ -304,11 +297,14 @@ mod tests { .respond_with(wiremock::ResponseTemplate::new(200)) .mount(&server) .await; - // Ensure the built-in OSS provider points at our mock server for this test - // to avoid depending on any globally configured environment from other tests. - let _guard = EnvVarGuard::set("CODEX_OSS_BASE_URL", format!("{}/v1", server.uri())); - let ollama_client = OllamaClient::from_oss_provider(); - assert!(ollama_client.probe_server().await.expect("probe compat")); + let ollama_client = + OllamaClient::try_from_provider_with_base_url(&format!("{}/v1", server.uri())) + .await + .expect("probe OpenAI compat"); + ollama_client + .probe_server() + .await + .expect("probe OpenAI compat"); } #[tokio::test] @@ -322,9 +318,6 @@ mod tests { } let server = wiremock::MockServer::start().await; - // Configure built‑in `oss` provider to point at this mock server. - // set_var is unsafe on Rust 2024 edition; use unsafe block in tests. - let _guard = EnvVarGuard::set("CODEX_OSS_BASE_URL", format!("{}/v1", server.uri())); // OpenAI‑compat models endpoint responds OK. wiremock::Mock::given(wiremock::matchers::method("GET")) @@ -333,7 +326,7 @@ mod tests { .mount(&server) .await; - let _client = OllamaClient::try_from_oss_provider() + OllamaClient::try_from_provider_with_base_url(&format!("{}/v1", server.uri())) .await .expect("client should be created when probe succeeds"); } @@ -349,18 +342,10 @@ mod tests { } let server = wiremock::MockServer::start().await; - // Point oss provider at our mock server but do NOT set up a handler - // for /v1/models so the request returns a non‑success status. - unsafe { std::env::set_var("CODEX_OSS_BASE_URL", format!("{}/v1", server.uri())) }; - - let err = OllamaClient::try_from_oss_provider() + let err = OllamaClient::try_from_provider_with_base_url(&format!("{}/v1", server.uri())) .await .err() .expect("expected error"); - let msg = err.to_string(); - assert!( - msg.contains("No running Ollama server detected."), - "msg = {msg}" - ); + assert_eq!(OLLAMA_CONNECTION_ERROR, err.to_string()); } } diff --git a/codex-rs/ollama/src/lib.rs b/codex-rs/ollama/src/lib.rs index d6f1e04d1f..0ebf1662ac 100644 --- a/codex-rs/ollama/src/lib.rs +++ b/codex-rs/ollama/src/lib.rs @@ -4,6 +4,7 @@ mod pull; mod url; pub use client::OllamaClient; +use codex_core::config::Config; pub use pull::CliProgressReporter; pub use pull::PullEvent; pub use pull::PullProgressReporter; @@ -15,38 +16,29 @@ pub const DEFAULT_OSS_MODEL: &str = "gpt-oss:20b"; /// Prepare the local OSS environment when `--oss` is selected. /// /// - Ensures a local Ollama server is reachable. -/// - Selects the final model name (CLI override or default). /// - Checks if the model exists locally and pulls it if missing. -/// -/// Returns the final model name that should be used by the caller. -pub async fn ensure_oss_ready(cli_model: Option) -> std::io::Result { +pub async fn ensure_oss_ready(config: &Config) -> std::io::Result<()> { // Only download when the requested model is the default OSS model (or when -m is not provided). - let should_download = cli_model - .as_deref() - .map(|name| name == DEFAULT_OSS_MODEL) - .unwrap_or(true); - let model = cli_model.unwrap_or_else(|| DEFAULT_OSS_MODEL.to_string()); + let model = config.model.as_ref(); // Verify local Ollama is reachable. - let ollama_client = crate::OllamaClient::try_from_oss_provider().await?; + let ollama_client = crate::OllamaClient::try_from_oss_provider(config).await?; - if should_download { - // If the model is not present locally, pull it. - match ollama_client.fetch_models().await { - Ok(models) => { - if !models.iter().any(|m| m == &model) { - let mut reporter = crate::CliProgressReporter::new(); - ollama_client - .pull_with_reporter(&model, &mut reporter) - .await?; - } - } - Err(err) => { - // Not fatal; higher layers may still proceed and surface errors later. - tracing::warn!("Failed to query local models from Ollama: {}.", err); + // If the model is not present locally, pull it. + match ollama_client.fetch_models().await { + Ok(models) => { + if !models.iter().any(|m| m == model) { + let mut reporter = crate::CliProgressReporter::new(); + ollama_client + .pull_with_reporter(model, &mut reporter) + .await?; } } + Err(err) => { + // Not fatal; higher layers may still proceed and surface errors later. + tracing::warn!("Failed to query local models from Ollama: {}.", err); + } } - Ok(model) + Ok(()) } diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index 0b833b13ae..bab728e124 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -10,6 +10,7 @@ use codex_core::config_types::SandboxMode; use codex_core::protocol::AskForApproval; use codex_core::util::is_inside_git_repo; use codex_login::load_auth; +use codex_ollama::DEFAULT_OSS_MODEL; use log_layer::TuiLogLayer; use std::fs::OpenOptions; use std::io::Write; @@ -71,25 +72,27 @@ pub async fn run_main( ) }; + // When using `--oss`, let the bootstrapper pick the model (defaulting to + // gpt-oss:20b) and ensure it is present locally. Also, force the built‑in + // `oss` model provider. + let model = if let Some(model) = &cli.model { + Some(model.clone()) + } else if cli.oss { + Some(DEFAULT_OSS_MODEL.to_owned()) + } else { + None // No model specified, will use the default. + }; + let model_provider_override = if cli.oss { Some(BUILT_IN_OSS_MODEL_PROVIDER_ID.to_owned()) } else { None }; + let config = { // Load configuration and support CLI overrides. let overrides = ConfigOverrides { - // When using `--oss`, let the bootstrapper pick the model - // (defaulting to gpt-oss:20b) and ensure it is present locally. - model: if cli.oss { - Some( - codex_ollama::ensure_oss_ready(cli.model.clone()) - .await - .map_err(|e| std::io::Error::other(format!("OSS setup failed: {e}")))?, - ) - } else { - cli.model.clone() - }, + model, approval_policy, sandbox_mode, cwd: cli.cwd.clone().map(|p| p.canonicalize().unwrap_or(p)), @@ -154,6 +157,12 @@ pub async fn run_main( .with_target(false) .with_filter(env_filter()); + if cli.oss { + codex_ollama::ensure_oss_ready(&config) + .await + .map_err(|e| std::io::Error::other(format!("OSS setup failed: {e}")))?; + } + // Channel that carries formatted log lines to the UI. let (log_tx, log_rx) = tokio::sync::mpsc::unbounded_channel::(); let tui_layer = TuiLogLayer::new(log_tx.clone(), 120).with_filter(env_filter()); From 42bd73e150c887a3ac102409a38d124c2f26ba0a Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Tue, 5 Aug 2025 14:42:49 -0700 Subject: [PATCH 076/100] chore: remove unnecessary default_ prefix (#1854) This prefix is not inline with the other fields on the `ConfigOverrides` struct. --- codex-rs/core/src/config.rs | 12 ++++++------ codex-rs/exec/src/lib.rs | 4 ++-- codex-rs/mcp-server/src/codex_tool_config.rs | 4 ++-- .../src/tool_handlers/create_conversation.rs | 4 ++-- codex-rs/tui/src/lib.rs | 4 ++-- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/codex-rs/core/src/config.rs b/codex-rs/core/src/config.rs index e62fcc39e2..0b53df5ab7 100644 --- a/codex-rs/core/src/config.rs +++ b/codex-rs/core/src/config.rs @@ -385,8 +385,8 @@ pub struct ConfigOverrides { pub codex_linux_sandbox_exe: Option, pub base_instructions: Option, pub include_plan_tool: Option, - pub default_disable_response_storage: Option, - pub default_show_raw_agent_reasoning: Option, + pub disable_response_storage: Option, + pub show_raw_agent_reasoning: Option, } impl Config { @@ -410,8 +410,8 @@ impl Config { codex_linux_sandbox_exe, base_instructions, include_plan_tool, - default_disable_response_storage, - default_show_raw_agent_reasoning, + disable_response_storage, + show_raw_agent_reasoning, } = overrides; let config_profile = match config_profile_key.as_ref().or(cfg.profile.as_ref()) { @@ -529,7 +529,7 @@ impl Config { disable_response_storage: config_profile .disable_response_storage .or(cfg.disable_response_storage) - .or(default_disable_response_storage) + .or(disable_response_storage) .unwrap_or(false), notify: cfg.notify, user_instructions, @@ -546,7 +546,7 @@ impl Config { hide_agent_reasoning: cfg.hide_agent_reasoning.unwrap_or(false), show_raw_agent_reasoning: cfg .show_raw_agent_reasoning - .or(default_show_raw_agent_reasoning) + .or(show_raw_agent_reasoning) .unwrap_or(false), model_reasoning_effort: config_profile .model_reasoning_effort diff --git a/codex-rs/exec/src/lib.rs b/codex-rs/exec/src/lib.rs index a0360182b4..288b6177e5 100644 --- a/codex-rs/exec/src/lib.rs +++ b/codex-rs/exec/src/lib.rs @@ -147,8 +147,8 @@ pub async fn run_main(cli: Cli, codex_linux_sandbox_exe: Option) -> any codex_linux_sandbox_exe, base_instructions: None, include_plan_tool: None, - default_disable_response_storage: oss.then_some(true), - default_show_raw_agent_reasoning: oss.then_some(true), + disable_response_storage: oss.then_some(true), + show_raw_agent_reasoning: oss.then_some(true), }; // Parse `-c` overrides. let cli_kv_overrides = match config_overrides.parse_overrides() { diff --git a/codex-rs/mcp-server/src/codex_tool_config.rs b/codex-rs/mcp-server/src/codex_tool_config.rs index f1a502bbb3..899451a50d 100644 --- a/codex-rs/mcp-server/src/codex_tool_config.rs +++ b/codex-rs/mcp-server/src/codex_tool_config.rs @@ -158,8 +158,8 @@ impl CodexToolCallParam { codex_linux_sandbox_exe, base_instructions, include_plan_tool, - default_disable_response_storage: None, - default_show_raw_agent_reasoning: None, + disable_response_storage: None, + show_raw_agent_reasoning: None, }; let cli_overrides = cli_overrides diff --git a/codex-rs/mcp-server/src/tool_handlers/create_conversation.rs b/codex-rs/mcp-server/src/tool_handlers/create_conversation.rs index c1f4035663..559bf72905 100644 --- a/codex-rs/mcp-server/src/tool_handlers/create_conversation.rs +++ b/codex-rs/mcp-server/src/tool_handlers/create_conversation.rs @@ -59,8 +59,8 @@ pub(crate) async fn handle_create_conversation( codex_linux_sandbox_exe: None, base_instructions, include_plan_tool: None, - default_disable_response_storage: None, - default_show_raw_agent_reasoning: None, + disable_response_storage: None, + show_raw_agent_reasoning: None, }; let cfg: CodexConfig = match CodexConfig::load_with_cli_overrides(cli_overrides, overrides) { diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index bab728e124..50535e5967 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -101,8 +101,8 @@ pub async fn run_main( codex_linux_sandbox_exe, base_instructions: None, include_plan_tool: Some(true), - default_disable_response_storage: cli.oss.then_some(true), - default_show_raw_agent_reasoning: cli.oss.then_some(true), + disable_response_storage: cli.oss.then_some(true), + show_raw_agent_reasoning: cli.oss.then_some(true), }; // Parse `-c` overrides from the CLI. let cli_kv_overrides = match cli.config_overrides.parse_overrides() { From f6c8d1117cfe8b17de3c5f5d077126279be44d8f Mon Sep 17 00:00:00 2001 From: ae Date: Tue, 5 Aug 2025 15:50:06 -0700 Subject: [PATCH 077/100] [feat] make approval key matching case insensitive (#1862) --- codex-rs/tui/src/user_approval_widget.rs | 75 +++++++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) diff --git a/codex-rs/tui/src/user_approval_widget.rs b/codex-rs/tui/src/user_approval_widget.rs index 91febde208..70b355d794 100644 --- a/codex-rs/tui/src/user_approval_widget.rs +++ b/codex-rs/tui/src/user_approval_widget.rs @@ -47,6 +47,8 @@ pub(crate) enum ApprovalRequest { } /// Options displayed in the *select* mode. +/// +/// The `key` is matched case-insensitively. struct SelectOption { label: Line<'static>, description: &'static str, @@ -187,6 +189,16 @@ impl UserApprovalWidget<'_> { } } + /// Normalize a key for comparison. + /// - For `KeyCode::Char`, converts to lowercase for case-insensitive matching. + /// - Other key codes are returned unchanged. + fn normalize_keycode(code: KeyCode) -> KeyCode { + match code { + KeyCode::Char(c) => KeyCode::Char(c.to_ascii_lowercase()), + other => other, + } + } + /// Handle Ctrl-C pressed by the user while the modal is visible. /// Behaves like pressing Escape: abort the request and close the modal. pub(crate) fn on_ctrl_c(&mut self) { @@ -210,7 +222,12 @@ impl UserApprovalWidget<'_> { self.send_decision(ReviewDecision::Abort); } other => { - if let Some(opt) = self.select_options.iter().find(|opt| opt.key == other) { + let normalized = Self::normalize_keycode(other); + if let Some(opt) = self + .select_options + .iter() + .find(|opt| Self::normalize_keycode(opt.key) == normalized) + { self.send_decision(opt.decision); } } @@ -330,3 +347,59 @@ impl WidgetRef for &UserApprovalWidget<'_> { ); } } + +#[cfg(test)] +mod tests { + use super::*; + use crossterm::event::KeyCode; + use crossterm::event::KeyEvent; + use crossterm::event::KeyModifiers; + use std::path::PathBuf; + use std::sync::mpsc::channel; + + #[test] + fn lowercase_shortcut_is_accepted() { + let (tx_raw, rx) = channel::(); + let tx = AppEventSender::new(tx_raw); + let req = ApprovalRequest::Exec { + id: "1".to_string(), + command: vec!["echo".to_string()], + cwd: PathBuf::new(), + reason: None, + }; + let mut widget = UserApprovalWidget::new(req, tx); + widget.handle_key_event(KeyEvent::new(KeyCode::Char('y'), KeyModifiers::NONE)); + assert!(widget.is_complete()); + let events: Vec = rx.try_iter().collect(); + assert!(events.iter().any(|e| matches!( + e, + AppEvent::CodexOp(Op::ExecApproval { + decision: ReviewDecision::Approved, + .. + }) + ))); + } + + #[test] + fn uppercase_shortcut_is_accepted() { + let (tx_raw, rx) = channel::(); + let tx = AppEventSender::new(tx_raw); + let req = ApprovalRequest::Exec { + id: "2".to_string(), + command: vec!["echo".to_string()], + cwd: PathBuf::new(), + reason: None, + }; + let mut widget = UserApprovalWidget::new(req, tx); + widget.handle_key_event(KeyEvent::new(KeyCode::Char('Y'), KeyModifiers::NONE)); + assert!(widget.is_complete()); + let events: Vec = rx.try_iter().collect(); + assert!(events.iter().any(|e| matches!( + e, + AppEvent::CodexOp(Op::ExecApproval { + decision: ReviewDecision::Approved, + .. + }) + ))); + } +} From ea7d3f27bdc1da61df979419515889f64f36c5ce Mon Sep 17 00:00:00 2001 From: Dylan Date: Tue, 5 Aug 2025 17:52:25 -0700 Subject: [PATCH 078/100] [core] Stop escalating timeouts (#1853) ## Summary Escalating out of sandbox is (almost always) not going to fix long-running commands timing out - therefore we should just pass the failure back to the model instead of asking the user to re-run a command that took a long time anyway. ## Testing - [x] Ran locally with a timeout and confirmed this worked as expected --- codex-rs/core/src/codex.rs | 14 ++++++++++++++ codex-rs/core/src/exec.rs | 39 +++++++++++++++++++------------------- 2 files changed, 34 insertions(+), 19 deletions(-) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 0ce0c4ea2b..81ec3cab6e 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -1957,6 +1957,20 @@ async fn handle_sandbox_error( }; } + // similarly, if the command timed out, we can simply return this failure to the model + if matches!(error, SandboxErr::Timeout) { + return ResponseInputItem::FunctionCallOutput { + call_id, + output: FunctionCallOutputPayload { + content: format!( + "command timed out after {} milliseconds", + params.timeout_duration().as_millis() + ), + success: Some(false), + }, + }; + } + // Note that when `error` is `SandboxErr::Denied`, it could be a false // positive. That is, it may have exited with a non-zero exit code, not // because the sandbox denied it, but because that is its expected behavior, diff --git a/codex-rs/core/src/exec.rs b/codex-rs/core/src/exec.rs index dce02cc5e2..e98aeeaece 100644 --- a/codex-rs/core/src/exec.rs +++ b/codex-rs/core/src/exec.rs @@ -51,6 +51,12 @@ pub struct ExecParams { pub env: HashMap, } +impl ExecParams { + pub fn timeout_duration(&self) -> Duration { + Duration::from_millis(self.timeout_ms.unwrap_or(DEFAULT_TIMEOUT_MS)) + } +} + #[derive(Clone, Copy, Debug, PartialEq)] pub enum SandboxType { None, @@ -83,11 +89,9 @@ pub async fn process_exec_tool_call( { SandboxType::None => exec(params, sandbox_policy, ctrl_c, stdout_stream.clone()).await, SandboxType::MacosSeatbelt => { + let timeout = params.timeout_duration(); let ExecParams { - command, - cwd, - timeout_ms, - env, + command, cwd, env, .. } = params; let child = spawn_command_under_seatbelt( command, @@ -97,14 +101,12 @@ pub async fn process_exec_tool_call( env, ) .await?; - consume_truncated_output(child, ctrl_c, timeout_ms, stdout_stream.clone()).await + consume_truncated_output(child, ctrl_c, timeout, stdout_stream.clone()).await } SandboxType::LinuxSeccomp => { + let timeout = params.timeout_duration(); let ExecParams { - command, - cwd, - timeout_ms, - env, + command, cwd, env, .. } = params; let codex_linux_sandbox_exe = codex_linux_sandbox_exe @@ -120,7 +122,7 @@ pub async fn process_exec_tool_call( ) .await?; - consume_truncated_output(child, ctrl_c, timeout_ms, stdout_stream).await + consume_truncated_output(child, ctrl_c, timeout, stdout_stream).await } }; let duration = start.elapsed(); @@ -255,16 +257,16 @@ pub struct ExecToolCallOutput { } async fn exec( - ExecParams { - command, - cwd, - timeout_ms, - env, - }: ExecParams, + params: ExecParams, sandbox_policy: &SandboxPolicy, ctrl_c: Arc, stdout_stream: Option, ) -> Result { + let timeout = params.timeout_duration(); + let ExecParams { + command, cwd, env, .. + } = params; + let (program, args) = command.split_first().ok_or_else(|| { CodexErr::Io(io::Error::new( io::ErrorKind::InvalidInput, @@ -282,7 +284,7 @@ async fn exec( env, ) .await?; - consume_truncated_output(child, ctrl_c, timeout_ms, stdout_stream).await + consume_truncated_output(child, ctrl_c, timeout, stdout_stream).await } /// Consumes the output of a child process, truncating it so it is suitable for @@ -290,7 +292,7 @@ async fn exec( pub(crate) async fn consume_truncated_output( mut child: Child, ctrl_c: Arc, - timeout_ms: Option, + timeout: Duration, stdout_stream: Option, ) -> Result { // Both stdout and stderr were configured with `Stdio::piped()` @@ -324,7 +326,6 @@ pub(crate) async fn consume_truncated_output( )); let interrupted = ctrl_c.notified(); - let timeout = Duration::from_millis(timeout_ms.unwrap_or(DEFAULT_TIMEOUT_MS)); let exit_status = tokio::select! { result = tokio::time::timeout(timeout, child.wait()) => { match result { From afa8f0d6177ad7016b3311f1ce1531fec9466973 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Tue, 5 Aug 2025 19:19:36 -0700 Subject: [PATCH 079/100] fix: exit cleanly when ShutdownComplete is received (#1864) Previous to this PR, `ShutdownComplete` was not being handled correctly in `codex exec`, so it always ended up printing the following to stderr: ``` ERROR codex_exec: Error receiving event: InternalAgentDied ``` Because we were not breaking out of the loop for `ShutdownComplete`, inevitably `codex.next_event()` would get called again and `rx_event.recv()` would fail and the error would get mapped to `InternalAgentDied`: https://github.com/openai/codex/blob/ea7d3f27bdc1da61df979419515889f64f36c5ce/codex-rs/core/src/codex.rs#L190-L197 For reference, https://github.com/openai/codex/pull/1647 introduced the `ShutdownComplete` variant. --- codex-rs/exec/src/lib.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/codex-rs/exec/src/lib.rs b/codex-rs/exec/src/lib.rs index 288b6177e5..06df2aebca 100644 --- a/codex-rs/exec/src/lib.rs +++ b/codex-rs/exec/src/lib.rs @@ -216,10 +216,16 @@ pub async fn run_main(cli: Cli, codex_linux_sandbox_exe: Option) -> any res = codex.next_event() => match res { Ok(event) => { debug!("Received event: {event:?}"); + + let is_shutdown_complete = matches!(event.msg, EventMsg::ShutdownComplete); if let Err(e) = tx.send(event) { error!("Error sending event: {e:?}"); break; } + if is_shutdown_complete { + info!("Received shutdown event, exiting event loop."); + break; + } }, Err(e) => { error!("Error receiving event: {e:?}"); From aff97ed7dda1cdfca3debf048b9b05902fbd4c15 Mon Sep 17 00:00:00 2001 From: Dylan Date: Tue, 5 Aug 2025 19:27:52 -0700 Subject: [PATCH 080/100] [core] Separate tools config from openai client (#1858) ## Summary In an effort to make tools easier to work with and more configurable, I'm introducing `ToolConfig` and updating `Prompt` to take in a general list of Tools. I think this is simpler and better for a few reasons: - We can easily assemble tools from various sources (our own harness, mcp servers, etc.) and we can consolidate the logic for constructing the logic in one place that is separate from serialization. - client.rs no longer needs arbitrary config values, it just takes in a list of tools to serialize A hefty portion of the PR is now updating our conversion of `mcp_types::Tool` to `OpenAITool`, but considering that @bolinfest accurately called this out as a TODO long ago, I think it's time we tackled it. ## Testing - [x] Experimented locally, no changes, as expected - [x] Added additional unit tests - [x] Responded to rust-review --- codex-rs/core/src/chat_completions.rs | 4 +- codex-rs/core/src/client.rs | 7 +- codex-rs/core/src/client_common.rs | 9 +- codex-rs/core/src/codex.rs | 14 +- codex-rs/core/src/lib.rs | 1 - codex-rs/core/src/openai_tools.rs | 277 +++++++++++++++++++++----- codex-rs/core/src/plan_tool.rs | 12 +- 7 files changed, 250 insertions(+), 74 deletions(-) diff --git a/codex-rs/core/src/chat_completions.rs b/codex-rs/core/src/chat_completions.rs index 956dcebda9..98ef7f26cc 100644 --- a/codex-rs/core/src/chat_completions.rs +++ b/codex-rs/core/src/chat_completions.rs @@ -32,7 +32,6 @@ use crate::util::backoff; pub(crate) async fn stream_chat_completions( prompt: &Prompt, model_family: &ModelFamily, - include_plan_tool: bool, client: &reqwest::Client, provider: &ModelProviderInfo, ) -> Result { @@ -112,8 +111,7 @@ pub(crate) async fn stream_chat_completions( } } - let tools_json = - create_tools_json_for_chat_completions_api(prompt, model_family, include_plan_tool)?; + let tools_json = create_tools_json_for_chat_completions_api(&prompt.tools)?; let payload = json!({ "model": model_family.slug, "messages": messages, diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index 514e683e53..e4bb30da26 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -83,7 +83,6 @@ impl ModelClient { let response_stream = stream_chat_completions( prompt, &self.config.model_family, - self.config.include_plan_tool, &self.client, &self.provider, ) @@ -132,11 +131,7 @@ impl ModelClient { let store = prompt.store && auth_mode != Some(AuthMode::ChatGPT); let full_instructions = prompt.get_full_instructions(&self.config.model_family); - let tools_json = create_tools_json_for_responses_api( - prompt, - &self.config.model_family, - self.config.include_plan_tool, - )?; + let tools_json = create_tools_json_for_responses_api(&prompt.tools)?; let reasoning = create_reasoning_param_for_request( &self.config.model_family, self.effort, diff --git a/codex-rs/core/src/client_common.rs b/codex-rs/core/src/client_common.rs index 8b845a52e6..60164f5fde 100644 --- a/codex-rs/core/src/client_common.rs +++ b/codex-rs/core/src/client_common.rs @@ -3,12 +3,12 @@ use crate::config_types::ReasoningSummary as ReasoningSummaryConfig; use crate::error::Result; use crate::model_family::ModelFamily; use crate::models::ResponseItem; +use crate::openai_tools::OpenAiTool; use crate::protocol::TokenUsage; use codex_apply_patch::APPLY_PATCH_TOOL_INSTRUCTIONS; use futures::Stream; use serde::Serialize; use std::borrow::Cow; -use std::collections::HashMap; use std::pin::Pin; use std::task::Context; use std::task::Poll; @@ -33,10 +33,9 @@ pub struct Prompt { /// Whether to store response on server side (disable_response_storage = !store). pub store: bool, - /// Additional tools sourced from external MCP servers. Note each key is - /// the "fully qualified" tool name (i.e., prefixed with the server name), - /// which should be reported to the model in place of Tool::name. - pub extra_tools: HashMap, + /// Tools available to the model, including additional tools sourced from + /// external MCP servers. + pub tools: Vec, /// Optional override for the built-in BASE_INSTRUCTIONS. pub base_instructions_override: Option, diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 81ec3cab6e..9c3a25c200 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -61,6 +61,8 @@ use crate::models::ReasoningItemReasoningSummary; use crate::models::ResponseInputItem; use crate::models::ResponseItem; use crate::models::ShellToolCallParams; +use crate::openai_tools::ToolsConfig; +use crate::openai_tools::get_openai_tools; use crate::plan_tool::handle_update_plan; use crate::project_doc::get_user_instructions; use crate::protocol::AgentMessageDeltaEvent; @@ -216,6 +218,7 @@ pub(crate) struct Session { shell_environment_policy: ShellEnvironmentPolicy, pub(crate) writable_roots: Mutex>, disable_response_storage: bool, + tools_config: ToolsConfig, /// Manager for external MCP servers/tools. mcp_connection_manager: McpConnectionManager, @@ -810,6 +813,7 @@ async fn submission_loop( let default_shell = shell::default_user_shell().await; sess = Some(Arc::new(Session { client, + tools_config: ToolsConfig::new(&config.model_family, config.include_plan_tool), tx_event: tx_event.clone(), ctrl_c: Arc::clone(&ctrl_c), user_instructions, @@ -1204,12 +1208,16 @@ async fn run_turn( sub_id: String, input: Vec, ) -> CodexResult> { - let extra_tools = sess.mcp_connection_manager.list_all_tools(); + let tools = get_openai_tools( + &sess.tools_config, + Some(sess.mcp_connection_manager.list_all_tools()), + ); + let prompt = Prompt { input, user_instructions: sess.user_instructions.clone(), store: !sess.disable_response_storage, - extra_tools, + tools, base_instructions_override: sess.base_instructions.clone(), }; @@ -1436,7 +1444,7 @@ async fn run_compact_task( input: turn_input, user_instructions: None, store: !sess.disable_response_storage, - extra_tools: HashMap::new(), + tools: Vec::new(), base_instructions_override: Some(compact_instructions.clone()), }; diff --git a/codex-rs/core/src/lib.rs b/codex-rs/core/src/lib.rs index d072613e10..c728bd3125 100644 --- a/codex-rs/core/src/lib.rs +++ b/codex-rs/core/src/lib.rs @@ -48,6 +48,5 @@ pub mod spawn; pub mod turn_diff_tracker; mod user_notification; pub mod util; - pub use apply_patch::CODEX_APPLY_PATCH_ARG1; pub use safety::get_platform_sandbox; diff --git a/codex-rs/core/src/openai_tools.rs b/codex-rs/core/src/openai_tools.rs index 7d4bf4aa1a..1dac70819e 100644 --- a/codex-rs/core/src/openai_tools.rs +++ b/codex-rs/core/src/openai_tools.rs @@ -1,22 +1,26 @@ +use serde::Deserialize; use serde::Serialize; use serde_json::json; use std::collections::BTreeMap; +use std::collections::HashMap; -use crate::client_common::Prompt; use crate::model_family::ModelFamily; use crate::plan_tool::PLAN_TOOL; -#[derive(Debug, Clone, Serialize)] -pub(crate) struct ResponsesApiTool { - pub(crate) name: &'static str, - pub(crate) description: &'static str, +#[derive(Debug, Clone, Serialize, PartialEq)] +pub struct ResponsesApiTool { + pub(crate) name: String, + pub(crate) description: String, + /// TODO: Validation. When strict is set to true, the JSON schema, + /// `required` and `additional_properties` must be present. All fields in + /// `properties` must be present in `required`. pub(crate) strict: bool, pub(crate) parameters: JsonSchema, } /// When serialized as JSON, this produces a valid "Tool" in the OpenAI /// Responses API. -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Serialize, PartialEq)] #[serde(tag = "type")] pub(crate) enum OpenAiTool { #[serde(rename = "function")] @@ -25,8 +29,35 @@ pub(crate) enum OpenAiTool { LocalShell {}, } +#[derive(Debug, Clone)] +pub enum ConfigShellToolType { + DefaultShell, + LocalShell, +} + +#[derive(Debug, Clone)] +pub struct ToolsConfig { + pub shell_type: ConfigShellToolType, + pub plan_tool: bool, +} + +impl ToolsConfig { + pub fn new(model_family: &ModelFamily, include_plan_tool: bool) -> Self { + let shell_type = if model_family.uses_local_shell_tool { + ConfigShellToolType::LocalShell + } else { + ConfigShellToolType::DefaultShell + }; + + Self { + shell_type, + plan_tool: include_plan_tool, + } + } +} + /// Generic JSON‑Schema subset needed for our tool definitions -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(tag = "type", rename_all = "lowercase")] pub(crate) enum JsonSchema { String, @@ -36,13 +67,17 @@ pub(crate) enum JsonSchema { }, Object { properties: BTreeMap, - required: &'static [&'static str], - #[serde(rename = "additionalProperties")] - additional_properties: bool, + #[serde(skip_serializing_if = "Option::is_none")] + required: Option>, + #[serde( + rename = "additionalProperties", + skip_serializing_if = "Option::is_none" + )] + additional_properties: Option, }, } -fn create_shell_tool() -> OpenAiTool { +pub(crate) fn create_shell_tool() -> OpenAiTool { let mut properties = BTreeMap::new(); properties.insert( "command".to_string(), @@ -54,13 +89,13 @@ fn create_shell_tool() -> OpenAiTool { properties.insert("timeout".to_string(), JsonSchema::Number); OpenAiTool::Function(ResponsesApiTool { - name: "shell", - description: "Runs a shell command and returns its output", + name: "shell".to_string(), + description: "Runs a shell command and returns its output".to_string(), strict: false, parameters: JsonSchema::Object { properties, - required: &["command"], - additional_properties: false, + required: Some(vec!["command".to_string()]), + additional_properties: Some(false), }, }) } @@ -69,31 +104,13 @@ fn create_shell_tool() -> OpenAiTool { /// Responses API: /// https://platform.openai.com/docs/guides/function-calling?api-mode=responses pub(crate) fn create_tools_json_for_responses_api( - prompt: &Prompt, - model_family: &ModelFamily, - include_plan_tool: bool, + tools: &Vec, ) -> crate::error::Result> { - // Assemble tool list: built-in tools + any extra tools from the prompt. - let mut openai_tools = vec![create_shell_tool()]; - if model_family.uses_local_shell_tool { - openai_tools.push(OpenAiTool::LocalShell {}); - } + let mut tools_json = Vec::new(); - let mut tools_json = Vec::with_capacity(openai_tools.len() + prompt.extra_tools.len() + 1); - for tool in openai_tools.iter() { + for tool in tools { tools_json.push(serde_json::to_value(tool)?); } - tools_json.extend( - prompt - .extra_tools - .clone() - .into_iter() - .map(|(name, tool)| mcp_tool_to_openai_tool(name, tool)), - ); - - if include_plan_tool { - tools_json.push(serde_json::to_value(PLAN_TOOL.clone())?); - } Ok(tools_json) } @@ -102,14 +119,11 @@ pub(crate) fn create_tools_json_for_responses_api( /// Chat Completions API: /// https://platform.openai.com/docs/guides/function-calling?api-mode=chat pub(crate) fn create_tools_json_for_chat_completions_api( - prompt: &Prompt, - model_family: &ModelFamily, - include_plan_tool: bool, + tools: &Vec, ) -> crate::error::Result> { // We start with the JSON for the Responses API and than rewrite it to match // the chat completions tool call format. - let responses_api_tools_json = - create_tools_json_for_responses_api(prompt, model_family, include_plan_tool)?; + let responses_api_tools_json = create_tools_json_for_responses_api(tools)?; let tools_json = responses_api_tools_json .into_iter() .filter_map(|mut tool| { @@ -132,10 +146,10 @@ pub(crate) fn create_tools_json_for_chat_completions_api( Ok(tools_json) } -fn mcp_tool_to_openai_tool( +pub(crate) fn mcp_tool_to_openai_tool( fully_qualified_name: String, tool: mcp_types::Tool, -) -> serde_json::Value { +) -> Result { let mcp_types::Tool { description, mut input_schema, @@ -150,12 +164,175 @@ fn mcp_tool_to_openai_tool( input_schema.properties = Some(serde_json::Value::Object(serde_json::Map::new())); } - // TODO(mbolin): Change the contract of this function to return - // ResponsesApiTool. - json!({ - "name": fully_qualified_name, - "description": description, - "parameters": input_schema, - "type": "function", + let serialized_input_schema = serde_json::to_value(input_schema)?; + let input_schema = serde_json::from_value::(serialized_input_schema)?; + + Ok(ResponsesApiTool { + name: fully_qualified_name, + description: description.unwrap_or_default(), + strict: false, + parameters: input_schema, }) } + +/// Returns a list of OpenAiTools based on the provided config and MCP tools. +/// Note that the keys of mcp_tools should be fully qualified names. See +/// [`McpConnectionManager`] for more details. +pub(crate) fn get_openai_tools( + config: &ToolsConfig, + mcp_tools: Option>, +) -> Vec { + let mut tools: Vec = Vec::new(); + + match config.shell_type { + ConfigShellToolType::DefaultShell => { + tools.push(create_shell_tool()); + } + ConfigShellToolType::LocalShell => { + tools.push(OpenAiTool::LocalShell {}); + } + } + + if config.plan_tool { + tools.push(PLAN_TOOL.clone()); + } + + if let Some(mcp_tools) = mcp_tools { + for (name, tool) in mcp_tools { + match mcp_tool_to_openai_tool(name.clone(), tool.clone()) { + Ok(converted_tool) => tools.push(OpenAiTool::Function(converted_tool)), + Err(e) => { + tracing::error!("Failed to convert {name:?} MCP tool to OpenAI tool: {e:?}"); + } + } + } + } + + tools +} + +#[cfg(test)] +#[allow(clippy::expect_used)] +mod tests { + use crate::model_family::find_family_for_model; + use mcp_types::ToolInputSchema; + + use super::*; + + fn assert_eq_tool_names(tools: &[OpenAiTool], expected_names: &[&str]) { + let tool_names = tools + .iter() + .map(|tool| match tool { + OpenAiTool::Function(ResponsesApiTool { name, .. }) => name, + OpenAiTool::LocalShell {} => "local_shell", + }) + .collect::>(); + + assert_eq!( + tool_names.len(), + expected_names.len(), + "tool_name mismatch, {tool_names:?}, {expected_names:?}", + ); + for (name, expected_name) in tool_names.iter().zip(expected_names.iter()) { + assert_eq!( + name, expected_name, + "tool_name mismatch, {name:?}, {expected_name:?}" + ); + } + } + + #[test] + fn test_get_openai_tools() { + let model_family = find_family_for_model("codex-mini-latest") + .expect("codex-mini-latest should be a valid model family"); + let config = ToolsConfig::new(&model_family, true); + let tools = get_openai_tools(&config, Some(HashMap::new())); + + assert_eq_tool_names(&tools, &["local_shell", "update_plan"]); + } + + #[test] + fn test_get_openai_tools_default_shell() { + let model_family = find_family_for_model("o3").expect("o3 should be a valid model family"); + let config = ToolsConfig::new(&model_family, true); + let tools = get_openai_tools(&config, Some(HashMap::new())); + + assert_eq_tool_names(&tools, &["shell", "update_plan"]); + } + + #[test] + fn test_get_openai_tools_mcp_tools() { + let model_family = find_family_for_model("o3").expect("o3 should be a valid model family"); + let config = ToolsConfig::new(&model_family, false); + let tools = get_openai_tools( + &config, + Some(HashMap::from([( + "test_server/do_something_cool".to_string(), + mcp_types::Tool { + name: "do_something_cool".to_string(), + input_schema: ToolInputSchema { + properties: Some(serde_json::json!({ + "string_argument": { + "type": "string", + }, + "number_argument": { + "type": "number", + }, + "object_argument": { + "type": "object", + "properties": { + "string_property": { "type": "string" }, + "number_property": { "type": "number" }, + }, + "required": [ + "string_property", + "number_property" + ], + "additionalProperties": Some(false), + }, + })), + required: None, + r#type: "object".to_string(), + }, + output_schema: None, + title: None, + annotations: None, + description: Some("Do something cool".to_string()), + }, + )])), + ); + + assert_eq_tool_names(&tools, &["shell", "test_server/do_something_cool"]); + + assert_eq!( + tools[1], + OpenAiTool::Function(ResponsesApiTool { + name: "test_server/do_something_cool".to_string(), + parameters: JsonSchema::Object { + properties: BTreeMap::from([ + ("string_argument".to_string(), JsonSchema::String), + ("number_argument".to_string(), JsonSchema::Number), + ( + "object_argument".to_string(), + JsonSchema::Object { + properties: BTreeMap::from([ + ("string_property".to_string(), JsonSchema::String), + ("number_property".to_string(), JsonSchema::Number), + ]), + required: Some(vec![ + "string_property".to_string(), + "number_property".to_string(), + ]), + additional_properties: Some(false), + }, + ), + ]), + required: None, + additional_properties: None, + }, + description: "Do something cool".to_string(), + strict: false, + }) + ); + } +} diff --git a/codex-rs/core/src/plan_tool.rs b/codex-rs/core/src/plan_tool.rs index dbddb8b5eb..cfc26a4021 100644 --- a/codex-rs/core/src/plan_tool.rs +++ b/codex-rs/core/src/plan_tool.rs @@ -45,8 +45,8 @@ pub(crate) static PLAN_TOOL: LazyLock = LazyLock::new(|| { let plan_items_schema = JsonSchema::Array { items: Box::new(JsonSchema::Object { properties: plan_item_props, - required: &["step", "status"], - additional_properties: false, + required: Some(vec!["step".to_string(), "status".to_string()]), + additional_properties: Some(false), }), }; @@ -55,7 +55,7 @@ pub(crate) static PLAN_TOOL: LazyLock = LazyLock::new(|| { properties.insert("plan".to_string(), plan_items_schema); OpenAiTool::Function(ResponsesApiTool { - name: "update_plan", + name: "update_plan".to_string(), description: r#"Use the update_plan tool to keep the user updated on the current plan for the task. After understanding the user's task, call the update_plan tool with an initial plan. An example of a plan: 1. Explore the codebase to find relevant files (status: in_progress) @@ -66,12 +66,12 @@ Until all the steps are finished, there should always be exactly one in_progress Call the update_plan tool whenever you finish a step, marking the completed step as `completed` and marking the next step as `in_progress`. Before running a command, consider whether or not you have completed the previous step, and make sure to mark it as completed before moving on to the next step. Sometimes, you may need to change plans in the middle of a task: call `update_plan` with the updated plan and make sure to provide an `explanation` of the rationale when doing so. -When all steps are completed, call update_plan one last time with all steps marked as `completed`."#, +When all steps are completed, call update_plan one last time with all steps marked as `completed`."#.to_string(), strict: false, parameters: JsonSchema::Object { properties, - required: &["plan"], - additional_properties: false, + required: Some(vec!["plan".to_string()]), + additional_properties: Some(false), }, }) }); From 725dd6be6a581eb304b6cae786bc848cc780e796 Mon Sep 17 00:00:00 2001 From: Dylan Date: Tue, 5 Aug 2025 20:44:20 -0700 Subject: [PATCH 081/100] [approval_policy] Add OnRequest approval_policy (#1865) ## Summary A split-up PR of #1763 , stacked on top of a tools refactor #1858 to make the change clearer. From the previous summary: > Let's try something new: tell the model about the sandbox, and let it decide when it will need to break the sandbox. Some local testing suggests that it works pretty well with zero iteration on the prompt! ## Testing - [x] Added unit tests - [x] Tested locally and it appears to work smoothly! --- codex-rs/common/src/approval_mode_cli_arg.rs | 4 + codex-rs/config.md | 8 + codex-rs/core/src/codex.rs | 48 +++-- codex-rs/core/src/exec.rs | 2 + codex-rs/core/src/models.rs | 6 + codex-rs/core/src/openai_tools.rs | 194 +++++++++++++++++-- codex-rs/core/src/plan_tool.rs | 13 +- codex-rs/core/src/protocol.rs | 3 + codex-rs/core/src/safety.rs | 67 ++++++- codex-rs/core/src/shell.rs | 2 + codex-rs/core/tests/exec.rs | 2 + codex-rs/core/tests/exec_stream_events.rs | 4 + codex-rs/linux-sandbox/tests/landlock.rs | 4 + 13 files changed, 320 insertions(+), 37 deletions(-) diff --git a/codex-rs/common/src/approval_mode_cli_arg.rs b/codex-rs/common/src/approval_mode_cli_arg.rs index a74ceb2b81..e8c0682687 100644 --- a/codex-rs/common/src/approval_mode_cli_arg.rs +++ b/codex-rs/common/src/approval_mode_cli_arg.rs @@ -18,6 +18,9 @@ pub enum ApprovalModeCliArg { /// will escalate to the user to ask for un-sandboxed execution. OnFailure, + /// The model decides when to ask the user for approval. + OnRequest, + /// Never ask for user approval /// Execution failures are immediately returned to the model. Never, @@ -28,6 +31,7 @@ impl From for AskForApproval { match value { ApprovalModeCliArg::Untrusted => AskForApproval::UnlessTrusted, ApprovalModeCliArg::OnFailure => AskForApproval::OnFailure, + ApprovalModeCliArg::OnRequest => AskForApproval::OnRequest, ApprovalModeCliArg::Never => AskForApproval::Never, } } diff --git a/codex-rs/config.md b/codex-rs/config.md index 992fe1aacc..f93a35ebca 100644 --- a/codex-rs/config.md +++ b/codex-rs/config.md @@ -148,12 +148,20 @@ Determines when the user should be prompted to approve whether Codex can execute approval_policy = "untrusted" ``` +If you want to be notified whenever a command fails, use "on-failure": ```toml # If the command fails when run in the sandbox, Codex asks for permission to # retry the command outside the sandbox. approval_policy = "on-failure" ``` +If you want the model to run until it decides that it needs to ask you for escalated permissions, use "on-request": +```toml +# The model decides when to escalate +approval_policy = "on-request" +``` + +Alternatively, you can have the model run until it is done, and never ask to run a command with escalated permissions: ```toml # User is never prompted: if the command fails, Codex will automatically try # something out. Note the `exec` subcommand always uses this mode. diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 9c3a25c200..a7ab664ee0 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -813,7 +813,12 @@ async fn submission_loop( let default_shell = shell::default_user_shell().await; sess = Some(Arc::new(Session { client, - tools_config: ToolsConfig::new(&config.model_family, config.include_plan_tool), + tools_config: ToolsConfig::new( + &config.model_family, + approval_policy, + sandbox_policy.clone(), + config.include_plan_tool, + ), tx_event: tx_event.clone(), ctrl_c: Arc::clone(&ctrl_c), user_instructions, @@ -1588,6 +1593,8 @@ async fn handle_response_item( command: action.command, workdir: action.working_directory, timeout_ms: action.timeout_ms, + with_escalated_permissions: None, + justification: None, }; let effective_call_id = match (call_id, id) { (Some(call_id), _) => call_id, @@ -1676,6 +1683,8 @@ fn to_exec_params(params: ShellToolCallParams, sess: &Session) -> ExecParams { cwd: sess.resolve_path(params.workdir.clone()), timeout_ms: params.timeout_ms, env: create_env(&sess.shell_environment_policy), + with_escalated_permissions: params.with_escalated_permissions, + justification: params.justification, } } @@ -1776,13 +1785,19 @@ async fn handle_container_exec_with_params( cwd: cwd.clone(), timeout_ms: params.timeout_ms, env: HashMap::new(), + with_escalated_permissions: params.with_escalated_permissions, + justification: params.justification.clone(), }; let safety = if *user_explicitly_approved_this_action { SafetyCheck::AutoApprove { sandbox_type: SandboxType::None, } } else { - assess_safety_for_untrusted_command(sess.approval_policy, &sess.sandbox_policy) + assess_safety_for_untrusted_command( + sess.approval_policy, + &sess.sandbox_policy, + params.with_escalated_permissions.unwrap_or(false), + ) }; ( params, @@ -1798,6 +1813,7 @@ async fn handle_container_exec_with_params( sess.approval_policy, &sess.sandbox_policy, &state.approved_commands, + params.with_escalated_permissions.unwrap_or(false), ) }; let command_for_display = params.command.clone(); @@ -1814,7 +1830,7 @@ async fn handle_container_exec_with_params( call_id.clone(), params.command.clone(), params.cwd.clone(), - None, + params.justification.clone(), ) .await; match rx_approve.await.unwrap_or_default() { @@ -1952,17 +1968,21 @@ async fn handle_sandbox_error( let cwd = exec_command_context.cwd.clone(); let is_apply_patch = exec_command_context.apply_patch.is_some(); - // Early out if the user never wants to be asked for approval; just return to the model immediately - if sess.approval_policy == AskForApproval::Never { - return ResponseInputItem::FunctionCallOutput { - call_id, - output: FunctionCallOutputPayload { - content: format!( - "failed in sandbox {sandbox_type:?} with execution error: {error}" - ), - success: Some(false), - }, - }; + // Early out if either the user never wants to be asked for approval, or + // we're letting the model manage escalation requests. Otherwise, continue + match sess.approval_policy { + AskForApproval::Never | AskForApproval::OnRequest => { + return ResponseInputItem::FunctionCallOutput { + call_id, + output: FunctionCallOutputPayload { + content: format!( + "failed in sandbox {sandbox_type:?} with execution error: {error}" + ), + success: Some(false), + }, + }; + } + AskForApproval::UnlessTrusted | AskForApproval::OnFailure => (), } // similarly, if the command timed out, we can simply return this failure to the model diff --git a/codex-rs/core/src/exec.rs b/codex-rs/core/src/exec.rs index e98aeeaece..10606b6821 100644 --- a/codex-rs/core/src/exec.rs +++ b/codex-rs/core/src/exec.rs @@ -49,6 +49,8 @@ pub struct ExecParams { pub cwd: PathBuf, pub timeout_ms: Option, pub env: HashMap, + pub with_escalated_permissions: Option, + pub justification: Option, } impl ExecParams { diff --git a/codex-rs/core/src/models.rs b/codex-rs/core/src/models.rs index fb48b53070..e052bc43a4 100644 --- a/codex-rs/core/src/models.rs +++ b/codex-rs/core/src/models.rs @@ -191,6 +191,10 @@ pub struct ShellToolCallParams { // The wire format uses `timeout`, which has ambiguous units, so we use // `timeout_ms` as the field name so it is clear in code. pub timeout_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub with_escalated_permissions: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub justification: Option, } #[derive(Debug, Clone, PartialEq)] @@ -302,6 +306,8 @@ mod tests { command: vec!["ls".to_string(), "-l".to_string()], workdir: Some("/tmp".to_string()), timeout_ms: Some(1000), + with_escalated_permissions: None, + justification: None, }, params ); diff --git a/codex-rs/core/src/openai_tools.rs b/codex-rs/core/src/openai_tools.rs index 1dac70819e..1c92c07c12 100644 --- a/codex-rs/core/src/openai_tools.rs +++ b/codex-rs/core/src/openai_tools.rs @@ -6,6 +6,8 @@ use std::collections::HashMap; use crate::model_family::ModelFamily; use crate::plan_tool::PLAN_TOOL; +use crate::protocol::AskForApproval; +use crate::protocol::SandboxPolicy; #[derive(Debug, Clone, Serialize, PartialEq)] pub struct ResponsesApiTool { @@ -32,6 +34,7 @@ pub(crate) enum OpenAiTool { #[derive(Debug, Clone)] pub enum ConfigShellToolType { DefaultShell, + ShellWithRequest { sandbox_policy: SandboxPolicy }, LocalShell, } @@ -42,12 +45,22 @@ pub struct ToolsConfig { } impl ToolsConfig { - pub fn new(model_family: &ModelFamily, include_plan_tool: bool) -> Self { - let shell_type = if model_family.uses_local_shell_tool { + pub fn new( + model_family: &ModelFamily, + approval_policy: AskForApproval, + sandbox_policy: SandboxPolicy, + include_plan_tool: bool, + ) -> Self { + let mut shell_type = if model_family.uses_local_shell_tool { ConfigShellToolType::LocalShell } else { ConfigShellToolType::DefaultShell }; + if matches!(approval_policy, AskForApproval::OnRequest) { + shell_type = ConfigShellToolType::ShellWithRequest { + sandbox_policy: sandbox_policy.clone(), + } + } Self { shell_type, @@ -60,10 +73,23 @@ impl ToolsConfig { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(tag = "type", rename_all = "lowercase")] pub(crate) enum JsonSchema { - String, - Number, + Boolean { + #[serde(skip_serializing_if = "Option::is_none")] + description: Option, + }, + String { + #[serde(skip_serializing_if = "Option::is_none")] + description: Option, + }, + Number { + #[serde(skip_serializing_if = "Option::is_none")] + description: Option, + }, Array { items: Box, + + #[serde(skip_serializing_if = "Option::is_none")] + description: Option, }, Object { properties: BTreeMap, @@ -77,16 +103,23 @@ pub(crate) enum JsonSchema { }, } -pub(crate) fn create_shell_tool() -> OpenAiTool { +fn create_shell_tool() -> OpenAiTool { let mut properties = BTreeMap::new(); properties.insert( "command".to_string(), JsonSchema::Array { - items: Box::new(JsonSchema::String), + items: Box::new(JsonSchema::String { description: None }), + description: None, }, ); - properties.insert("workdir".to_string(), JsonSchema::String); - properties.insert("timeout".to_string(), JsonSchema::Number); + properties.insert( + "workdir".to_string(), + JsonSchema::String { description: None }, + ); + properties.insert( + "timeout".to_string(), + JsonSchema::Number { description: None }, + ); OpenAiTool::Function(ResponsesApiTool { name: "shell".to_string(), @@ -100,6 +133,105 @@ pub(crate) fn create_shell_tool() -> OpenAiTool { }) } +fn create_shell_tool_for_sandbox(sandbox_policy: &SandboxPolicy) -> OpenAiTool { + let mut properties = BTreeMap::new(); + properties.insert( + "command".to_string(), + JsonSchema::Array { + items: Box::new(JsonSchema::String { description: None }), + description: Some("The command to execute".to_string()), + }, + ); + properties.insert( + "workdir".to_string(), + JsonSchema::String { + description: Some("The working directory to execute the command in".to_string()), + }, + ); + properties.insert( + "timeout".to_string(), + JsonSchema::Number { + description: Some("The timeout for the command in milliseconds".to_string()), + }, + ); + + if matches!(sandbox_policy, SandboxPolicy::WorkspaceWrite { .. }) { + properties.insert( + "with_escalated_permissions".to_string(), + JsonSchema::Boolean { + description: Some("Whether to request escalated permissions. Set to true if command needs to be run without sandbox restrictions".to_string()), + }, + ); + properties.insert( + "justification".to_string(), + JsonSchema::String { + description: Some("Only set if ask_for_escalated_permissions is true. 1-sentence explanation of why we want to run this command.".to_string()), + }, + ); + } + + let description = match sandbox_policy { + SandboxPolicy::WorkspaceWrite { + network_access, + .. + } => { + format!( + r#" +The shell tool is used to execute shell commands. +- When invoking the shell tool, your call will be running in a landlock sandbox, and some shell commands will require escalated privileges: + - Types of actions that require escalated privileges: + - Reading files outside the current directory + - Writing files outside the current directory, and protected folders like .git or .env{} + - Examples of commands that require escalated privileges: + - git commit + - npm install or pnpm install + - cargo build + - cargo test +- When invoking a command that will require escalated privileges: + - Provide the with_escalated_permissions parameter with the boolean value true + - Include a short, 1 sentence explanation for why we need to run with_escalated_permissions in the justification parameter."#, + if !network_access { + "\n - Commands that require network access\n" + } else { + "" + } + ) + } + SandboxPolicy::DangerFullAccess => { + "Runs a shell command and returns its output.".to_string() + } + SandboxPolicy::ReadOnly => { + r#" +The shell tool is used to execute shell commands. +- When invoking the shell tool, your call will be running in a landlock sandbox, and some shell commands (including apply_patch) will require escalated permissions: + - Types of actions that require escalated privileges: + - Reading files outside the current directory + - Writing files + - Applying patches + - Examples of commands that require escalated privileges: + - apply_patch + - git commit + - npm install or pnpm install + - cargo build + - cargo test +- When invoking a command that will require escalated privileges: + - Provide the with_escalated_permissions parameter with the boolean value true + - Include a short, 1 sentence explanation for why we need to run with_escalated_permissions in the justification parameter"#.to_string() + } + }; + + OpenAiTool::Function(ResponsesApiTool { + name: "shell".to_string(), + description, + strict: false, + parameters: JsonSchema::Object { + properties, + required: Some(vec!["command".to_string()]), + additional_properties: Some(false), + }, + }) +} + /// Returns JSON values that are compatible with Function Calling in the /// Responses API: /// https://platform.openai.com/docs/guides/function-calling?api-mode=responses @@ -184,10 +316,13 @@ pub(crate) fn get_openai_tools( ) -> Vec { let mut tools: Vec = Vec::new(); - match config.shell_type { + match &config.shell_type { ConfigShellToolType::DefaultShell => { tools.push(create_shell_tool()); } + ConfigShellToolType::ShellWithRequest { sandbox_policy } => { + tools.push(create_shell_tool_for_sandbox(sandbox_policy)); + } ConfigShellToolType::LocalShell => { tools.push(OpenAiTool::LocalShell {}); } @@ -245,7 +380,12 @@ mod tests { fn test_get_openai_tools() { let model_family = find_family_for_model("codex-mini-latest") .expect("codex-mini-latest should be a valid model family"); - let config = ToolsConfig::new(&model_family, true); + let config = ToolsConfig::new( + &model_family, + AskForApproval::Never, + SandboxPolicy::ReadOnly, + true, + ); let tools = get_openai_tools(&config, Some(HashMap::new())); assert_eq_tool_names(&tools, &["local_shell", "update_plan"]); @@ -254,7 +394,12 @@ mod tests { #[test] fn test_get_openai_tools_default_shell() { let model_family = find_family_for_model("o3").expect("o3 should be a valid model family"); - let config = ToolsConfig::new(&model_family, true); + let config = ToolsConfig::new( + &model_family, + AskForApproval::Never, + SandboxPolicy::ReadOnly, + true, + ); let tools = get_openai_tools(&config, Some(HashMap::new())); assert_eq_tool_names(&tools, &["shell", "update_plan"]); @@ -263,7 +408,12 @@ mod tests { #[test] fn test_get_openai_tools_mcp_tools() { let model_family = find_family_for_model("o3").expect("o3 should be a valid model family"); - let config = ToolsConfig::new(&model_family, false); + let config = ToolsConfig::new( + &model_family, + AskForApproval::Never, + SandboxPolicy::ReadOnly, + false, + ); let tools = get_openai_tools( &config, Some(HashMap::from([( @@ -310,14 +460,26 @@ mod tests { name: "test_server/do_something_cool".to_string(), parameters: JsonSchema::Object { properties: BTreeMap::from([ - ("string_argument".to_string(), JsonSchema::String), - ("number_argument".to_string(), JsonSchema::Number), + ( + "string_argument".to_string(), + JsonSchema::String { description: None } + ), + ( + "number_argument".to_string(), + JsonSchema::Number { description: None } + ), ( "object_argument".to_string(), JsonSchema::Object { properties: BTreeMap::from([ - ("string_property".to_string(), JsonSchema::String), - ("number_property".to_string(), JsonSchema::Number), + ( + "string_property".to_string(), + JsonSchema::String { description: None } + ), + ( + "number_property".to_string(), + JsonSchema::Number { description: None } + ), ]), required: Some(vec![ "string_property".to_string(), diff --git a/codex-rs/core/src/plan_tool.rs b/codex-rs/core/src/plan_tool.rs index cfc26a4021..bba5363266 100644 --- a/codex-rs/core/src/plan_tool.rs +++ b/codex-rs/core/src/plan_tool.rs @@ -39,10 +39,14 @@ pub struct UpdatePlanArgs { pub(crate) static PLAN_TOOL: LazyLock = LazyLock::new(|| { let mut plan_item_props = BTreeMap::new(); - plan_item_props.insert("step".to_string(), JsonSchema::String); - plan_item_props.insert("status".to_string(), JsonSchema::String); + plan_item_props.insert("step".to_string(), JsonSchema::String { description: None }); + plan_item_props.insert( + "status".to_string(), + JsonSchema::String { description: None }, + ); let plan_items_schema = JsonSchema::Array { + description: Some("The list of steps".to_string()), items: Box::new(JsonSchema::Object { properties: plan_item_props, required: Some(vec!["step".to_string(), "status".to_string()]), @@ -51,7 +55,10 @@ pub(crate) static PLAN_TOOL: LazyLock = LazyLock::new(|| { }; let mut properties = BTreeMap::new(); - properties.insert("explanation".to_string(), JsonSchema::String); + properties.insert( + "explanation".to_string(), + JsonSchema::String { description: None }, + ); properties.insert("plan".to_string(), plan_items_schema); OpenAiTool::Function(ResponsesApiTool { diff --git a/codex-rs/core/src/protocol.rs b/codex-rs/core/src/protocol.rs index aa330f6bae..9bf85ec49a 100644 --- a/codex-rs/core/src/protocol.rs +++ b/codex-rs/core/src/protocol.rs @@ -150,6 +150,9 @@ pub enum AskForApproval { /// the user to approve execution without a sandbox. OnFailure, + /// The model decides when to ask the user for approval. + OnRequest, + /// Never ask the user to approve commands. Failures are immediately returned /// to the model, and never escalated to the user for approval. Never, diff --git a/codex-rs/core/src/safety.rs b/codex-rs/core/src/safety.rs index 224705f8f3..860a728def 100644 --- a/codex-rs/core/src/safety.rs +++ b/codex-rs/core/src/safety.rs @@ -11,7 +11,7 @@ use crate::is_safe_command::is_known_safe_command; use crate::protocol::AskForApproval; use crate::protocol::SandboxPolicy; -#[derive(Debug)] +#[derive(Debug, PartialEq)] pub enum SafetyCheck { AutoApprove { sandbox_type: SandboxType }, AskUser, @@ -31,7 +31,7 @@ pub fn assess_patch_safety( } match policy { - AskForApproval::OnFailure | AskForApproval::Never => { + AskForApproval::OnFailure | AskForApproval::Never | AskForApproval::OnRequest => { // Continue to see if this can be auto-approved. } // TODO(ragona): I'm not sure this is actually correct? I believe in this case @@ -76,6 +76,7 @@ pub fn assess_command_safety( approval_policy: AskForApproval, sandbox_policy: &SandboxPolicy, approved: &HashSet>, + with_escalated_permissions: bool, ) -> SafetyCheck { // A command is "trusted" because either: // - it belongs to a set of commands we consider "safe" by default, or @@ -96,12 +97,13 @@ pub fn assess_command_safety( }; } - assess_safety_for_untrusted_command(approval_policy, sandbox_policy) + assess_safety_for_untrusted_command(approval_policy, sandbox_policy, with_escalated_permissions) } pub(crate) fn assess_safety_for_untrusted_command( approval_policy: AskForApproval, sandbox_policy: &SandboxPolicy, + with_escalated_permissions: bool, ) -> SafetyCheck { use AskForApproval::*; use SandboxPolicy::*; @@ -113,9 +115,23 @@ pub(crate) fn assess_safety_for_untrusted_command( // commands. SafetyCheck::AskUser } - (OnFailure, DangerFullAccess) | (Never, DangerFullAccess) => SafetyCheck::AutoApprove { + (OnFailure, DangerFullAccess) + | (Never, DangerFullAccess) + | (OnRequest, DangerFullAccess) => SafetyCheck::AutoApprove { sandbox_type: SandboxType::None, }, + (OnRequest, ReadOnly) | (OnRequest, WorkspaceWrite { .. }) => { + if with_escalated_permissions { + SafetyCheck::AskUser + } else { + match get_platform_sandbox() { + Some(sandbox_type) => SafetyCheck::AutoApprove { sandbox_type }, + // Fall back to asking since the command is untrusted and + // we do not have a sandbox available + None => SafetyCheck::AskUser, + } + } + } (Never, ReadOnly) | (Never, WorkspaceWrite { .. }) | (OnFailure, ReadOnly) @@ -264,4 +280,47 @@ mod tests { &cwd, )) } + + #[test] + fn test_request_escalated_privileges() { + // Should not be a trusted command + let command = vec!["git commit".to_string()]; + let approval_policy = AskForApproval::OnRequest; + let sandbox_policy = SandboxPolicy::ReadOnly; + let approved: HashSet> = HashSet::new(); + let request_escalated_privileges = true; + + let safety_check = assess_command_safety( + &command, + approval_policy, + &sandbox_policy, + &approved, + request_escalated_privileges, + ); + + assert_eq!(safety_check, SafetyCheck::AskUser); + } + + #[test] + fn test_request_escalated_privileges_no_sandbox_fallback() { + let command = vec!["git".to_string(), "commit".to_string()]; + let approval_policy = AskForApproval::OnRequest; + let sandbox_policy = SandboxPolicy::ReadOnly; + let approved: HashSet> = HashSet::new(); + let request_escalated_privileges = false; + + let safety_check = assess_command_safety( + &command, + approval_policy, + &sandbox_policy, + &approved, + request_escalated_privileges, + ); + + let expected = match get_platform_sandbox() { + Some(sandbox_type) => SafetyCheck::AutoApprove { sandbox_type }, + None => SafetyCheck::AskUser, + }; + assert_eq!(safety_check, expected); + } } diff --git a/codex-rs/core/src/shell.rs b/codex-rs/core/src/shell.rs index 1e895a3701..de0764f75e 100644 --- a/codex-rs/core/src/shell.rs +++ b/codex-rs/core/src/shell.rs @@ -215,6 +215,8 @@ mod tests { "HOME".to_string(), temp_home.path().to_str().unwrap().to_string(), )]), + with_escalated_permissions: None, + justification: None, }, SandboxType::None, Arc::new(Notify::new()), diff --git a/codex-rs/core/tests/exec.rs b/codex-rs/core/tests/exec.rs index da169296ed..f1b9e78e67 100644 --- a/codex-rs/core/tests/exec.rs +++ b/codex-rs/core/tests/exec.rs @@ -28,6 +28,8 @@ async fn run_test_cmd(tmp: TempDir, cmd: Vec<&str>, should_be_ok: bool) { cwd: tmp.path().to_path_buf(), timeout_ms: Some(1000), env: HashMap::new(), + with_escalated_permissions: None, + justification: None, }; let ctrl_c = Arc::new(Notify::new()); diff --git a/codex-rs/core/tests/exec_stream_events.rs b/codex-rs/core/tests/exec_stream_events.rs index 50f6888f73..534b25513a 100644 --- a/codex-rs/core/tests/exec_stream_events.rs +++ b/codex-rs/core/tests/exec_stream_events.rs @@ -53,6 +53,8 @@ async fn test_exec_stdout_stream_events_echo() { cwd: std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")), timeout_ms: Some(5_000), env: HashMap::new(), + with_escalated_permissions: None, + justification: None, }; let ctrl_c = Arc::new(Notify::new()); @@ -103,6 +105,8 @@ async fn test_exec_stderr_stream_events_echo() { cwd: std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")), timeout_ms: Some(5_000), env: HashMap::new(), + with_escalated_permissions: None, + justification: None, }; let ctrl_c = Arc::new(Notify::new()); diff --git a/codex-rs/linux-sandbox/tests/landlock.rs b/codex-rs/linux-sandbox/tests/landlock.rs index 1375a4c686..041e64e208 100644 --- a/codex-rs/linux-sandbox/tests/landlock.rs +++ b/codex-rs/linux-sandbox/tests/landlock.rs @@ -44,6 +44,8 @@ async fn run_cmd(cmd: &[&str], writable_roots: &[PathBuf], timeout_ms: u64) { cwd: std::env::current_dir().expect("cwd should exist"), timeout_ms: Some(timeout_ms), env: create_env_from_core_vars(), + with_escalated_permissions: None, + justification: None, }; let sandbox_policy = SandboxPolicy::WorkspaceWrite { @@ -139,6 +141,8 @@ async fn assert_network_blocked(cmd: &[&str]) { // do not stall the suite. timeout_ms: Some(NETWORK_TIMEOUT_MS), env: create_env_from_core_vars(), + with_escalated_permissions: None, + justification: None, }; let sandbox_policy = SandboxPolicy::new_read_only_policy(); From 31dcae67dbb9f7941cf3a058ee050ad453cf42db Mon Sep 17 00:00:00 2001 From: aibrahim-oai Date: Tue, 5 Aug 2025 21:32:03 -0700 Subject: [PATCH 082/100] Remove Turndiff and Apply patch from the render (#1868) Make the tui more specific on what to render. Apply patch End and Turn diff needs special handling. Avoiding this issue: image --- codex-rs/tui/src/chatwidget.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 94bd2f121b..7751404f7d 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -492,11 +492,13 @@ impl ChatWidget<'_> { EventMsg::ShutdownComplete => { self.app_event_tx.send(AppEvent::ExitRequest); } - event => { - let text = format!("{event:?}"); - self.add_to_history(HistoryCell::new_background_event(text.clone())); - self.update_latest_log(text); + EventMsg::BackgroundEvent(event) => { + let message = event.message; + self.add_to_history(HistoryCell::new_background_event(message.clone())); + self.update_latest_log(message); } + // TODO: Think of how are we going to render these events. + EventMsg::PatchApplyEnd(_) | EventMsg::TurnDiff(_) => {} } } From b90c15abc42f4a84f9c37482d3c937642a808104 Mon Sep 17 00:00:00 2001 From: ae Date: Tue, 5 Aug 2025 22:01:34 -0700 Subject: [PATCH 083/100] clear terminal on launch (#1870) --- codex-rs/tui/src/tui.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/codex-rs/tui/src/tui.rs b/codex-rs/tui/src/tui.rs index 268483cbcf..e4f85363ad 100644 --- a/codex-rs/tui/src/tui.rs +++ b/codex-rs/tui/src/tui.rs @@ -3,11 +3,14 @@ use std::io::Stdout; use std::io::stdout; use codex_core::config::Config; +use crossterm::cursor::MoveTo; use crossterm::event::DisableBracketedPaste; use crossterm::event::EnableBracketedPaste; use crossterm::event::KeyboardEnhancementFlags; use crossterm::event::PopKeyboardEnhancementFlags; use crossterm::event::PushKeyboardEnhancementFlags; +use crossterm::terminal::Clear; +use crossterm::terminal::ClearType; use ratatui::backend::CrosstermBackend; use ratatui::crossterm::execute; use ratatui::crossterm::terminal::disable_raw_mode; @@ -36,6 +39,12 @@ pub fn init(_config: &Config) -> Result { )?; set_panic_hook(); + // Ensure the UI starts at the top of the terminal by clearing the + // current screen and moving the cursor to (0, 0) before creating the + // Terminal. This makes the initial welcome message render at the very top + // of the viewport, while keeping the normal scrollback history intact. + execute!(stdout(), Clear(ClearType::All), MoveTo(0, 0))?; + let backend = CrosstermBackend::new(stdout()); let tui = Terminal::with_options(backend)?; Ok(tui) From 966d957faf364ebd30df1db991081f9b039bbab4 Mon Sep 17 00:00:00 2001 From: easong-openai Date: Tue, 5 Aug 2025 22:34:14 -0700 Subject: [PATCH 084/100] fixes no git repo warning (#1863) Fix broken git warning broken-screen --- codex-rs/tui/src/app.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index 1142bd87fc..da5410d392 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -233,7 +233,8 @@ impl App<'_> { widget.on_ctrl_c(); } AppState::GitWarning { .. } => { - // No-op. + // Allow exiting the app with Ctrl+C from the warning screen. + self.app_event_tx.send(AppEvent::ExitRequest); } } } @@ -415,7 +416,7 @@ impl App<'_> { let size = terminal.size()?; let desired_height = match &self.app_state { AppState::Chat { widget } => widget.desired_height(size.width), - AppState::GitWarning { .. } => 10, + AppState::GitWarning { .. } => size.height, }; let mut area = terminal.viewport_area; From f8d70d67b6d6d78c91498a34d4c2d6e40fd8bc88 Mon Sep 17 00:00:00 2001 From: easong-openai Date: Tue, 5 Aug 2025 22:35:00 -0700 Subject: [PATCH 085/100] Add OSS model info (#1860) Add somewhat arbitrarily chosen context window/output limit. --- codex-rs/core/src/openai_model_info.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/codex-rs/core/src/openai_model_info.rs b/codex-rs/core/src/openai_model_info.rs index 51f028cbdd..935eb8be4f 100644 --- a/codex-rs/core/src/openai_model_info.rs +++ b/codex-rs/core/src/openai_model_info.rs @@ -14,10 +14,19 @@ pub(crate) struct ModelInfo { pub(crate) max_output_tokens: u64, } -/// Note details such as what a model like gpt-4o is aliased to may be out of -/// date. pub(crate) fn get_model_info(model_family: &ModelFamily) -> Option { match model_family.slug.as_str() { + // OSS models have a 128k shared token pool. + // Arbitrarily splitting it: 3/4 input context, 1/4 output. + // https://openai.com/index/gpt-oss-model-card/ + "gpt-oss-20b" => Some(ModelInfo { + context_window: 96_000, + max_output_tokens: 32_000, + }), + "gpt-oss-120b" => Some(ModelInfo { + context_window: 96_000, + max_output_tokens: 32_000, + }), // https://platform.openai.com/docs/models/o3 "o3" => Some(ModelInfo { context_window: 200_000, From eaf2fb5b4f76f62ede52af9053d827c1959b8c68 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Tue, 5 Aug 2025 22:44:27 -0700 Subject: [PATCH 086/100] fix: fully enumerate EventMsg in chatwidget.rs (#1866) https://github.com/openai/codex/pull/1868 is a related fix that was in flight simultaenously, but after talking to @easong-openai, this: - logs instead of renders for `BackgroundEvent` - logs for `TurnDiff` - renders for `PatchApplyEnd` --- codex-rs/tui/src/chatwidget.rs | 17 ++++--- codex-rs/tui/src/history_cell.rs | 80 ++++++++++++++++++++++++++------ 2 files changed, 78 insertions(+), 19 deletions(-) diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 7751404f7d..86bf765f2c 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -12,6 +12,7 @@ use codex_core::protocol::AgentReasoningEvent; use codex_core::protocol::AgentReasoningRawContentDeltaEvent; use codex_core::protocol::AgentReasoningRawContentEvent; use codex_core::protocol::ApplyPatchApprovalRequestEvent; +use codex_core::protocol::BackgroundEventEvent; use codex_core::protocol::ErrorEvent; use codex_core::protocol::Event; use codex_core::protocol::EventMsg; @@ -25,6 +26,7 @@ use codex_core::protocol::Op; use codex_core::protocol::PatchApplyBeginEvent; use codex_core::protocol::TaskCompleteEvent; use codex_core::protocol::TokenUsage; +use codex_core::protocol::TurnDiffEvent; use crossterm::event::KeyEvent; use crossterm::event::KeyEventKind; use ratatui::buffer::Buffer; @@ -33,6 +35,7 @@ use ratatui::widgets::Widget; use ratatui::widgets::WidgetRef; use tokio::sync::mpsc::UnboundedSender; use tokio::sync::mpsc::unbounded_channel; +use tracing::info; use crate::app_event::AppEvent; use crate::app_event_sender::AppEventSender; @@ -435,6 +438,9 @@ impl ChatWidget<'_> { changes, )); } + EventMsg::PatchApplyEnd(patch_apply_end_event) => { + self.add_to_history(HistoryCell::new_patch_end_event(patch_apply_end_event)); + } EventMsg::ExecCommandEnd(ExecCommandEndEvent { call_id, exit_code, @@ -492,13 +498,12 @@ impl ChatWidget<'_> { EventMsg::ShutdownComplete => { self.app_event_tx.send(AppEvent::ExitRequest); } - EventMsg::BackgroundEvent(event) => { - let message = event.message; - self.add_to_history(HistoryCell::new_background_event(message.clone())); - self.update_latest_log(message); + EventMsg::TurnDiff(TurnDiffEvent { unified_diff }) => { + info!("TurnDiffEvent: {unified_diff}"); + } + EventMsg::BackgroundEvent(BackgroundEventEvent { message }) => { + info!("BackgroundEvent: {message}"); } - // TODO: Think of how are we going to render these events. - EventMsg::PatchApplyEnd(_) | EventMsg::TurnDiff(_) => {} } } diff --git a/codex-rs/tui/src/history_cell.rs b/codex-rs/tui/src/history_cell.rs index 2fb0eecb28..332d90a647 100644 --- a/codex-rs/tui/src/history_cell.rs +++ b/codex-rs/tui/src/history_cell.rs @@ -12,6 +12,7 @@ use codex_core::plan_tool::StepStatus; use codex_core::plan_tool::UpdatePlanArgs; use codex_core::protocol::FileChange; use codex_core::protocol::McpInvocation; +use codex_core::protocol::PatchApplyEndEvent; use codex_core::protocol::SessionConfiguredEvent; use image::DynamicImage; use image::ImageReader; @@ -61,23 +62,35 @@ fn line_to_static(line: &Line) -> Line<'static> { /// scrollable list. pub(crate) enum HistoryCell { /// Welcome message. - WelcomeMessage { view: TextBlock }, + WelcomeMessage { + view: TextBlock, + }, /// Message from the user. - UserPrompt { view: TextBlock }, + UserPrompt { + view: TextBlock, + }, // AgentMessage and AgentReasoning variants were unused and have been removed. /// An exec tool call that has not finished yet. - ActiveExecCommand { view: TextBlock }, + ActiveExecCommand { + view: TextBlock, + }, /// Completed exec tool call. - CompletedExecCommand { view: TextBlock }, + CompletedExecCommand { + view: TextBlock, + }, /// An MCP tool call that has not finished yet. - ActiveMcpToolCall { view: TextBlock }, + ActiveMcpToolCall { + view: TextBlock, + }, /// Completed MCP tool call where we show the result serialized as JSON. - CompletedMcpToolCall { view: TextBlock }, + CompletedMcpToolCall { + view: TextBlock, + }, /// Completed MCP tool call where the result is an image. /// Admittedly, [mcp_types::CallToolResult] can have multiple content types, @@ -87,28 +100,46 @@ pub(crate) enum HistoryCell { // resized version avoids doing the potentially expensive rescale twice // because the scroll-view first calls `height()` for layouting and then // `render_window()` for painting. - CompletedMcpToolCallWithImageOutput { _image: DynamicImage }, + CompletedMcpToolCallWithImageOutput { + _image: DynamicImage, + }, /// Background event. - BackgroundEvent { view: TextBlock }, + BackgroundEvent { + view: TextBlock, + }, /// Output from the `/diff` command. - GitDiffOutput { view: TextBlock }, + GitDiffOutput { + view: TextBlock, + }, /// Error event from the backend. - ErrorEvent { view: TextBlock }, + ErrorEvent { + view: TextBlock, + }, /// Info describing the newly-initialized session. - SessionInfo { view: TextBlock }, + SessionInfo { + view: TextBlock, + }, /// A pending code patch that is awaiting user approval. Mirrors the /// behaviour of `ActiveExecCommand` so the user sees *what* patch the /// model wants to apply before being prompted to approve or deny it. - PendingPatch { view: TextBlock }, + PendingPatch { + view: TextBlock, + }, + + PatchEventEnd { + view: TextBlock, + }, /// A human‑friendly rendering of the model's current plan and step /// statuses provided via the `update_plan` tool. - PlanUpdate { view: TextBlock }, + PlanUpdate { + view: TextBlock, + }, } const TOOL_CALL_MAX_LINES: usize = 5; @@ -128,6 +159,7 @@ impl HistoryCell { | HistoryCell::CompletedExecCommand { view } | HistoryCell::CompletedMcpToolCall { view } | HistoryCell::PendingPatch { view } + | HistoryCell::PatchEventEnd { view } | HistoryCell::PlanUpdate { view } | HistoryCell::ActiveExecCommand { view, .. } | HistoryCell::ActiveMcpToolCall { view, .. } => { @@ -598,6 +630,28 @@ impl HistoryCell { view: TextBlock::new(lines), } } + + pub(crate) fn new_patch_end_event(patch_apply_end_event: PatchApplyEndEvent) -> Self { + let PatchApplyEndEvent { + call_id: _, + stdout: _, + stderr, + success, + } = patch_apply_end_event; + + let mut lines: Vec> = if success { + vec![Line::from("patch applied successfully".italic())] + } else { + let mut lines = vec![Line::from("patch failed".italic())]; + lines.extend(stderr.lines().map(|l| Line::from(l.to_string()))); + lines + }; + lines.push(Line::from("")); + + HistoryCell::PatchEventEnd { + view: TextBlock::new(lines), + } + } } fn create_diff_summary(changes: HashMap) -> Vec { From 1f7003b47633e56a9e8fa50fa4f277f338f36a00 Mon Sep 17 00:00:00 2001 From: ae Date: Tue, 5 Aug 2025 23:02:00 -0700 Subject: [PATCH 087/100] tweak comment (#1871) Belatedly address CR feedback about a comment. ------ https://chatgpt.com/codex/tasks/task_i_6892e8070be4832cba379f2955f5b8bc --- codex-rs/tui/src/tui.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/codex-rs/tui/src/tui.rs b/codex-rs/tui/src/tui.rs index e4f85363ad..e0bf9bcc57 100644 --- a/codex-rs/tui/src/tui.rs +++ b/codex-rs/tui/src/tui.rs @@ -39,10 +39,7 @@ pub fn init(_config: &Config) -> Result { )?; set_panic_hook(); - // Ensure the UI starts at the top of the terminal by clearing the - // current screen and moving the cursor to (0, 0) before creating the - // Terminal. This makes the initial welcome message render at the very top - // of the viewport, while keeping the normal scrollback history intact. + // Clear screen and move cursor to top-left before drawing UI execute!(stdout(), Clear(ClearType::All), MoveTo(0, 0))?; let backend = CrosstermBackend::new(stdout()); From 493e4c94631b7322f43cc08d898469f20c0902b4 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Tue, 5 Aug 2025 23:11:29 -0700 Subject: [PATCH 088/100] fix: only tag as prerelease when the version has an -alpha or -beta suffix (#1872) Hardcoding to `prerelease: true` is a holdover from before we had migrated to the Rust CLI for releases and decided on how we were doing version numbers. To date, I have had to change the release status from "prerelease" to "actual release" manually through the GitHub Releases web page. This is a semi-serious problem because I've discovered that it messes up Homebrew's automation if the version number _looks_ like a real release but turns out to be a prerelease. The release potentially gets skipped from being published on Homebrew, so it's important to set the value correctly from the start. I verified that `steps.release_name.outputs.name` does not include the `rust-v` prefix from the tag name. --- .github/workflows/rust-release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/rust-release.yml b/.github/workflows/rust-release.yml index 3f1c084d91..812d7a3cfe 100644 --- a/.github/workflows/rust-release.yml +++ b/.github/workflows/rust-release.yml @@ -181,9 +181,9 @@ jobs: name: ${{ steps.release_name.outputs.name }} tag_name: ${{ github.ref_name }} files: dist/** - # For now, tag releases as "prerelease" because we are not claiming - # the Rust CLI is stable yet. - prerelease: true + # Mark as prerelease only when the version has a suffix after x.y.z + # (e.g. -alpha, -beta). Otherwise publish a normal release. + prerelease: ${{ contains(steps.release_name.outputs.name, '-') }} - uses: facebook/dotslash-publish-release@v2 env: From 02e796522869071d658f899356608a242e2f85e7 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Tue, 5 Aug 2025 23:33:21 -0700 Subject: [PATCH 089/100] fix: add stricter checks and better error messages to create_github_release.sh (#1874) This script attempts to verify that: - You have no local, uncommitted changes. - You are on `main` - The commit you are on exists on `main` also exists on the origin `https://github.com/openai/codex`, i.e., it is not just a commit you have pushed to your local version of `main` As part of this, try to print better error message if/when these conditions are violated. --- codex-rs/scripts/create_github_release.sh | 28 ++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/codex-rs/scripts/create_github_release.sh b/codex-rs/scripts/create_github_release.sh index 84dcb95fa0..4bc4e25125 100755 --- a/codex-rs/scripts/create_github_release.sh +++ b/codex-rs/scripts/create_github_release.sh @@ -19,7 +19,33 @@ if ! git diff --quiet || ! git diff --cached --quiet || [ -n "$(git ls-files --o fi # Fail if in a detached HEAD state. -CURRENT_BRANCH=$(git symbolic-ref --short -q HEAD) +CURRENT_BRANCH=$(git symbolic-ref --short -q HEAD 2>/dev/null || true) +if [ -z "${CURRENT_BRANCH:-}" ]; then + echo "ERROR: Could not determine the current branch (detached HEAD?)." >&2 + echo " Please run this script from a checked-out branch." >&2 + exit 1 +fi + +# Ensure we are on the 'main' branch before proceeding. +if [ "${CURRENT_BRANCH}" != "main" ]; then + echo "ERROR: Releases must be created from the 'main' branch (current: '${CURRENT_BRANCH}')." >&2 + echo " Please switch to 'main' and try again." >&2 + exit 1 +fi + +# Ensure the current local commit on 'main' is present on 'origin/main'. +# This guarantees we only create releases from commits that are already on +# the canonical repository (https://github.com/openai/codex). +if ! git fetch --quiet origin main; then + echo "ERROR: Failed to fetch 'origin/main'. Ensure the 'origin' remote is configured and reachable." >&2 + exit 1 +fi + +if ! git merge-base --is-ancestor HEAD origin/main; then + echo "ERROR: Your local 'main' HEAD commit is not present on 'origin/main'." >&2 + echo " Please push your commits first (git push origin main) or check out a commit on 'origin/main'." >&2 + exit 1 +fi # Create a new branch for the release and make a commit with the new version. if [ $# -ge 1 ]; then From 7b3ab968a0ee599b4eae03956322caac1642f55c Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Tue, 5 Aug 2025 23:36:10 -0700 Subject: [PATCH 090/100] docs: add more detail to the codex-rust-review (#1875) This PR attempts to break `codex-rust-review.md` into sections so that it is easier to consume. It also adds a healthy new section on "Assertions in Tests" that has been on my mind for awhile. --- .github/codex/labels/codex-rust-review.md | 128 +++++++++++++++++++++- 1 file changed, 122 insertions(+), 6 deletions(-) diff --git a/.github/codex/labels/codex-rust-review.md b/.github/codex/labels/codex-rust-review.md index 2c2893a1fe..ae82d272d7 100644 --- a/.github/codex/labels/codex-rust-review.md +++ b/.github/codex/labels/codex-rust-review.md @@ -6,18 +6,134 @@ Then provide the **review** (1-2 sentences plus bullet points, friendly tone). Things to look out for when doing the review: +## General Principles + - **Make sure the pull request body explains the motivation behind the change.** If the author has failed to do this, call it out, and if you think you can deduce the motivation behind the change, propose copy. - Ideally, the PR body also contains a small summary of the change. For small changes, the PR title may be sufficient. - Each PR should ideally do one conceptual thing. For example, if a PR does a refactoring as well as introducing a new feature, push back and suggest the refactoring be done in a separate PR. This makes things easier for the reviewer, as refactoring changes can often be far-reaching, yet quick to review. -- If the nature of the change seems to have a visual component (which is often the case for changes to `codex-rs/tui`), recommend including a screenshot or video to demonstrate the change, if appropriate. -- Rust files should generally be organized such that the public parts of the API appear near the top of the file and helper functions go below. This is analagous to the "inverted pyramid" structure that is favored in journalism. -- Encourage the use of small enums or the newtype pattern in Rust if it helps readability without adding significant cognitive load or lines of code. -- Be wary of large files and offer suggestions for how to break things into more reasonably-sized files. -- When modifying a `Cargo.toml` file, make sure that dependency lists stay alphabetically sorted. Also consider whether a new dependency is added to the appropriate place (e.g., `[dependencies]` versus `[dev-dependencies]`) -- If you see opportunities for the changes in a diff to use more idiomatic Rust, please make specific recommendations. For example, favor the use of expressions over `return`. - When introducing new code, be on the lookout for code that duplicates existing code. When found, propose a way to refactor the existing code such that it should be reused. + +## Code Organization + - Each create in the Cargo workspace in `codex-rs` has a specific purpose: make a note if you believe new code is not introduced in the correct crate. - When possible, try to keep the `core` crate as small as possible. Non-core but shared logic is often a good candidate for `codex-rs/common`. +- Be wary of large files and offer suggestions for how to break things into more reasonably-sized files. +- Rust files should generally be organized such that the public parts of the API appear near the top of the file and helper functions go below. This is analagous to the "inverted pyramid" structure that is favored in journalism. + +## Assertions in Tests + +Assert the equality of the entire objects instead of doing "piecemeal comparisons," performing `assert_eq!()` on individual fields. + +Note that unit tests also function as "executable documentation." As shown in the following example, "piecemeal comparisons" are often more verbose, provide less coverage, and are not as useful as executable documentation. + +For example, suppose you have the following enum: + +```rust +#[derive(Debug, PartialEq)] +enum Message { + Request { + id: String, + method: String, + params: Option, + }, + Notification { + method: String, + params: Option, + }, +} +``` + +This is an example of a _piecemeal_ comparison: + +```rust +// BAD: Piecemeal Comparison + +#[test] +fn test_get_latest_messages() { + let messages = get_latest_messages(); + assert_eq!(messages.len(), 2); + + let m0 = &messages[0]; + match m0 { + Message::Request { id, method, params } => { + assert_eq!(id, "123"); + assert_eq!(method, "subscribe"); + assert_eq!( + *params, + Some(json!({ + "conversation_id": "x42z86" + })) + ) + } + Message::Notification { .. } => { + panic!("expected Request"); + } + } + + let m1 = &messages[1]; + match m1 { + Message::Request { .. } => { + panic!("expected Notification"); + } + Message::Notification { method, params } => { + assert_eq!(method, "log"); + assert_eq!( + *params, + Some(json!({ + "level": "info", + "message": "subscribed" + })) + ) + } + } +} +``` + +This is a _deep_ comparison: + +```rust +// GOOD: Verify the entire structure with a single assert_eq!(). + +use pretty_assertions::assert_eq; + +#[test] +fn test_get_latest_messages() { + let messages = get_latest_messages(); + + assert_eq!( + vec![ + Message::Request { + id: "123".to_string(), + method: "subscribe".to_string(), + params: Some(json!({ + "conversation_id": "x42z86" + })), + }, + Message::Notification { + method: "log".to_string(), + params: Some(json!({ + "level": "info", + "message": "subscribed" + })), + }, + ], + messages, + ); +} +``` + +## More Tactical Rust Things To Look Out For + +- Do not use `unsafe` (unless you have a really, really good reason like using an operating system API directly and no safe wrapper exists). For example, there are cases where it is tempting to use `unsafe` in order to use `std::env::set_var()`, but this indeed `unsafe` and has led to race conditions on multiple occasions. (When this happens, find a mechanism other than environment variables to use for configuration.) +- Encourage the use of small enums or the newtype pattern in Rust if it helps readability without adding significant cognitive load or lines of code. +- If you see opportunities for the changes in a diff to use more idiomatic Rust, please make specific recommendations. For example, favor the use of expressions over `return`. +- When modifying a `Cargo.toml` file, make sure that dependency lists stay alphabetically sorted. Also consider whether a new dependency is added to the appropriate place (e.g., `[dependencies]` versus `[dev-dependencies]`) + +## Pull Request Body + +- If the nature of the change seems to have a visual component (which is often the case for changes to `codex-rs/tui`), recommend including a screenshot or video to demonstrate the change, if appropriate. - References to existing GitHub issues and PRs are encouraged, where appropriate, though you likely do not have network access, so may not be able to help here. +# PR Information + {CODEX_ACTION_GITHUB_EVENT_PATH} contains the JSON that triggered this GitHub workflow. It contains the `base` and `head` refs that define this PR. Both refs are available locally. From d642b07fcccdd8554316c61b88743378db43252c Mon Sep 17 00:00:00 2001 From: ae Date: Tue, 5 Aug 2025 23:57:52 -0700 Subject: [PATCH 091/100] [feat] add /status slash command (#1873) - Added a `/status` command, which will be useful when we update the home screen to print less status. - Moved `create_config_summary_entries` to common since it's used in a few places. - Noticed we inconsistently had periods in slash command descriptions and just removed them everywhere. - Noticed the diff description was overflowing so made it shorter. --- codex-rs/common/src/config_summary.rs | 29 ++++++++ codex-rs/common/src/lib.rs | 4 + codex-rs/exec/src/event_processor.rs | 26 ------- .../src/event_processor_with_human_output.rs | 2 +- .../src/event_processor_with_json_output.rs | 2 +- codex-rs/tui/src/app.rs | 5 ++ codex-rs/tui/src/chatwidget.rs | 7 ++ codex-rs/tui/src/history_cell.rs | 74 +++++++++++++------ codex-rs/tui/src/slash_command.rs | 12 +-- 9 files changed, 105 insertions(+), 56 deletions(-) create mode 100644 codex-rs/common/src/config_summary.rs diff --git a/codex-rs/common/src/config_summary.rs b/codex-rs/common/src/config_summary.rs new file mode 100644 index 0000000000..39d524731f --- /dev/null +++ b/codex-rs/common/src/config_summary.rs @@ -0,0 +1,29 @@ +use codex_core::WireApi; +use codex_core::config::Config; + +use crate::sandbox_summary::summarize_sandbox_policy; + +/// Build a list of key/value pairs summarizing the effective configuration. +pub fn create_config_summary_entries(config: &Config) -> Vec<(&'static str, String)> { + let mut entries = vec![ + ("workdir", config.cwd.display().to_string()), + ("model", config.model.clone()), + ("provider", config.model_provider_id.clone()), + ("approval", config.approval_policy.to_string()), + ("sandbox", summarize_sandbox_policy(&config.sandbox_policy)), + ]; + if config.model_provider.wire_api == WireApi::Responses + && config.model_family.supports_reasoning_summaries + { + entries.push(( + "reasoning effort", + config.model_reasoning_effort.to_string(), + )); + entries.push(( + "reasoning summaries", + config.model_reasoning_summary.to_string(), + )); + } + + entries +} diff --git a/codex-rs/common/src/lib.rs b/codex-rs/common/src/lib.rs index 3d498a8e2c..38f3832bfd 100644 --- a/codex-rs/common/src/lib.rs +++ b/codex-rs/common/src/lib.rs @@ -23,3 +23,7 @@ mod sandbox_summary; #[cfg(feature = "sandbox_summary")] pub use sandbox_summary::summarize_sandbox_policy; + +mod config_summary; + +pub use config_summary::create_config_summary_entries; diff --git a/codex-rs/exec/src/event_processor.rs b/codex-rs/exec/src/event_processor.rs index 0a2a141eca..b7b3c27dc5 100644 --- a/codex-rs/exec/src/event_processor.rs +++ b/codex-rs/exec/src/event_processor.rs @@ -1,7 +1,5 @@ use std::path::Path; -use codex_common::summarize_sandbox_policy; -use codex_core::WireApi; use codex_core::config::Config; use codex_core::protocol::Event; @@ -19,30 +17,6 @@ pub(crate) trait EventProcessor { fn process_event(&mut self, event: Event) -> CodexStatus; } -pub(crate) fn create_config_summary_entries(config: &Config) -> Vec<(&'static str, String)> { - let mut entries = vec![ - ("workdir", config.cwd.display().to_string()), - ("model", config.model.clone()), - ("provider", config.model_provider_id.clone()), - ("approval", config.approval_policy.to_string()), - ("sandbox", summarize_sandbox_policy(&config.sandbox_policy)), - ]; - if config.model_provider.wire_api == WireApi::Responses - && config.model_family.supports_reasoning_summaries - { - entries.push(( - "reasoning effort", - config.model_reasoning_effort.to_string(), - )); - entries.push(( - "reasoning summaries", - config.model_reasoning_summary.to_string(), - )); - } - - entries -} - pub(crate) fn handle_last_message(last_agent_message: Option<&str>, output_file: &Path) { let message = last_agent_message.unwrap_or_default(); write_last_message_file(message, Some(output_file)); diff --git a/codex-rs/exec/src/event_processor_with_human_output.rs b/codex-rs/exec/src/event_processor_with_human_output.rs index 393ef4ab1b..6b03ed7882 100644 --- a/codex-rs/exec/src/event_processor_with_human_output.rs +++ b/codex-rs/exec/src/event_processor_with_human_output.rs @@ -33,8 +33,8 @@ use std::time::Instant; use crate::event_processor::CodexStatus; use crate::event_processor::EventProcessor; -use crate::event_processor::create_config_summary_entries; use crate::event_processor::handle_last_message; +use codex_common::create_config_summary_entries; /// This should be configurable. When used in CI, users may not want to impose /// a limit so they can see the full transcript. diff --git a/codex-rs/exec/src/event_processor_with_json_output.rs b/codex-rs/exec/src/event_processor_with_json_output.rs index 1d153add6e..76985518e6 100644 --- a/codex-rs/exec/src/event_processor_with_json_output.rs +++ b/codex-rs/exec/src/event_processor_with_json_output.rs @@ -9,8 +9,8 @@ use serde_json::json; use crate::event_processor::CodexStatus; use crate::event_processor::EventProcessor; -use crate::event_processor::create_config_summary_entries; use crate::event_processor::handle_last_message; +use codex_common::create_config_summary_entries; pub(crate) struct EventProcessorWithJsonOutput { last_message_path: Option, diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index da5410d392..eee2a61c2e 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -330,6 +330,11 @@ impl App<'_> { widget.add_diff_output(text); } } + SlashCommand::Status => { + if let AppState::Chat { widget } = &mut self.app_state { + widget.add_status_output(); + } + } #[cfg(debug_assertions)] SlashCommand::TestApproval => { use std::collections::HashMap; diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 86bf765f2c..6d03be783b 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -522,6 +522,13 @@ impl ChatWidget<'_> { self.add_to_history(HistoryCell::new_diff_output(diff_output.clone())); } + pub(crate) fn add_status_output(&mut self) { + self.add_to_history(HistoryCell::new_status_output( + &self.config, + &self.token_usage, + )); + } + /// Forward file-search results to the bottom pane. pub(crate) fn apply_file_search_result(&mut self, query: String, matches: Vec) { self.bottom_pane.on_file_search_result(query, matches); diff --git a/codex-rs/tui/src/history_cell.rs b/codex-rs/tui/src/history_cell.rs index 332d90a647..5b7d9246f7 100644 --- a/codex-rs/tui/src/history_cell.rs +++ b/codex-rs/tui/src/history_cell.rs @@ -3,9 +3,8 @@ use crate::text_block::TextBlock; use crate::text_formatting::format_and_truncate_tool_result; use base64::Engine; use codex_ansi_escape::ansi_escape_line; +use codex_common::create_config_summary_entries; use codex_common::elapsed::format_duration; -use codex_common::summarize_sandbox_policy; -use codex_core::WireApi; use codex_core::config::Config; use codex_core::plan_tool::PlanItemArg; use codex_core::plan_tool::StepStatus; @@ -14,6 +13,7 @@ use codex_core::protocol::FileChange; use codex_core::protocol::McpInvocation; use codex_core::protocol::PatchApplyEndEvent; use codex_core::protocol::SessionConfiguredEvent; +use codex_core::protocol::TokenUsage; use image::DynamicImage; use image::ImageReader; use mcp_types::EmbeddedResourceResource; @@ -114,6 +114,11 @@ pub(crate) enum HistoryCell { view: TextBlock, }, + /// Output from the `/status` command. + StatusOutput { + view: TextBlock, + }, + /// Error event from the backend. ErrorEvent { view: TextBlock, @@ -154,6 +159,7 @@ impl HistoryCell { | HistoryCell::UserPrompt { view } | HistoryCell::BackgroundEvent { view } | HistoryCell::GitDiffOutput { view } + | HistoryCell::StatusOutput { view } | HistoryCell::ErrorEvent { view } | HistoryCell::SessionInfo { view } | HistoryCell::CompletedExecCommand { view } @@ -200,26 +206,7 @@ impl HistoryCell { ]), ]; - let mut entries = vec![ - ("workdir", config.cwd.display().to_string()), - ("model", config.model.clone()), - ("provider", config.model_provider_id.clone()), - ("approval", config.approval_policy.to_string()), - ("sandbox", summarize_sandbox_policy(&config.sandbox_policy)), - ]; - if config.model_provider.wire_api == WireApi::Responses - && config.model_family.supports_reasoning_summaries - { - entries.push(( - "reasoning effort", - config.model_reasoning_effort.to_string(), - )); - entries.push(( - "reasoning summaries", - config.model_reasoning_summary.to_string(), - )); - } - for (key, value) in entries { + for (key, value) in create_config_summary_entries(config) { lines.push(Line::from(vec![format!("{key}: ").bold(), value.into()])); } lines.push(Line::from("")); @@ -476,6 +463,49 @@ impl HistoryCell { } } + pub(crate) fn new_status_output(config: &Config, usage: &TokenUsage) -> Self { + let mut lines: Vec> = Vec::new(); + lines.push(Line::from("/status".magenta())); + + // Config + for (key, value) in create_config_summary_entries(config) { + lines.push(Line::from(vec![format!("{key}: ").bold(), value.into()])); + } + + // Token usage + lines.push(Line::from("")); + lines.push(Line::from("token usage".bold())); + lines.push(Line::from(vec![ + " input: ".bold(), + usage.input_tokens.to_string().into(), + ])); + lines.push(Line::from(vec![ + " cached input: ".bold(), + usage.cached_input_tokens.unwrap_or(0).to_string().into(), + ])); + lines.push(Line::from(vec![ + " output: ".bold(), + usage.output_tokens.to_string().into(), + ])); + lines.push(Line::from(vec![ + " reasoning output: ".bold(), + usage + .reasoning_output_tokens + .unwrap_or(0) + .to_string() + .into(), + ])); + lines.push(Line::from(vec![ + " total: ".bold(), + usage.total_tokens.to_string().into(), + ])); + + lines.push(Line::from("")); + HistoryCell::StatusOutput { + view: TextBlock::new(lines), + } + } + pub(crate) fn new_error_event(message: String) -> Self { let lines: Vec> = vec![ vec!["ERROR: ".red().bold(), message.into()].into(), diff --git a/codex-rs/tui/src/slash_command.rs b/codex-rs/tui/src/slash_command.rs index d82a16608f..85dde7a113 100644 --- a/codex-rs/tui/src/slash_command.rs +++ b/codex-rs/tui/src/slash_command.rs @@ -15,6 +15,7 @@ pub enum SlashCommand { New, Compact, Diff, + Status, Quit, #[cfg(debug_assertions)] TestApproval, @@ -24,12 +25,11 @@ impl SlashCommand { /// User-visible description shown in the popup. pub fn description(self) -> &'static str { match self { - SlashCommand::New => "Start a new chat.", - SlashCommand::Compact => "Compact the chat history.", - SlashCommand::Quit => "Exit the application.", - SlashCommand::Diff => { - "Show git diff of the working directory (including untracked files)" - } + SlashCommand::New => "Start a new chat", + SlashCommand::Compact => "Compact the chat history", + SlashCommand::Quit => "Exit the application", + SlashCommand::Diff => "Show git diff (including untracked files)", + SlashCommand::Status => "Show current session configuration and token usage", #[cfg(debug_assertions)] SlashCommand::TestApproval => "Test approval request", } From cda39e417fb3c4d91f02ccc93acc296b77f5b947 Mon Sep 17 00:00:00 2001 From: Dylan Date: Wed, 6 Aug 2025 00:07:58 -0700 Subject: [PATCH 092/100] [tests] Investigate flakey mcp-server test (#1877) ## Summary Have seen these tests flaking over the course of today on different boxes. `wiremock` seems to be generally written with tokio/threads in mind but based on the weird panics from the tests, let's see if this helps. --- codex-rs/mcp-server/tests/send_message.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codex-rs/mcp-server/tests/send_message.rs b/codex-rs/mcp-server/tests/send_message.rs index fd4b210b0b..fd3718e8f3 100644 --- a/codex-rs/mcp-server/tests/send_message.rs +++ b/codex-rs/mcp-server/tests/send_message.rs @@ -18,7 +18,7 @@ use tokio::time::timeout; const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[tokio::test] async fn test_send_message_success() { // Spin up a mock completions server that immediately ends the Codex turn. // Two Codex turns hit the mock model (session start + send-user-message). Provide two SSE responses. @@ -105,7 +105,7 @@ async fn test_send_message_success() { drop(server); } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[tokio::test] async fn test_send_message_session_not_found() { // Start MCP without creating a Codex session let codex_home = TempDir::new().expect("tempdir"); From 3e8bcf0247ee37a226dd3c8b1e7e6695ccb9321b Mon Sep 17 00:00:00 2001 From: Dylan Date: Wed, 6 Aug 2025 01:13:31 -0700 Subject: [PATCH 093/100] [prompts] Add (#1869) ## Summary Includes a new user message in the api payload which provides useful environment context for the model, so it knows about things like the current working directory and the sandbox. ## Testing Updated unit tests --- codex-rs/core/src/chat_completions.rs | 6 +- codex-rs/core/src/client.rs | 11 +--- codex-rs/core/src/client_common.rs | 77 ++++++++++++++++++++++- codex-rs/core/src/codex.rs | 9 +++ codex-rs/core/src/git_info.rs | 2 +- codex-rs/core/src/protocol.rs | 3 +- codex-rs/core/tests/client.rs | 47 ++++++++++---- codex-rs/mcp-server/tests/send_message.rs | 2 +- 8 files changed, 126 insertions(+), 31 deletions(-) diff --git a/codex-rs/core/src/chat_completions.rs b/codex-rs/core/src/chat_completions.rs index 98ef7f26cc..dae140bc02 100644 --- a/codex-rs/core/src/chat_completions.rs +++ b/codex-rs/core/src/chat_completions.rs @@ -41,11 +41,9 @@ pub(crate) async fn stream_chat_completions( let full_instructions = prompt.get_full_instructions(model_family); messages.push(json!({"role": "system", "content": full_instructions})); - if let Some(instr) = &prompt.get_formatted_user_instructions() { - messages.push(json!({"role": "user", "content": instr})); - } + let input = prompt.get_formatted_input(); - for item in &prompt.input { + for item in &input { match item { ResponseItem::Message { role, content, .. } => { let mut text = String::new(); diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index e4bb30da26..9748cde7cb 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -34,7 +34,6 @@ use crate::error::Result; use crate::flags::CODEX_RS_SSE_FIXTURE; use crate::model_provider_info::ModelProviderInfo; use crate::model_provider_info::WireApi; -use crate::models::ContentItem; use crate::models::ResponseItem; use crate::openai_tools::create_tools_json_for_responses_api; use crate::protocol::TokenUsage; @@ -146,15 +145,7 @@ impl ModelClient { vec![] }; - let mut input_with_instructions = Vec::with_capacity(prompt.input.len() + 1); - if let Some(ui) = prompt.get_formatted_user_instructions() { - input_with_instructions.push(ResponseItem::Message { - id: None, - role: "user".to_string(), - content: vec![ContentItem::InputText { text: ui }], - }); - } - input_with_instructions.extend(prompt.input.clone()); + let input_with_instructions = prompt.get_formatted_input(); let payload = ResponsesApiRequest { model: &self.config.model, diff --git a/codex-rs/core/src/client_common.rs b/codex-rs/core/src/client_common.rs index 60164f5fde..2ca060f4a4 100644 --- a/codex-rs/core/src/client_common.rs +++ b/codex-rs/core/src/client_common.rs @@ -1,14 +1,20 @@ use crate::config_types::ReasoningEffort as ReasoningEffortConfig; use crate::config_types::ReasoningSummary as ReasoningSummaryConfig; use crate::error::Result; +use crate::git_info::GitInfo; use crate::model_family::ModelFamily; +use crate::models::ContentItem; use crate::models::ResponseItem; use crate::openai_tools::OpenAiTool; +use crate::protocol::AskForApproval; +use crate::protocol::SandboxPolicy; use crate::protocol::TokenUsage; use codex_apply_patch::APPLY_PATCH_TOOL_INSTRUCTIONS; use futures::Stream; use serde::Serialize; use std::borrow::Cow; +use std::fmt::Display; +use std::path::PathBuf; use std::pin::Pin; use std::task::Context; use std::task::Poll; @@ -18,10 +24,49 @@ use tokio::sync::mpsc; /// with this content. const BASE_INSTRUCTIONS: &str = include_str!("../prompt.md"); +/// wraps environment context message in a tag for the model to parse more easily. +const ENVIRONMENT_CONTEXT_START: &str = "\n\n"; +const ENVIRONMENT_CONTEXT_END: &str = "\n\n"; + /// wraps user instructions message in a tag for the model to parse more easily. const USER_INSTRUCTIONS_START: &str = "\n\n"; const USER_INSTRUCTIONS_END: &str = "\n\n"; +#[derive(Debug, Clone)] +pub(crate) struct EnvironmentContext { + pub cwd: PathBuf, + pub git_info: Option, + pub approval_policy: AskForApproval, + pub sandbox_policy: SandboxPolicy, +} + +impl Display for EnvironmentContext { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + writeln!( + f, + "Current working directory: {}", + self.cwd.to_string_lossy() + )?; + writeln!(f, "Is directory a git repo: {}", self.git_info.is_some())?; + writeln!(f, "Approval policy: {}", self.approval_policy)?; + writeln!(f, "Sandbox policy: {}", self.sandbox_policy)?; + + let network_access = match self.sandbox_policy.clone() { + SandboxPolicy::DangerFullAccess => "enabled", + SandboxPolicy::ReadOnly => "restricted", + SandboxPolicy::WorkspaceWrite { network_access, .. } => { + if network_access { + "enabled" + } else { + "restricted" + } + } + }; + writeln!(f, "Network access: {network_access}")?; + Ok(()) + } +} + /// API request payload for a single model turn. #[derive(Default, Debug, Clone)] pub struct Prompt { @@ -33,6 +78,10 @@ pub struct Prompt { /// Whether to store response on server side (disable_response_storage = !store). pub store: bool, + /// A list of key-value pairs that will be added as a developer message + /// for the model to use + pub environment_context: Option, + /// Tools available to the model, including additional tools sourced from /// external MCP servers. pub tools: Vec, @@ -54,11 +103,37 @@ impl Prompt { Cow::Owned(sections.join("\n")) } - pub(crate) fn get_formatted_user_instructions(&self) -> Option { + fn get_formatted_user_instructions(&self) -> Option { self.user_instructions .as_ref() .map(|ui| format!("{USER_INSTRUCTIONS_START}{ui}{USER_INSTRUCTIONS_END}")) } + + fn get_formatted_environment_context(&self) -> Option { + self.environment_context + .as_ref() + .map(|ec| format!("{ENVIRONMENT_CONTEXT_START}{ec}{ENVIRONMENT_CONTEXT_END}")) + } + + pub(crate) fn get_formatted_input(&self) -> Vec { + let mut input_with_instructions = Vec::with_capacity(self.input.len() + 2); + if let Some(ec) = self.get_formatted_environment_context() { + input_with_instructions.push(ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::InputText { text: ec }], + }); + } + if let Some(ui) = self.get_formatted_user_instructions() { + input_with_instructions.push(ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::InputText { text: ui }], + }); + } + input_with_instructions.extend(self.input.clone()); + input_with_instructions + } } #[derive(Debug)] diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index a7ab664ee0..c85b1ce2b9 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -37,6 +37,7 @@ use crate::apply_patch::convert_apply_patch_to_protocol; use crate::apply_patch::get_writable_roots; use crate::apply_patch::{self}; use crate::client::ModelClient; +use crate::client_common::EnvironmentContext; use crate::client_common::Prompt; use crate::client_common::ResponseEvent; use crate::config::Config; @@ -51,6 +52,7 @@ use crate::exec::SandboxType; use crate::exec::StdoutStream; use crate::exec::process_exec_tool_call; use crate::exec_env::create_env; +use crate::git_info::collect_git_info; use crate::mcp_connection_manager::McpConnectionManager; use crate::mcp_tool_call::handle_mcp_tool_call; use crate::models::ContentItem; @@ -1224,6 +1226,12 @@ async fn run_turn( store: !sess.disable_response_storage, tools, base_instructions_override: sess.base_instructions.clone(), + environment_context: Some(EnvironmentContext { + cwd: sess.cwd.clone(), + git_info: collect_git_info(&sess.cwd).await, + approval_policy: sess.approval_policy, + sandbox_policy: sess.sandbox_policy.clone(), + }), }; let mut retries = 0; @@ -1449,6 +1457,7 @@ async fn run_compact_task( input: turn_input, user_instructions: None, store: !sess.disable_response_storage, + environment_context: None, tools: Vec::new(), base_instructions_override: Some(compact_instructions.clone()), }; diff --git a/codex-rs/core/src/git_info.rs b/codex-rs/core/src/git_info.rs index f5dc016e66..52d029f669 100644 --- a/codex-rs/core/src/git_info.rs +++ b/codex-rs/core/src/git_info.rs @@ -9,7 +9,7 @@ use tokio::time::timeout; /// Timeout for git commands to prevent freezing on large repositories const GIT_COMMAND_TIMEOUT: TokioDuration = TokioDuration::from_secs(5); -#[derive(Serialize, Deserialize, Clone)] +#[derive(Serialize, Deserialize, Clone, Debug)] pub struct GitInfo { /// Current commit hash (SHA) #[serde(skip_serializing_if = "Option::is_none")] diff --git a/codex-rs/core/src/protocol.rs b/codex-rs/core/src/protocol.rs index 9bf85ec49a..55000fb6d7 100644 --- a/codex-rs/core/src/protocol.rs +++ b/codex-rs/core/src/protocol.rs @@ -159,7 +159,8 @@ pub enum AskForApproval { } /// Determines execution restrictions for model shell commands. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Display)] +#[strum(serialize_all = "kebab-case")] #[serde(tag = "mode", rename_all = "kebab-case")] pub enum SandboxPolicy { /// No restrictions whatsoever. Use with caution. diff --git a/codex-rs/core/tests/client.rs b/codex-rs/core/tests/client.rs index f493020210..00f91a879e 100644 --- a/codex-rs/core/tests/client.rs +++ b/codex-rs/core/tests/client.rs @@ -1,3 +1,5 @@ +#![allow(clippy::expect_used)] +#![allow(clippy::unwrap_used)] use std::path::PathBuf; use chrono::Utc; @@ -32,6 +34,32 @@ fn sse_completed(id: &str) -> String { load_sse_fixture_with_id("tests/fixtures/completed_template.json", id) } +fn assert_message_role(request_body: &serde_json::Value, role: &str) { + assert_eq!(request_body["role"].as_str().unwrap(), role); +} + +fn assert_message_starts_with(request_body: &serde_json::Value, text: &str) { + let content = request_body["content"][0]["text"] + .as_str() + .expect("invalid message content"); + + assert!( + content.starts_with(text), + "expected message content '{content}' to start with '{text}'" + ); +} + +fn assert_message_ends_with(request_body: &serde_json::Value, text: &str) { + let content = request_body["content"][0]["text"] + .as_str() + .expect("invalid message content"); + + assert!( + content.ends_with(text), + "expected message content '{content}' to end with '{text}'" + ); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn includes_session_id_and_model_headers_in_request() { #![allow(clippy::unwrap_used)] @@ -371,19 +399,12 @@ async fn includes_user_instructions_message_in_request() { .unwrap() .contains("be nice") ); - assert_eq!(request_body["input"][0]["role"], "user"); - assert!( - request_body["input"][0]["content"][0]["text"] - .as_str() - .unwrap() - .starts_with("\n\nbe nice") - ); - assert!( - request_body["input"][0]["content"][0]["text"] - .as_str() - .unwrap() - .ends_with("") - ); + assert_message_role(&request_body["input"][0], "user"); + assert_message_starts_with(&request_body["input"][0], "\n\n"); + assert_message_ends_with(&request_body["input"][0], ""); + assert_message_role(&request_body["input"][1], "user"); + assert_message_starts_with(&request_body["input"][1], "\n\n"); + assert_message_ends_with(&request_body["input"][1], ""); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] diff --git a/codex-rs/mcp-server/tests/send_message.rs b/codex-rs/mcp-server/tests/send_message.rs index fd3718e8f3..6e1389093c 100644 --- a/codex-rs/mcp-server/tests/send_message.rs +++ b/codex-rs/mcp-server/tests/send_message.rs @@ -99,7 +99,7 @@ async fn test_send_message_success() { response ); // wait for the server to hear the user message - sleep(Duration::from_secs(1)); + sleep(Duration::from_secs(10)); // Ensure the server and tempdir live until end of test drop(server); From dc468d563f1cdf3dfbeeb97dc29543b3ae9442e2 Mon Sep 17 00:00:00 2001 From: Dylan Date: Wed, 6 Aug 2025 08:05:17 -0700 Subject: [PATCH 094/100] [env] Remove git config for now (#1884) ## Summary Forgot to remove this in #1869 last night! Too much of a performance hit on the main thread. We can bring it back via an async thread on startup. --- codex-rs/core/src/client_common.rs | 3 --- codex-rs/core/src/codex.rs | 2 -- codex-rs/mcp-server/tests/send_message.rs | 2 +- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/codex-rs/core/src/client_common.rs b/codex-rs/core/src/client_common.rs index 2ca060f4a4..b37b1e3f80 100644 --- a/codex-rs/core/src/client_common.rs +++ b/codex-rs/core/src/client_common.rs @@ -1,7 +1,6 @@ use crate::config_types::ReasoningEffort as ReasoningEffortConfig; use crate::config_types::ReasoningSummary as ReasoningSummaryConfig; use crate::error::Result; -use crate::git_info::GitInfo; use crate::model_family::ModelFamily; use crate::models::ContentItem; use crate::models::ResponseItem; @@ -35,7 +34,6 @@ const USER_INSTRUCTIONS_END: &str = "\n\n"; #[derive(Debug, Clone)] pub(crate) struct EnvironmentContext { pub cwd: PathBuf, - pub git_info: Option, pub approval_policy: AskForApproval, pub sandbox_policy: SandboxPolicy, } @@ -47,7 +45,6 @@ impl Display for EnvironmentContext { "Current working directory: {}", self.cwd.to_string_lossy() )?; - writeln!(f, "Is directory a git repo: {}", self.git_info.is_some())?; writeln!(f, "Approval policy: {}", self.approval_policy)?; writeln!(f, "Sandbox policy: {}", self.sandbox_policy)?; diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index c85b1ce2b9..98d13b4cd6 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -52,7 +52,6 @@ use crate::exec::SandboxType; use crate::exec::StdoutStream; use crate::exec::process_exec_tool_call; use crate::exec_env::create_env; -use crate::git_info::collect_git_info; use crate::mcp_connection_manager::McpConnectionManager; use crate::mcp_tool_call::handle_mcp_tool_call; use crate::models::ContentItem; @@ -1228,7 +1227,6 @@ async fn run_turn( base_instructions_override: sess.base_instructions.clone(), environment_context: Some(EnvironmentContext { cwd: sess.cwd.clone(), - git_info: collect_git_info(&sess.cwd).await, approval_policy: sess.approval_policy, sandbox_policy: sess.sandbox_policy.clone(), }), diff --git a/codex-rs/mcp-server/tests/send_message.rs b/codex-rs/mcp-server/tests/send_message.rs index 6e1389093c..f06c1587e3 100644 --- a/codex-rs/mcp-server/tests/send_message.rs +++ b/codex-rs/mcp-server/tests/send_message.rs @@ -99,7 +99,7 @@ async fn test_send_message_success() { response ); // wait for the server to hear the user message - sleep(Duration::from_secs(10)); + sleep(Duration::from_secs(5)); // Ensure the server and tempdir live until end of test drop(server); From ffe24991b7157ca27c78bb3ac4d422225bf7c031 Mon Sep 17 00:00:00 2001 From: Charlie Weems Date: Wed, 6 Aug 2025 09:10:23 -0700 Subject: [PATCH 095/100] Initial implementation of /init (#1822) Basic /init command that appends an instruction to create AGENTS.md to the conversation history. --- INIT.md | 40 +++++++++++++++++ codex-rs/tui/src/app.rs | 7 +++ codex-rs/tui/src/bottom_pane/chat_composer.rs | 44 +++++++++++++++++++ codex-rs/tui/src/bottom_pane/command_popup.rs | 35 +++++++++++++++ codex-rs/tui/src/chatwidget.rs | 10 +++++ codex-rs/tui/src/slash_command.rs | 2 + 6 files changed, 138 insertions(+) create mode 100644 INIT.md diff --git a/INIT.md b/INIT.md new file mode 100644 index 0000000000..b8fd3886b3 --- /dev/null +++ b/INIT.md @@ -0,0 +1,40 @@ +Generate a file named AGENTS.md that serves as a contributor guide for this repository. +Your goal is to produce a clear, concise, and well-structured document with descriptive headings and actionable explanations for each section. +Follow the outline below, but adapt as needed — add sections if relevant, and omit those that do not apply to this project. + +Document Requirements + +- Title the document "Repository Guidelines". +- Use Markdown headings (#, ##, etc.) for structure. +- Keep the document concise. 200-400 words is optimal. +- Keep explanations short, direct, and specific to this repository. +- Provide examples where helpful (commands, directory paths, naming patterns). +- Maintain a professional, instructional tone. + +Recommended Sections + +Project Structure & Module Organization + +- Outline the project structure, including where the source code, tests, and assets are located. + +Build, Test, and Development Commands + +- List key commands for building, testing, and running locally (e.g., npm test, make build). +- Briefly explain what each command does. + +Coding Style & Naming Conventions + +- Specify indentation rules, language-specific style preferences, and naming patterns. +- Include any formatting or linting tools used. + +Testing Guidelines + +- Identify testing frameworks and coverage requirements. +- State test naming conventions and how to run tests. + +Commit & Pull Request Guidelines + +- Summarize commit message conventions found in the project’s Git history. +- Outline pull request requirements (descriptions, linked issues, screenshots, etc.). + +(Optional) Add other sections if relevant, such as Security & Configuration Tips, Architecture Overview, or Agent-Specific Instructions. diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index eee2a61c2e..f1807da1c9 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -300,6 +300,13 @@ impl App<'_> { self.app_state = AppState::Chat { widget: new_widget }; self.app_event_tx.send(AppEvent::RequestRedraw); } + SlashCommand::Init => { + // Guard: do not run if a task is active. + if let AppState::Chat { widget } = &mut self.app_state { + const INIT_PROMPT: &str = include_str!("../../../INIT.md"); + widget.submit_text_message(INIT_PROMPT.to_string()); + } + } SlashCommand::Compact => { if let AppState::Chat { widget } = &mut self.app_state { widget.clear_token_usage(); diff --git a/codex-rs/tui/src/bottom_pane/chat_composer.rs b/codex-rs/tui/src/bottom_pane/chat_composer.rs index c9ad719771..f30b980da9 100644 --- a/codex-rs/tui/src/bottom_pane/chat_composer.rs +++ b/codex-rs/tui/src/bottom_pane/chat_composer.rs @@ -729,6 +729,7 @@ impl WidgetRef for &ChatComposer { #[cfg(test)] mod tests { + use crate::app_event::AppEvent; use crate::bottom_pane::AppEventSender; use crate::bottom_pane::ChatComposer; use crate::bottom_pane::InputResult; @@ -1004,6 +1005,49 @@ mod tests { } } + #[test] + fn slash_init_dispatches_command_and_does_not_submit_literal_text() { + use crossterm::event::KeyCode; + use crossterm::event::KeyEvent; + use crossterm::event::KeyModifiers; + use std::sync::mpsc::TryRecvError; + + let (tx, rx) = std::sync::mpsc::channel(); + let sender = AppEventSender::new(tx); + let mut composer = ChatComposer::new(true, sender, false); + + // Type the slash command. + for ch in [ + '/', 'i', 'n', 'i', 't', // "/init" + ] { + let _ = composer.handle_key_event(KeyEvent::new(KeyCode::Char(ch), KeyModifiers::NONE)); + } + + // Press Enter to dispatch the selected command. + let (result, _needs_redraw) = + composer.handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); + + // When a slash command is dispatched, the composer should not submit + // literal text and should clear its textarea. + match result { + InputResult::None => {} + InputResult::Submitted(text) => { + panic!("expected command dispatch, but composer submitted literal text: {text}") + } + } + assert!(composer.textarea.is_empty(), "composer should be cleared"); + + // Verify a DispatchCommand event for the "init" command was sent. + match rx.try_recv() { + Ok(AppEvent::DispatchCommand(cmd)) => { + assert_eq!(cmd.command(), "init"); + } + Ok(_other) => panic!("unexpected app event"), + Err(TryRecvError::Empty) => panic!("expected a DispatchCommand event for '/init'"), + Err(TryRecvError::Disconnected) => panic!("app event channel disconnected"), + } + } + #[test] fn test_multiple_pastes_submission() { use crossterm::event::KeyCode; diff --git a/codex-rs/tui/src/bottom_pane/command_popup.rs b/codex-rs/tui/src/bottom_pane/command_popup.rs index 364a8472dc..1027df1a67 100644 --- a/codex-rs/tui/src/bottom_pane/command_popup.rs +++ b/codex-rs/tui/src/bottom_pane/command_popup.rs @@ -188,3 +188,38 @@ impl WidgetRef for CommandPopup { table.render(area, buf); } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn filter_includes_init_when_typing_prefix() { + let mut popup = CommandPopup::new(); + // Simulate the composer line starting with '/in' so the popup filters + // matching commands by prefix. + popup.on_composer_text_change("/in".to_string()); + + // Access the filtered list via the selected command and ensure that + // one of the matches is the new "init" command. + let matches = popup.filtered_commands(); + assert!( + matches.iter().any(|cmd| cmd.command() == "init"), + "expected '/init' to appear among filtered commands" + ); + } + + #[test] + fn selecting_init_by_exact_match() { + let mut popup = CommandPopup::new(); + popup.on_composer_text_change("/init".to_string()); + + // When an exact match exists, the selected command should be that + // command by default. + let selected = popup.selected_command(); + match selected { + Some(cmd) => assert_eq!(cmd.command(), "init"), + None => panic!("expected a selected command for exact match"), + } + } +} diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 6d03be783b..64b65b3d11 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -575,6 +575,16 @@ impl ChatWidget<'_> { } } + /// Programmatically submit a user text message as if typed in the + /// composer. The text will be added to conversation history and sent to + /// the agent. + pub(crate) fn submit_text_message(&mut self, text: String) { + if text.is_empty() { + return; + } + self.submit_user_message(text.into()); + } + pub(crate) fn token_usage(&self) -> &TokenUsage { &self.token_usage } diff --git a/codex-rs/tui/src/slash_command.rs b/codex-rs/tui/src/slash_command.rs index 85dde7a113..daa663884b 100644 --- a/codex-rs/tui/src/slash_command.rs +++ b/codex-rs/tui/src/slash_command.rs @@ -13,6 +13,7 @@ pub enum SlashCommand { // DO NOT ALPHA-SORT! Enum order is presentation order in the popup, so // more frequently used commands should be listed first. New, + Init, Compact, Diff, Status, @@ -26,6 +27,7 @@ impl SlashCommand { pub fn description(self) -> &'static str { match self { SlashCommand::New => "Start a new chat", + SlashCommand::Init => "Create an AGENTS.md file with instructions for Codex.", SlashCommand::Compact => "Compact the chat history", SlashCommand::Quit => "Exit the application", SlashCommand::Diff => "Show git diff (including untracked files)", From ae88b69b09f876a3017196a9cd66f83dac79d9d7 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Wed, 6 Aug 2025 10:39:58 -0700 Subject: [PATCH 096/100] fix: add more instructions to ensure GitHub Action reviews only the necessary code (#1887) Empirically, we have seen the GitHub Action comment on code outside of the PR, so try to provide additional instructions in the prompt to avoid this. --- .github/actions/codex/src/process-label.ts | 33 +++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/.github/actions/codex/src/process-label.ts b/.github/actions/codex/src/process-label.ts index 4b4361e118..60207efaaa 100644 --- a/.github/actions/codex/src/process-label.ts +++ b/.github/actions/codex/src/process-label.ts @@ -91,7 +91,38 @@ async function processLabel( labelConfig: LabelConfig, ): Promise { const template = labelConfig.getPromptTemplate(); - const populatedTemplate = await renderPromptTemplate(template, ctx); + + // If this is a review label, prepend explicit PR-diff scoping guidance to + // reduce out-of-scope feedback. Do this before rendering so placeholders in + // the guidance (e.g., {CODEX_ACTION_GITHUB_EVENT_PATH}) are substituted. + const isReview = label.toLowerCase().includes("review"); + const reviewScopeGuidance = ` +PR Diff Scope +- Only review changes between the PR's merge-base and head; do not comment on commits or files outside this range. +- Derive the base/head SHAs from the event JSON at {CODEX_ACTION_GITHUB_EVENT_PATH}, then compute and use the PR diff for all analysis and comments. + +Commands to determine scope +- Resolve SHAs: + - BASE_SHA=$(jq -r '.pull_request.base.sha // .pull_request.base.ref' "{CODEX_ACTION_GITHUB_EVENT_PATH}") + - HEAD_SHA=$(jq -r '.pull_request.head.sha // .pull_request.head.ref' "{CODEX_ACTION_GITHUB_EVENT_PATH}") + - BASE_SHA=$(git rev-parse "$BASE_SHA") + - HEAD_SHA=$(git rev-parse "$HEAD_SHA") +- Prefer triple-dot (merge-base) semantics for PR diffs: + - Changed commits: git log --oneline "$BASE_SHA...$HEAD_SHA" + - Changed files: git diff --name-status "$BASE_SHA...$HEAD_SHA" + - Review hunks: git diff -U0 "$BASE_SHA...$HEAD_SHA" + +Review rules +- Anchor every comment to a file and hunk present in git diff "$BASE_SHA...$HEAD_SHA". +- If you mention context outside the diff, label it as "Follow-up (outside this PR scope)" and keep it brief (<=2 bullets). +- Do not critique commits or files not reachable in the PR range (merge-base(base, head) → head). +`.trim(); + + const effectiveTemplate = isReview + ? `${reviewScopeGuidance}\n\n${template}` + : template; + + const populatedTemplate = await renderPromptTemplate(effectiveTemplate, ctx); // Always run Codex and post the resulting message as a comment. let commentBody = await runCodex(populatedTemplate, ctx); From 64f2f2eca227c1a9f8087f5a0a696c1e3d4fb934 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Wed, 6 Aug 2025 11:48:03 -0700 Subject: [PATCH 097/100] fix: support $CODEX_HOME/AGENTS.md instead of $CODEX_HOME/instructions.md (#1891) The docs and code do not match. It turns out the docs are "right" in they are what we have been meaning to support, so this PR updates the code: https://github.com/openai/codex/blob/ae88b69b09f876a3017196a9cd66f83dac79d9d7/README.md#L298-L302 Support for `instructions.md` is a holdover from the TypeScript CLI, so we are just going to drop support for it altogether rather than maintain it in perpetuity. --- codex-rs/core/src/config.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codex-rs/core/src/config.rs b/codex-rs/core/src/config.rs index 0b53df5ab7..f48cc9340b 100644 --- a/codex-rs/core/src/config.rs +++ b/codex-rs/core/src/config.rs @@ -70,7 +70,7 @@ pub struct Config { /// who have opted into Zero Data Retention (ZDR). pub disable_response_storage: bool, - /// User-provided instructions from instructions.md. + /// User-provided instructions from AGENTS.md. pub user_instructions: Option, /// Base instructions override. @@ -575,7 +575,7 @@ impl Config { None => return None, }; - p.push("instructions.md"); + p.push("AGENTS.md"); std::fs::read_to_string(&p).ok().and_then(|s| { let s = s.trim(); if s.is_empty() { From 4344537742f321aca82889625f55b2271f5e1e97 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Wed, 6 Aug 2025 11:58:57 -0700 Subject: [PATCH 098/100] chore: rename INIT.md to prompt_for_init_command.md and move closer to usage (#1886) Addressing my post-commit review feedback on https://github.com/openai/codex/pull/1822. --- INIT.md => codex-rs/tui/prompt_for_init_command.md | 0 codex-rs/tui/src/app.rs | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename INIT.md => codex-rs/tui/prompt_for_init_command.md (100%) diff --git a/INIT.md b/codex-rs/tui/prompt_for_init_command.md similarity index 100% rename from INIT.md rename to codex-rs/tui/prompt_for_init_command.md diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index f1807da1c9..47e20287bd 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -303,7 +303,7 @@ impl App<'_> { SlashCommand::Init => { // Guard: do not run if a task is active. if let AppState::Chat { widget } = &mut self.app_state { - const INIT_PROMPT: &str = include_str!("../../../INIT.md"); + const INIT_PROMPT: &str = include_str!("../prompt_for_init_command.md"); widget.submit_text_message(INIT_PROMPT.to_string()); } } From 081caa5a6b77cde2624d414f280af6f0701fb22f Mon Sep 17 00:00:00 2001 From: Jeremy Rose <172423086+nornagon-openai@users.noreply.github.com> Date: Wed, 6 Aug 2025 12:03:45 -0700 Subject: [PATCH 099/100] show a transient history cell for commands (#1824) Adds a new "active history cell" for history bits that need to render more than once before they're inserted into the history. Only used for commands right now. https://github.com/user-attachments/assets/925f01a0-e56d-4613-bc25-fdaa85d8aea5 --------- Co-authored-by: easong-openai --- codex-rs/tui/src/chatwidget.rs | 47 ++++++++-- codex-rs/tui/src/history_cell.rs | 143 +++++++++++++++++-------------- 2 files changed, 119 insertions(+), 71 deletions(-) diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 64b65b3d11..69f1600cd2 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -30,6 +30,8 @@ use codex_core::protocol::TurnDiffEvent; use crossterm::event::KeyEvent; use crossterm::event::KeyEventKind; use ratatui::buffer::Buffer; +use ratatui::layout::Constraint; +use ratatui::layout::Layout; use ratatui::layout::Rect; use ratatui::widgets::Widget; use ratatui::widgets::WidgetRef; @@ -62,6 +64,7 @@ pub(crate) struct ChatWidget<'a> { app_event_tx: AppEventSender, codex_op_tx: UnboundedSender, bottom_pane: BottomPane<'a>, + active_history_cell: Option, config: Config, initial_user_message: Option, token_usage: TokenUsage, @@ -107,6 +110,17 @@ fn create_initial_user_message(text: String, image_paths: Vec) -> Optio } impl ChatWidget<'_> { + fn layout_areas(&self, area: Rect) -> [Rect; 2] { + Layout::vertical([ + Constraint::Max( + self.active_history_cell + .as_ref() + .map_or(0, |c| c.desired_height(area.width)), + ), + Constraint::Min(self.bottom_pane.desired_height(area.width)), + ]) + .areas(area) + } fn emit_stream_header(&mut self, kind: StreamKind) { use ratatui::text::Line as RLine; if self.stream_header_emitted { @@ -178,6 +192,7 @@ impl ChatWidget<'_> { has_input_focus: true, enhanced_keys_supported, }), + active_history_cell: None, config, initial_user_message: create_initial_user_message( initial_prompt.unwrap_or_default(), @@ -197,6 +212,10 @@ impl ChatWidget<'_> { pub fn desired_height(&self, width: u16) -> u16 { self.bottom_pane.desired_height(width) + + self + .active_history_cell + .as_ref() + .map_or(0, |c| c.desired_height(width)) } pub(crate) fn handle_key_event(&mut self, key_event: KeyEvent) { @@ -425,9 +444,11 @@ impl ChatWidget<'_> { cwd: cwd.clone(), }, ); - self.add_to_history(HistoryCell::new_active_exec_command(command)); + self.active_history_cell = Some(HistoryCell::new_active_exec_command(command)); + } + EventMsg::ExecCommandOutputDelta(_) => { + // TODO } - EventMsg::ExecCommandOutputDelta(_) => {} EventMsg::PatchApplyBegin(PatchApplyBeginEvent { call_id: _, auto_approved, @@ -438,8 +459,12 @@ impl ChatWidget<'_> { changes, )); } - EventMsg::PatchApplyEnd(patch_apply_end_event) => { - self.add_to_history(HistoryCell::new_patch_end_event(patch_apply_end_event)); + EventMsg::PatchApplyEnd(event) => { + self.add_to_history(HistoryCell::new_patch_apply_end( + event.stdout, + event.stderr, + event.success, + )); } EventMsg::ExecCommandEnd(ExecCommandEndEvent { call_id, @@ -450,6 +475,7 @@ impl ChatWidget<'_> { }) => { // Compute summary before moving stdout into the history cell. let cmd = self.running_commands.remove(&call_id); + self.active_history_cell = None; self.add_to_history(HistoryCell::new_completed_exec_command( cmd.map(|cmd| cmd.command).unwrap_or_else(|| vec![call_id]), CommandOutput { @@ -543,6 +569,7 @@ impl ChatWidget<'_> { CancellationEvent::Ignored => {} } if self.bottom_pane.is_task_running() { + self.active_history_cell = None; self.bottom_pane.clear_ctrl_c_quit_hint(); self.submit_op(Op::Interrupt); self.bottom_pane.set_task_running(false); @@ -596,7 +623,8 @@ impl ChatWidget<'_> { } pub fn cursor_pos(&self, area: Rect) -> Option<(u16, u16)> { - self.bottom_pane.cursor_pos(area) + let [_, bottom_pane_area] = self.layout_areas(area); + self.bottom_pane.cursor_pos(bottom_pane_area) } } @@ -700,10 +728,11 @@ impl ChatWidget<'_> { impl WidgetRef for &ChatWidget<'_> { fn render_ref(&self, area: Rect, buf: &mut Buffer) { - // In the hybrid inline viewport mode we only draw the interactive - // bottom pane; history entries are injected directly into scrollback - // via `Terminal::insert_before`. - (&self.bottom_pane).render(area, buf); + let [active_cell_area, bottom_pane_area] = self.layout_areas(area); + (&self.bottom_pane).render(bottom_pane_area, buf); + if let Some(cell) = &self.active_history_cell { + cell.render_ref(active_cell_area, buf); + } } } diff --git a/codex-rs/tui/src/history_cell.rs b/codex-rs/tui/src/history_cell.rs index 5b7d9246f7..facb0e0a8f 100644 --- a/codex-rs/tui/src/history_cell.rs +++ b/codex-rs/tui/src/history_cell.rs @@ -11,7 +11,6 @@ use codex_core::plan_tool::StepStatus; use codex_core::plan_tool::UpdatePlanArgs; use codex_core::protocol::FileChange; use codex_core::protocol::McpInvocation; -use codex_core::protocol::PatchApplyEndEvent; use codex_core::protocol::SessionConfiguredEvent; use codex_core::protocol::TokenUsage; use image::DynamicImage; @@ -24,6 +23,9 @@ use ratatui::style::Modifier; use ratatui::style::Style; use ratatui::text::Line as RtLine; use ratatui::text::Span as RtSpan; +use ratatui::widgets::Paragraph; +use ratatui::widgets::WidgetRef; +use ratatui::widgets::Wrap; use std::collections::HashMap; use std::io::Cursor; use std::path::PathBuf; @@ -62,35 +64,23 @@ fn line_to_static(line: &Line) -> Line<'static> { /// scrollable list. pub(crate) enum HistoryCell { /// Welcome message. - WelcomeMessage { - view: TextBlock, - }, + WelcomeMessage { view: TextBlock }, /// Message from the user. - UserPrompt { - view: TextBlock, - }, + UserPrompt { view: TextBlock }, // AgentMessage and AgentReasoning variants were unused and have been removed. /// An exec tool call that has not finished yet. - ActiveExecCommand { - view: TextBlock, - }, + ActiveExecCommand { view: TextBlock }, /// Completed exec tool call. - CompletedExecCommand { - view: TextBlock, - }, + CompletedExecCommand { view: TextBlock }, /// An MCP tool call that has not finished yet. - ActiveMcpToolCall { - view: TextBlock, - }, + ActiveMcpToolCall { view: TextBlock }, /// Completed MCP tool call where we show the result serialized as JSON. - CompletedMcpToolCall { - view: TextBlock, - }, + CompletedMcpToolCall { view: TextBlock }, /// Completed MCP tool call where the result is an image. /// Admittedly, [mcp_types::CallToolResult] can have multiple content types, @@ -100,51 +90,34 @@ pub(crate) enum HistoryCell { // resized version avoids doing the potentially expensive rescale twice // because the scroll-view first calls `height()` for layouting and then // `render_window()` for painting. - CompletedMcpToolCallWithImageOutput { - _image: DynamicImage, - }, + CompletedMcpToolCallWithImageOutput { _image: DynamicImage }, /// Background event. - BackgroundEvent { - view: TextBlock, - }, + BackgroundEvent { view: TextBlock }, /// Output from the `/diff` command. - GitDiffOutput { - view: TextBlock, - }, + GitDiffOutput { view: TextBlock }, /// Output from the `/status` command. - StatusOutput { - view: TextBlock, - }, + StatusOutput { view: TextBlock }, /// Error event from the backend. - ErrorEvent { - view: TextBlock, - }, + ErrorEvent { view: TextBlock }, /// Info describing the newly-initialized session. - SessionInfo { - view: TextBlock, - }, + SessionInfo { view: TextBlock }, /// A pending code patch that is awaiting user approval. Mirrors the /// behaviour of `ActiveExecCommand` so the user sees *what* patch the /// model wants to apply before being prompted to approve or deny it. - PendingPatch { - view: TextBlock, - }, - - PatchEventEnd { - view: TextBlock, - }, + PendingPatch { view: TextBlock }, /// A human‑friendly rendering of the model's current plan and step /// statuses provided via the `update_plan` tool. - PlanUpdate { - view: TextBlock, - }, + PlanUpdate { view: TextBlock }, + + /// Result of applying a patch (success or failure) with optional output. + PatchApplyResult { view: TextBlock }, } const TOOL_CALL_MAX_LINES: usize = 5; @@ -165,8 +138,8 @@ impl HistoryCell { | HistoryCell::CompletedExecCommand { view } | HistoryCell::CompletedMcpToolCall { view } | HistoryCell::PendingPatch { view } - | HistoryCell::PatchEventEnd { view } | HistoryCell::PlanUpdate { view } + | HistoryCell::PatchApplyResult { view } | HistoryCell::ActiveExecCommand { view, .. } | HistoryCell::ActiveMcpToolCall { view, .. } => { view.lines.iter().map(line_to_static).collect() @@ -177,6 +150,15 @@ impl HistoryCell { ], } } + + pub(crate) fn desired_height(&self, width: u16) -> u16 { + Paragraph::new(Text::from(self.plain_lines())) + .wrap(Wrap { trim: false }) + .line_count(width) + .try_into() + .unwrap_or(0) + } + pub(crate) fn new_session_info( config: &Config, event: SessionConfiguredEvent, @@ -612,7 +594,10 @@ impl HistoryCell { PatchEventType::ApplyBegin { auto_approved: false, } => { - let lines = vec![Line::from("patch applied".magenta().bold())]; + let lines: Vec> = vec![ + Line::from("applying patch".magenta().bold()), + Line::from(""), + ]; return Self::PendingPatch { view: TextBlock::new(lines), }; @@ -661,29 +646,63 @@ impl HistoryCell { } } - pub(crate) fn new_patch_end_event(patch_apply_end_event: PatchApplyEndEvent) -> Self { - let PatchApplyEndEvent { - call_id: _, - stdout: _, - stderr, - success, - } = patch_apply_end_event; + pub(crate) fn new_patch_apply_end(stdout: String, stderr: String, success: bool) -> Self { + let mut lines: Vec> = Vec::new(); - let mut lines: Vec> = if success { - vec![Line::from("patch applied successfully".italic())] + let status = if success { + RtSpan::styled("patch applied", Style::default().fg(Color::Green)) } else { - let mut lines = vec![Line::from("patch failed".italic())]; - lines.extend(stderr.lines().map(|l| Line::from(l.to_string()))); - lines + RtSpan::styled( + "patch failed", + Style::default().fg(Color::Red).add_modifier(Modifier::BOLD), + ) }; + lines.push(RtLine::from(vec![ + "patch".magenta().bold(), + " ".into(), + status, + ])); + + let src = if success { + if stdout.trim().is_empty() { + &stderr + } else { + &stdout + } + } else if stderr.trim().is_empty() { + &stdout + } else { + &stderr + }; + + if !src.trim().is_empty() { + lines.push(Line::from("")); + let mut iter = src.lines(); + for raw in iter.by_ref().take(TOOL_CALL_MAX_LINES) { + lines.push(ansi_escape_line(raw).dim()); + } + let remaining = iter.count(); + if remaining > 0 { + lines.push(Line::from(format!("... {remaining} additional lines")).dim()); + } + } + lines.push(Line::from("")); - HistoryCell::PatchEventEnd { + HistoryCell::PatchApplyResult { view: TextBlock::new(lines), } } } +impl WidgetRef for &HistoryCell { + fn render_ref(&self, area: Rect, buf: &mut Buffer) { + Paragraph::new(Text::from(self.plain_lines())) + .wrap(Wrap { trim: false }) + .render(area, buf); + } +} + fn create_diff_summary(changes: HashMap) -> Vec { // Build a concise, human‑readable summary list similar to the // `git status` short format so the user can reason about the From 8262ba58b28631377f2f63bcbe11b69997673d2c Mon Sep 17 00:00:00 2001 From: pakrym-oai Date: Wed, 6 Aug 2025 13:02:00 -0700 Subject: [PATCH 100/100] Prefer env var auth over default codex auth (#1861) ## Summary - Prioritize provider-specific API keys over default Codex auth when building requests - Add test to ensure provider env var auth overrides default auth ## Testing - `just fmt` - `just fix` *(fails: `let` expressions in this position are unstable)* - `cargo test --all-features` *(fails: `let` expressions in this position are unstable)* ------ https://chatgpt.com/codex/tasks/task_i_68926a104f7483208f2c8fd36763e0e3 --- codex-rs/core/src/client.rs | 6 +- codex-rs/core/src/config.rs | 2 +- codex-rs/core/src/model_provider_info.rs | 43 ++++++------ codex-rs/core/tests/client.rs | 82 +++++++++++++++++++++- codex-rs/core/tests/stream_no_completed.rs | 2 +- codex-rs/tui/src/lib.rs | 2 +- 6 files changed, 107 insertions(+), 30 deletions(-) diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index 9748cde7cb..ed05fb5db0 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -623,7 +623,7 @@ mod tests { request_max_retries: Some(0), stream_max_retries: Some(0), stream_idle_timeout_ms: Some(1000), - requires_auth: false, + requires_openai_auth: false, }; let events = collect_events( @@ -683,7 +683,7 @@ mod tests { request_max_retries: Some(0), stream_max_retries: Some(0), stream_idle_timeout_ms: Some(1000), - requires_auth: false, + requires_openai_auth: false, }; let events = collect_events(&[sse1.as_bytes()], provider).await; @@ -786,7 +786,7 @@ mod tests { request_max_retries: Some(0), stream_max_retries: Some(0), stream_idle_timeout_ms: Some(1000), - requires_auth: false, + requires_openai_auth: false, }; let out = run_sse(evs, provider).await; diff --git a/codex-rs/core/src/config.rs b/codex-rs/core/src/config.rs index f48cc9340b..63a2e5949f 100644 --- a/codex-rs/core/src/config.rs +++ b/codex-rs/core/src/config.rs @@ -842,7 +842,7 @@ disable_response_storage = true request_max_retries: Some(4), stream_max_retries: Some(10), stream_idle_timeout_ms: Some(300_000), - requires_auth: false, + requires_openai_auth: false, }; let model_provider_map = { let mut model_provider_map = built_in_model_providers(); diff --git a/codex-rs/core/src/model_provider_info.rs b/codex-rs/core/src/model_provider_info.rs index db369df3b7..a980211199 100644 --- a/codex-rs/core/src/model_provider_info.rs +++ b/codex-rs/core/src/model_provider_info.rs @@ -9,7 +9,6 @@ use codex_login::AuthMode; use codex_login::CodexAuth; use serde::Deserialize; use serde::Serialize; -use std::borrow::Cow; use std::collections::HashMap; use std::env::VarError; use std::time::Duration; @@ -79,7 +78,7 @@ pub struct ModelProviderInfo { /// Whether this provider requires some form of standard authentication (API key, ChatGPT token). #[serde(default)] - pub requires_auth: bool, + pub requires_openai_auth: bool, } impl ModelProviderInfo { @@ -87,26 +86,32 @@ impl ModelProviderInfo { /// reqwest Client applying: /// • provider-specific headers (static + env based) /// • Bearer auth header when an API key is available. + /// • Auth token for OAuth. /// - /// When `require_api_key` is true and the provider declares an `env_key` - /// but the variable is missing/empty, returns an [`Err`] identical to the + /// If the provider declares an `env_key` but the variable is missing/empty, returns an [`Err`] identical to the /// one produced by [`ModelProviderInfo::api_key`]. pub async fn create_request_builder<'a>( &'a self, client: &'a reqwest::Client, auth: &Option, ) -> crate::error::Result { - let auth: Cow<'_, Option> = if auth.is_some() { - Cow::Borrowed(auth) - } else { - Cow::Owned(self.get_fallback_auth()?) + let effective_auth = match self.api_key() { + Ok(Some(key)) => Some(CodexAuth::from_api_key(key)), + Ok(None) => auth.clone(), + Err(err) => { + if auth.is_some() { + auth.clone() + } else { + return Err(err); + } + } }; - let url = self.get_full_url(&auth); + let url = self.get_full_url(&effective_auth); let mut builder = client.post(url); - if let Some(auth) = auth.as_ref() { + if let Some(auth) = effective_auth.as_ref() { builder = builder.bearer_auth(auth.get_token().await?); } @@ -216,14 +221,6 @@ impl ModelProviderInfo { .map(Duration::from_millis) .unwrap_or(Duration::from_millis(DEFAULT_STREAM_IDLE_TIMEOUT_MS)) } - - fn get_fallback_auth(&self) -> crate::error::Result> { - let api_key = self.api_key()?; - if let Some(api_key) = api_key { - return Ok(Some(CodexAuth::from_api_key(api_key))); - } - Ok(None) - } } const DEFAULT_OLLAMA_PORT: u32 = 11434; @@ -275,7 +272,7 @@ pub fn built_in_model_providers() -> HashMap { request_max_retries: None, stream_max_retries: None, stream_idle_timeout_ms: None, - requires_auth: true, + requires_openai_auth: true, }, ), (BUILT_IN_OSS_MODEL_PROVIDER_ID, create_oss_provider()), @@ -319,7 +316,7 @@ pub fn create_oss_provider_with_base_url(base_url: &str) -> ModelProviderInfo { request_max_retries: None, stream_max_retries: None, stream_idle_timeout_ms: None, - requires_auth: false, + requires_openai_auth: false, } } @@ -347,7 +344,7 @@ base_url = "http://localhost:11434/v1" request_max_retries: None, stream_max_retries: None, stream_idle_timeout_ms: None, - requires_auth: false, + requires_openai_auth: false, }; let provider: ModelProviderInfo = toml::from_str(azure_provider_toml).unwrap(); @@ -376,7 +373,7 @@ query_params = { api-version = "2025-04-01-preview" } request_max_retries: None, stream_max_retries: None, stream_idle_timeout_ms: None, - requires_auth: false, + requires_openai_auth: false, }; let provider: ModelProviderInfo = toml::from_str(azure_provider_toml).unwrap(); @@ -408,7 +405,7 @@ env_http_headers = { "X-Example-Env-Header" = "EXAMPLE_ENV_VAR" } request_max_retries: None, stream_max_retries: None, stream_idle_timeout_ms: None, - requires_auth: false, + requires_openai_auth: false, }; let provider: ModelProviderInfo = toml::from_str(azure_provider_toml).unwrap(); diff --git a/codex-rs/core/tests/client.rs b/codex-rs/core/tests/client.rs index 00f91a879e..60eb922474 100644 --- a/codex-rs/core/tests/client.rs +++ b/codex-rs/core/tests/client.rs @@ -458,7 +458,7 @@ async fn azure_overrides_assign_properties_used_for_responses_url() { request_max_retries: None, stream_max_retries: None, stream_idle_timeout_ms: None, - requires_auth: false, + requires_openai_auth: false, }; // Init session @@ -481,6 +481,86 @@ async fn azure_overrides_assign_properties_used_for_responses_url() { wait_for_event(&codex, |ev| matches!(ev, EventMsg::TaskComplete(_))).await; } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn env_var_overrides_loaded_auth() { + #![allow(clippy::unwrap_used)] + + let existing_env_var_with_random_value = if cfg!(windows) { "USERNAME" } else { "USER" }; + + // Mock server + let server = MockServer::start().await; + + // First request – must NOT include `previous_response_id`. + let first = ResponseTemplate::new(200) + .insert_header("content-type", "text/event-stream") + .set_body_raw(sse_completed("resp1"), "text/event-stream"); + + // Expect POST to /openai/responses with api-version query param + Mock::given(method("POST")) + .and(path("/openai/responses")) + .and(query_param("api-version", "2025-04-01-preview")) + .and(header_regex("Custom-Header", "Value")) + .and(header_regex( + "Authorization", + format!( + "Bearer {}", + std::env::var(existing_env_var_with_random_value).unwrap() + ) + .as_str(), + )) + .respond_with(first) + .expect(1) + .mount(&server) + .await; + + let provider = ModelProviderInfo { + name: "custom".to_string(), + base_url: Some(format!("{}/openai", server.uri())), + // Reuse the existing environment variable to avoid using unsafe code + env_key: Some(existing_env_var_with_random_value.to_string()), + query_params: Some(std::collections::HashMap::from([( + "api-version".to_string(), + "2025-04-01-preview".to_string(), + )])), + env_key_instructions: None, + wire_api: WireApi::Responses, + http_headers: Some(std::collections::HashMap::from([( + "Custom-Header".to_string(), + "Value".to_string(), + )])), + env_http_headers: None, + request_max_retries: None, + stream_max_retries: None, + stream_idle_timeout_ms: None, + requires_openai_auth: false, + }; + + // Init session + let codex_home = TempDir::new().unwrap(); + let mut config = load_default_config_for_test(&codex_home); + config.model_provider = provider; + + let ctrl_c = std::sync::Arc::new(tokio::sync::Notify::new()); + let CodexSpawnOk { codex, .. } = Codex::spawn( + config, + Some(auth_from_token("Default Access Token".to_string())), + ctrl_c.clone(), + ) + .await + .unwrap(); + + codex + .submit(Op::UserInput { + items: vec![InputItem::Text { + text: "hello".into(), + }], + }) + .await + .unwrap(); + + wait_for_event(&codex, |ev| matches!(ev, EventMsg::TaskComplete(_))).await; +} + fn auth_from_token(id_token: String) -> CodexAuth { CodexAuth::new( None, diff --git a/codex-rs/core/tests/stream_no_completed.rs b/codex-rs/core/tests/stream_no_completed.rs index 3e30d93709..8a4216b129 100644 --- a/codex-rs/core/tests/stream_no_completed.rs +++ b/codex-rs/core/tests/stream_no_completed.rs @@ -90,7 +90,7 @@ async fn retries_on_early_close() { request_max_retries: Some(0), stream_max_retries: Some(1), stream_idle_timeout_ms: Some(2000), - requires_auth: false, + requires_openai_auth: false, }; let ctrl_c = std::sync::Arc::new(tokio::sync::Notify::new()); diff --git a/codex-rs/tui/src/lib.rs b/codex-rs/tui/src/lib.rs index 50535e5967..0228a56859 100644 --- a/codex-rs/tui/src/lib.rs +++ b/codex-rs/tui/src/lib.rs @@ -287,7 +287,7 @@ fn restore() { #[allow(clippy::unwrap_used)] fn should_show_login_screen(config: &Config) -> bool { - if config.model_provider.requires_auth { + if config.model_provider.requires_openai_auth { // Reading the OpenAI API key is an async operation because it may need // to refresh the token. Block on it. let codex_home = config.codex_home.clone();