From 0543d0a022376f0440c2ea5b9aab8c2fd0813758 Mon Sep 17 00:00:00 2001 From: Curtis 'Fjord' Hawthorne Date: Wed, 25 Feb 2026 09:44:52 -0800 Subject: [PATCH 1/4] Promote js_repl to experimental with Node requirement (#12712) ## Summary - Promote `js_repl` to an experimental feature that users can enable from `/experimental`. - Add `js_repl` experimental metadata, including the Node prerequisite and activation guidance. - Add regression coverage for the feature metadata and the `/experimental` popup. ## What Changed - Changed `Feature::JsRepl` from `Stage::UnderDevelopment` to `Stage::Experimental`. - Added experimental metadata for `js_repl` in `core/src/features.rs`: - name: `JavaScript REPL` - description: calls out interactive website debugging, inline JavaScript execution, and the required Node version (`>= v24.13.1`) - announcement: tells users to enable it, then start a new chat or restart Codex - Added a core unit test that verifies: - `js_repl` is experimental - `js_repl` is disabled by default - the hardcoded Node version in the description matches `node-version.txt` - Added a TUI test that opens the `/experimental` popup and verifies the rendered `js_repl` entry includes the Node requirement text. ## Testing - `just fmt` - `cargo test -p codex-tui` - `cargo test -p codex-core` (unit-test phase passed; stopped during the long `tests/all.rs` integration suite) --- codex-rs/core/src/features.rs | 23 ++++++++++++++++++++++- codex-rs/tui/src/chatwidget/tests.rs | 25 +++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/codex-rs/core/src/features.rs b/codex-rs/core/src/features.rs index d3bace8519..546652ae60 100644 --- a/codex-rs/core/src/features.rs +++ b/codex-rs/core/src/features.rs @@ -460,7 +460,11 @@ pub const FEATURES: &[FeatureSpec] = &[ FeatureSpec { id: Feature::JsRepl, key: "js_repl", - stage: Stage::UnderDevelopment, + stage: Stage::Experimental { + name: "JavaScript REPL", + menu_description: "Enable a persistent Node-backed JavaScript REPL for interactive website debugging and other inline JavaScript execution capabilities. Requires Node >= v24.13.1 installed.", + announcement: "NEW: JavaScript REPL is now available in /experimental. Enable it, then start a new chat or restart Codex to use it.", + }, default_enabled: false, }, FeatureSpec { @@ -795,6 +799,23 @@ mod tests { assert_eq!(Feature::UseLinuxSandboxBwrap.default_enabled(), false); } + #[test] + fn js_repl_is_experimental_and_user_toggleable() { + let spec = Feature::JsRepl.info(); + let stage = spec.stage; + let expected_node_version = include_str!("../../node-version.txt").trim_end(); + + assert!(matches!(stage, Stage::Experimental { .. })); + assert_eq!(stage.experimental_menu_name(), Some("JavaScript REPL")); + assert_eq!( + stage.experimental_menu_description().map(str::to_owned), + Some(format!( + "Enable a persistent Node-backed JavaScript REPL for interactive website debugging and other inline JavaScript execution capabilities. Requires Node >= v{expected_node_version} installed." + )) + ); + assert_eq!(Feature::JsRepl.default_enabled(), false); + } + #[test] fn collab_is_legacy_alias_for_multi_agent() { assert_eq!(feature_for_key("multi_agent"), Some(Feature::Collab)); diff --git a/codex-rs/tui/src/chatwidget/tests.rs b/codex-rs/tui/src/chatwidget/tests.rs index 358a9615c2..22ba19af63 100644 --- a/codex-rs/tui/src/chatwidget/tests.rs +++ b/codex-rs/tui/src/chatwidget/tests.rs @@ -23,6 +23,7 @@ use codex_core::config::ConstraintError; #[cfg(target_os = "windows")] use codex_core::config::types::WindowsSandboxModeToml; use codex_core::config_loader::RequirementSource; +use codex_core::features::FEATURES; use codex_core::features::Feature; use codex_core::models_manager::manager::ModelsManager; use codex_core::skills::model::SkillMetadata; @@ -5945,6 +5946,30 @@ async fn experimental_features_toggle_saves_on_exit() { assert_eq!(updates, vec![(expected_feature, true)]); } +#[tokio::test] +async fn experimental_popup_shows_js_repl_node_requirement() { + let (mut chat, _rx, _op_rx) = make_chatwidget_manual(None).await; + + let js_repl_description = FEATURES + .iter() + .find(|spec| spec.id == Feature::JsRepl) + .and_then(|spec| spec.stage.experimental_menu_description()) + .expect("expected js_repl experimental description"); + let node_requirement = js_repl_description + .split(". ") + .find(|sentence| sentence.starts_with("Requires Node >= v")) + .map(|sentence| sentence.trim_end_matches(" installed.")) + .expect("expected js_repl description to mention the Node requirement"); + + chat.open_experimental_popup(); + + let popup = render_bottom_popup(&chat, 120); + assert!( + popup.contains(node_requirement), + "expected js_repl feature description to mention the required Node version, got:\n{popup}" + ); +} + #[tokio::test] async fn model_selection_popup_snapshot() { let (mut chat, _rx, _op_rx) = make_chatwidget_manual(Some("gpt-5-codex")).await; From 947092283ac3680582fe2fee4b246a61f3c87129 Mon Sep 17 00:00:00 2001 From: Ahmed Ibrahim Date: Wed, 25 Feb 2026 09:59:10 -0800 Subject: [PATCH 2/4] Add app-server v2 thread realtime API (#12715) Add experimental `thread/realtime/*` v2 requests and notifications, then route app-server realtime events through that thread-scoped surface with integration coverage. --------- Co-authored-by: Codex --- .../schema/json/ClientRequest.json | 32 ++ .../schema/json/ServerNotification.json | 214 ++++++++++ .../codex_app_server_protocol.schemas.json | 224 ++++++++++ .../v2/ThreadRealtimeClosedNotification.json | 20 + .../v2/ThreadRealtimeErrorNotification.json | 18 + .../ThreadRealtimeItemAddedNotification.json | 16 + ...dRealtimeOutputAudioDeltaNotification.json | 52 +++ .../v2/ThreadRealtimeStartedNotification.json | 20 + .../schema/typescript/ServerNotification.ts | 7 +- .../typescript/v2/ThreadRealtimeAudioChunk.ts | 8 + .../v2/ThreadRealtimeClosedNotification.ts | 8 + .../v2/ThreadRealtimeErrorNotification.ts | 8 + .../v2/ThreadRealtimeItemAddedNotification.ts | 9 + ...eadRealtimeOutputAudioDeltaNotification.ts | 9 + .../v2/ThreadRealtimeStartedNotification.ts | 8 + .../schema/typescript/v2/index.ts | 6 + .../src/protocol/common.rs | 138 +++++- .../app-server-protocol/src/protocol/v2.rs | 152 +++++++ codex-rs/app-server/README.md | 18 + .../app-server/src/bespoke_event_handling.rs | 73 ++++ .../app-server/src/codex_message_processor.rs | 198 +++++++++ .../app-server/tests/common/mcp_process.rs | 42 ++ .../tests/suite/v2/experimental_api.rs | 35 ++ codex-rs/app-server/tests/suite/v2/mod.rs | 1 + .../tests/suite/v2/realtime_conversation.rs | 392 ++++++++++++++++++ 25 files changed, 1706 insertions(+), 2 deletions(-) create mode 100644 codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeClosedNotification.json create mode 100644 codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeErrorNotification.json create mode 100644 codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeItemAddedNotification.json create mode 100644 codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeOutputAudioDeltaNotification.json create mode 100644 codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeStartedNotification.json create mode 100644 codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeAudioChunk.ts create mode 100644 codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeClosedNotification.ts create mode 100644 codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeErrorNotification.ts create mode 100644 codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeItemAddedNotification.ts create mode 100644 codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeOutputAudioDeltaNotification.ts create mode 100644 codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeStartedNotification.ts create mode 100644 codex-rs/app-server/tests/suite/v2/realtime_conversation.rs diff --git a/codex-rs/app-server-protocol/schema/json/ClientRequest.json b/codex-rs/app-server-protocol/schema/json/ClientRequest.json index f4b94339ed..b5b85d88a4 100644 --- a/codex-rs/app-server-protocol/schema/json/ClientRequest.json +++ b/codex-rs/app-server-protocol/schema/json/ClientRequest.json @@ -2052,6 +2052,38 @@ ], "type": "object" }, + "ThreadRealtimeAudioChunk": { + "description": "EXPERIMENTAL - thread realtime audio chunk.", + "properties": { + "data": { + "type": "string" + }, + "numChannels": { + "format": "uint16", + "minimum": 0.0, + "type": "integer" + }, + "sampleRate": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "samplesPerChannel": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "data", + "numChannels", + "sampleRate" + ], + "type": "object" + }, "ThreadResumeParams": { "description": "There are three ways to resume a thread: 1. By thread_id: load the thread from disk by thread_id and resume it. 2. By history: instantiate the thread from memory and resume it. 3. By path: load the thread from disk by path and resume it.\n\nThe precedence is: history > path > thread_id. If using history or path, the thread_id param will be ignored.\n\nPrefer using thread_id whenever possible.", "properties": { diff --git a/codex-rs/app-server-protocol/schema/json/ServerNotification.json b/codex-rs/app-server-protocol/schema/json/ServerNotification.json index b2dc4c3c3e..47cdc64723 100644 --- a/codex-rs/app-server-protocol/schema/json/ServerNotification.json +++ b/codex-rs/app-server-protocol/schema/json/ServerNotification.json @@ -2177,6 +2177,120 @@ ], "type": "object" }, + "ThreadRealtimeAudioChunk": { + "description": "EXPERIMENTAL - thread realtime audio chunk.", + "properties": { + "data": { + "type": "string" + }, + "numChannels": { + "format": "uint16", + "minimum": 0.0, + "type": "integer" + }, + "sampleRate": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "samplesPerChannel": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "data", + "numChannels", + "sampleRate" + ], + "type": "object" + }, + "ThreadRealtimeClosedNotification": { + "description": "EXPERIMENTAL - emitted when thread realtime transport closes.", + "properties": { + "reason": { + "type": [ + "string", + "null" + ] + }, + "threadId": { + "type": "string" + } + }, + "required": [ + "threadId" + ], + "type": "object" + }, + "ThreadRealtimeErrorNotification": { + "description": "EXPERIMENTAL - emitted when thread realtime encounters an error.", + "properties": { + "message": { + "type": "string" + }, + "threadId": { + "type": "string" + } + }, + "required": [ + "message", + "threadId" + ], + "type": "object" + }, + "ThreadRealtimeItemAddedNotification": { + "description": "EXPERIMENTAL - raw non-audio thread realtime item emitted by the backend.", + "properties": { + "item": true, + "threadId": { + "type": "string" + } + }, + "required": [ + "item", + "threadId" + ], + "type": "object" + }, + "ThreadRealtimeOutputAudioDeltaNotification": { + "description": "EXPERIMENTAL - streamed output audio emitted by thread realtime.", + "properties": { + "audio": { + "$ref": "#/definitions/ThreadRealtimeAudioChunk" + }, + "threadId": { + "type": "string" + } + }, + "required": [ + "audio", + "threadId" + ], + "type": "object" + }, + "ThreadRealtimeStartedNotification": { + "description": "EXPERIMENTAL - emitted when thread realtime startup is accepted.", + "properties": { + "sessionId": { + "type": [ + "string", + "null" + ] + }, + "threadId": { + "type": "string" + } + }, + "required": [ + "threadId" + ], + "type": "object" + }, "ThreadStartedNotification": { "properties": { "thread": { @@ -3453,6 +3567,106 @@ "title": "FuzzyFileSearch/sessionCompletedNotification", "type": "object" }, + { + "properties": { + "method": { + "enum": [ + "thread/realtime/started" + ], + "title": "Thread/realtime/startedNotificationMethod", + "type": "string" + }, + "params": { + "$ref": "#/definitions/ThreadRealtimeStartedNotification" + } + }, + "required": [ + "method", + "params" + ], + "title": "Thread/realtime/startedNotification", + "type": "object" + }, + { + "properties": { + "method": { + "enum": [ + "thread/realtime/itemAdded" + ], + "title": "Thread/realtime/itemAddedNotificationMethod", + "type": "string" + }, + "params": { + "$ref": "#/definitions/ThreadRealtimeItemAddedNotification" + } + }, + "required": [ + "method", + "params" + ], + "title": "Thread/realtime/itemAddedNotification", + "type": "object" + }, + { + "properties": { + "method": { + "enum": [ + "thread/realtime/outputAudio/delta" + ], + "title": "Thread/realtime/outputAudio/deltaNotificationMethod", + "type": "string" + }, + "params": { + "$ref": "#/definitions/ThreadRealtimeOutputAudioDeltaNotification" + } + }, + "required": [ + "method", + "params" + ], + "title": "Thread/realtime/outputAudio/deltaNotification", + "type": "object" + }, + { + "properties": { + "method": { + "enum": [ + "thread/realtime/error" + ], + "title": "Thread/realtime/errorNotificationMethod", + "type": "string" + }, + "params": { + "$ref": "#/definitions/ThreadRealtimeErrorNotification" + } + }, + "required": [ + "method", + "params" + ], + "title": "Thread/realtime/errorNotification", + "type": "object" + }, + { + "properties": { + "method": { + "enum": [ + "thread/realtime/closed" + ], + "title": "Thread/realtime/closedNotificationMethod", + "type": "string" + }, + "params": { + "$ref": "#/definitions/ThreadRealtimeClosedNotification" + } + }, + "required": [ + "method", + "params" + ], + "title": "Thread/realtime/closedNotification", + "type": "object" + }, { "description": "Notifies the user of world-writable directories on Windows, which cannot be protected by the sandbox.", "properties": { diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index 4282f1c050..4bceb1d98d 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -6425,6 +6425,106 @@ "title": "FuzzyFileSearch/sessionCompletedNotification", "type": "object" }, + { + "properties": { + "method": { + "enum": [ + "thread/realtime/started" + ], + "title": "Thread/realtime/startedNotificationMethod", + "type": "string" + }, + "params": { + "$ref": "#/definitions/v2/ThreadRealtimeStartedNotification" + } + }, + "required": [ + "method", + "params" + ], + "title": "Thread/realtime/startedNotification", + "type": "object" + }, + { + "properties": { + "method": { + "enum": [ + "thread/realtime/itemAdded" + ], + "title": "Thread/realtime/itemAddedNotificationMethod", + "type": "string" + }, + "params": { + "$ref": "#/definitions/v2/ThreadRealtimeItemAddedNotification" + } + }, + "required": [ + "method", + "params" + ], + "title": "Thread/realtime/itemAddedNotification", + "type": "object" + }, + { + "properties": { + "method": { + "enum": [ + "thread/realtime/outputAudio/delta" + ], + "title": "Thread/realtime/outputAudio/deltaNotificationMethod", + "type": "string" + }, + "params": { + "$ref": "#/definitions/v2/ThreadRealtimeOutputAudioDeltaNotification" + } + }, + "required": [ + "method", + "params" + ], + "title": "Thread/realtime/outputAudio/deltaNotification", + "type": "object" + }, + { + "properties": { + "method": { + "enum": [ + "thread/realtime/error" + ], + "title": "Thread/realtime/errorNotificationMethod", + "type": "string" + }, + "params": { + "$ref": "#/definitions/v2/ThreadRealtimeErrorNotification" + } + }, + "required": [ + "method", + "params" + ], + "title": "Thread/realtime/errorNotification", + "type": "object" + }, + { + "properties": { + "method": { + "enum": [ + "thread/realtime/closed" + ], + "title": "Thread/realtime/closedNotificationMethod", + "type": "string" + }, + "params": { + "$ref": "#/definitions/v2/ThreadRealtimeClosedNotification" + } + }, + "required": [ + "method", + "params" + ], + "title": "Thread/realtime/closedNotification", + "type": "object" + }, { "description": "Notifies the user of world-writable directories on Windows, which cannot be protected by the sandbox.", "properties": { @@ -13185,6 +13285,130 @@ "title": "ThreadReadResponse", "type": "object" }, + "ThreadRealtimeAudioChunk": { + "description": "EXPERIMENTAL - thread realtime audio chunk.", + "properties": { + "data": { + "type": "string" + }, + "numChannels": { + "format": "uint16", + "minimum": 0.0, + "type": "integer" + }, + "sampleRate": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "samplesPerChannel": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "data", + "numChannels", + "sampleRate" + ], + "type": "object" + }, + "ThreadRealtimeClosedNotification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "EXPERIMENTAL - emitted when thread realtime transport closes.", + "properties": { + "reason": { + "type": [ + "string", + "null" + ] + }, + "threadId": { + "type": "string" + } + }, + "required": [ + "threadId" + ], + "title": "ThreadRealtimeClosedNotification", + "type": "object" + }, + "ThreadRealtimeErrorNotification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "EXPERIMENTAL - emitted when thread realtime encounters an error.", + "properties": { + "message": { + "type": "string" + }, + "threadId": { + "type": "string" + } + }, + "required": [ + "message", + "threadId" + ], + "title": "ThreadRealtimeErrorNotification", + "type": "object" + }, + "ThreadRealtimeItemAddedNotification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "EXPERIMENTAL - raw non-audio thread realtime item emitted by the backend.", + "properties": { + "item": true, + "threadId": { + "type": "string" + } + }, + "required": [ + "item", + "threadId" + ], + "title": "ThreadRealtimeItemAddedNotification", + "type": "object" + }, + "ThreadRealtimeOutputAudioDeltaNotification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "EXPERIMENTAL - streamed output audio emitted by thread realtime.", + "properties": { + "audio": { + "$ref": "#/definitions/v2/ThreadRealtimeAudioChunk" + }, + "threadId": { + "type": "string" + } + }, + "required": [ + "audio", + "threadId" + ], + "title": "ThreadRealtimeOutputAudioDeltaNotification", + "type": "object" + }, + "ThreadRealtimeStartedNotification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "EXPERIMENTAL - emitted when thread realtime startup is accepted.", + "properties": { + "sessionId": { + "type": [ + "string", + "null" + ] + }, + "threadId": { + "type": "string" + } + }, + "required": [ + "threadId" + ], + "title": "ThreadRealtimeStartedNotification", + "type": "object" + }, "ThreadResumeParams": { "$schema": "http://json-schema.org/draft-07/schema#", "description": "There are three ways to resume a thread: 1. By thread_id: load the thread from disk by thread_id and resume it. 2. By history: instantiate the thread from memory and resume it. 3. By path: load the thread from disk by path and resume it.\n\nThe precedence is: history > path > thread_id. If using history or path, the thread_id param will be ignored.\n\nPrefer using thread_id whenever possible.", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeClosedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeClosedNotification.json new file mode 100644 index 0000000000..edfba83b7b --- /dev/null +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeClosedNotification.json @@ -0,0 +1,20 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "EXPERIMENTAL - emitted when thread realtime transport closes.", + "properties": { + "reason": { + "type": [ + "string", + "null" + ] + }, + "threadId": { + "type": "string" + } + }, + "required": [ + "threadId" + ], + "title": "ThreadRealtimeClosedNotification", + "type": "object" +} \ No newline at end of file diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeErrorNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeErrorNotification.json new file mode 100644 index 0000000000..e7ec760302 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeErrorNotification.json @@ -0,0 +1,18 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "EXPERIMENTAL - emitted when thread realtime encounters an error.", + "properties": { + "message": { + "type": "string" + }, + "threadId": { + "type": "string" + } + }, + "required": [ + "message", + "threadId" + ], + "title": "ThreadRealtimeErrorNotification", + "type": "object" +} \ No newline at end of file diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeItemAddedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeItemAddedNotification.json new file mode 100644 index 0000000000..06de7e00e7 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeItemAddedNotification.json @@ -0,0 +1,16 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "EXPERIMENTAL - raw non-audio thread realtime item emitted by the backend.", + "properties": { + "item": true, + "threadId": { + "type": "string" + } + }, + "required": [ + "item", + "threadId" + ], + "title": "ThreadRealtimeItemAddedNotification", + "type": "object" +} \ No newline at end of file diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeOutputAudioDeltaNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeOutputAudioDeltaNotification.json new file mode 100644 index 0000000000..d4df6194fa --- /dev/null +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeOutputAudioDeltaNotification.json @@ -0,0 +1,52 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "ThreadRealtimeAudioChunk": { + "description": "EXPERIMENTAL - thread realtime audio chunk.", + "properties": { + "data": { + "type": "string" + }, + "numChannels": { + "format": "uint16", + "minimum": 0.0, + "type": "integer" + }, + "sampleRate": { + "format": "uint32", + "minimum": 0.0, + "type": "integer" + }, + "samplesPerChannel": { + "format": "uint32", + "minimum": 0.0, + "type": [ + "integer", + "null" + ] + } + }, + "required": [ + "data", + "numChannels", + "sampleRate" + ], + "type": "object" + } + }, + "description": "EXPERIMENTAL - streamed output audio emitted by thread realtime.", + "properties": { + "audio": { + "$ref": "#/definitions/ThreadRealtimeAudioChunk" + }, + "threadId": { + "type": "string" + } + }, + "required": [ + "audio", + "threadId" + ], + "title": "ThreadRealtimeOutputAudioDeltaNotification", + "type": "object" +} \ No newline at end of file diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeStartedNotification.json new file mode 100644 index 0000000000..1584112640 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeStartedNotification.json @@ -0,0 +1,20 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "EXPERIMENTAL - emitted when thread realtime startup is accepted.", + "properties": { + "sessionId": { + "type": [ + "string", + "null" + ] + }, + "threadId": { + "type": "string" + } + }, + "required": [ + "threadId" + ], + "title": "ThreadRealtimeStartedNotification", + "type": "object" +} \ No newline at end of file diff --git a/codex-rs/app-server-protocol/schema/typescript/ServerNotification.ts b/codex-rs/app-server-protocol/schema/typescript/ServerNotification.ts index 20dfa846f5..92fecbfccd 100644 --- a/codex-rs/app-server-protocol/schema/typescript/ServerNotification.ts +++ b/codex-rs/app-server-protocol/schema/typescript/ServerNotification.ts @@ -30,6 +30,11 @@ import type { ReasoningTextDeltaNotification } from "./v2/ReasoningTextDeltaNoti import type { TerminalInteractionNotification } from "./v2/TerminalInteractionNotification"; import type { ThreadArchivedNotification } from "./v2/ThreadArchivedNotification"; import type { ThreadNameUpdatedNotification } from "./v2/ThreadNameUpdatedNotification"; +import type { ThreadRealtimeClosedNotification } from "./v2/ThreadRealtimeClosedNotification"; +import type { ThreadRealtimeErrorNotification } from "./v2/ThreadRealtimeErrorNotification"; +import type { ThreadRealtimeItemAddedNotification } from "./v2/ThreadRealtimeItemAddedNotification"; +import type { ThreadRealtimeOutputAudioDeltaNotification } from "./v2/ThreadRealtimeOutputAudioDeltaNotification"; +import type { ThreadRealtimeStartedNotification } from "./v2/ThreadRealtimeStartedNotification"; import type { ThreadStartedNotification } from "./v2/ThreadStartedNotification"; import type { ThreadStatusChangedNotification } from "./v2/ThreadStatusChangedNotification"; import type { ThreadTokenUsageUpdatedNotification } from "./v2/ThreadTokenUsageUpdatedNotification"; @@ -44,4 +49,4 @@ import type { WindowsWorldWritableWarningNotification } from "./v2/WindowsWorldW /** * Notification sent from the server to the client. */ -export type ServerNotification = { "method": "error", "params": ErrorNotification } | { "method": "thread/started", "params": ThreadStartedNotification } | { "method": "thread/status/changed", "params": ThreadStatusChangedNotification } | { "method": "thread/archived", "params": ThreadArchivedNotification } | { "method": "thread/unarchived", "params": ThreadUnarchivedNotification } | { "method": "thread/name/updated", "params": ThreadNameUpdatedNotification } | { "method": "thread/tokenUsage/updated", "params": ThreadTokenUsageUpdatedNotification } | { "method": "turn/started", "params": TurnStartedNotification } | { "method": "turn/completed", "params": TurnCompletedNotification } | { "method": "turn/diff/updated", "params": TurnDiffUpdatedNotification } | { "method": "turn/plan/updated", "params": TurnPlanUpdatedNotification } | { "method": "item/started", "params": ItemStartedNotification } | { "method": "item/completed", "params": ItemCompletedNotification } | { "method": "rawResponseItem/completed", "params": RawResponseItemCompletedNotification } | { "method": "item/agentMessage/delta", "params": AgentMessageDeltaNotification } | { "method": "item/plan/delta", "params": PlanDeltaNotification } | { "method": "item/commandExecution/outputDelta", "params": CommandExecutionOutputDeltaNotification } | { "method": "item/commandExecution/terminalInteraction", "params": TerminalInteractionNotification } | { "method": "item/fileChange/outputDelta", "params": FileChangeOutputDeltaNotification } | { "method": "item/mcpToolCall/progress", "params": McpToolCallProgressNotification } | { "method": "mcpServer/oauthLogin/completed", "params": McpServerOauthLoginCompletedNotification } | { "method": "account/updated", "params": AccountUpdatedNotification } | { "method": "account/rateLimits/updated", "params": AccountRateLimitsUpdatedNotification } | { "method": "app/list/updated", "params": AppListUpdatedNotification } | { "method": "item/reasoning/summaryTextDelta", "params": ReasoningSummaryTextDeltaNotification } | { "method": "item/reasoning/summaryPartAdded", "params": ReasoningSummaryPartAddedNotification } | { "method": "item/reasoning/textDelta", "params": ReasoningTextDeltaNotification } | { "method": "thread/compacted", "params": ContextCompactedNotification } | { "method": "model/rerouted", "params": ModelReroutedNotification } | { "method": "deprecationNotice", "params": DeprecationNoticeNotification } | { "method": "configWarning", "params": ConfigWarningNotification } | { "method": "fuzzyFileSearch/sessionUpdated", "params": FuzzyFileSearchSessionUpdatedNotification } | { "method": "fuzzyFileSearch/sessionCompleted", "params": FuzzyFileSearchSessionCompletedNotification } | { "method": "windows/worldWritableWarning", "params": WindowsWorldWritableWarningNotification } | { "method": "windowsSandbox/setupCompleted", "params": WindowsSandboxSetupCompletedNotification } | { "method": "account/login/completed", "params": AccountLoginCompletedNotification } | { "method": "authStatusChange", "params": AuthStatusChangeNotification } | { "method": "loginChatGptComplete", "params": LoginChatGptCompleteNotification } | { "method": "sessionConfigured", "params": SessionConfiguredNotification }; +export type ServerNotification = { "method": "error", "params": ErrorNotification } | { "method": "thread/started", "params": ThreadStartedNotification } | { "method": "thread/status/changed", "params": ThreadStatusChangedNotification } | { "method": "thread/archived", "params": ThreadArchivedNotification } | { "method": "thread/unarchived", "params": ThreadUnarchivedNotification } | { "method": "thread/name/updated", "params": ThreadNameUpdatedNotification } | { "method": "thread/tokenUsage/updated", "params": ThreadTokenUsageUpdatedNotification } | { "method": "turn/started", "params": TurnStartedNotification } | { "method": "turn/completed", "params": TurnCompletedNotification } | { "method": "turn/diff/updated", "params": TurnDiffUpdatedNotification } | { "method": "turn/plan/updated", "params": TurnPlanUpdatedNotification } | { "method": "item/started", "params": ItemStartedNotification } | { "method": "item/completed", "params": ItemCompletedNotification } | { "method": "rawResponseItem/completed", "params": RawResponseItemCompletedNotification } | { "method": "item/agentMessage/delta", "params": AgentMessageDeltaNotification } | { "method": "item/plan/delta", "params": PlanDeltaNotification } | { "method": "item/commandExecution/outputDelta", "params": CommandExecutionOutputDeltaNotification } | { "method": "item/commandExecution/terminalInteraction", "params": TerminalInteractionNotification } | { "method": "item/fileChange/outputDelta", "params": FileChangeOutputDeltaNotification } | { "method": "item/mcpToolCall/progress", "params": McpToolCallProgressNotification } | { "method": "mcpServer/oauthLogin/completed", "params": McpServerOauthLoginCompletedNotification } | { "method": "account/updated", "params": AccountUpdatedNotification } | { "method": "account/rateLimits/updated", "params": AccountRateLimitsUpdatedNotification } | { "method": "app/list/updated", "params": AppListUpdatedNotification } | { "method": "item/reasoning/summaryTextDelta", "params": ReasoningSummaryTextDeltaNotification } | { "method": "item/reasoning/summaryPartAdded", "params": ReasoningSummaryPartAddedNotification } | { "method": "item/reasoning/textDelta", "params": ReasoningTextDeltaNotification } | { "method": "thread/compacted", "params": ContextCompactedNotification } | { "method": "model/rerouted", "params": ModelReroutedNotification } | { "method": "deprecationNotice", "params": DeprecationNoticeNotification } | { "method": "configWarning", "params": ConfigWarningNotification } | { "method": "fuzzyFileSearch/sessionUpdated", "params": FuzzyFileSearchSessionUpdatedNotification } | { "method": "fuzzyFileSearch/sessionCompleted", "params": FuzzyFileSearchSessionCompletedNotification } | { "method": "thread/realtime/started", "params": ThreadRealtimeStartedNotification } | { "method": "thread/realtime/itemAdded", "params": ThreadRealtimeItemAddedNotification } | { "method": "thread/realtime/outputAudio/delta", "params": ThreadRealtimeOutputAudioDeltaNotification } | { "method": "thread/realtime/error", "params": ThreadRealtimeErrorNotification } | { "method": "thread/realtime/closed", "params": ThreadRealtimeClosedNotification } | { "method": "windows/worldWritableWarning", "params": WindowsWorldWritableWarningNotification } | { "method": "windowsSandbox/setupCompleted", "params": WindowsSandboxSetupCompletedNotification } | { "method": "account/login/completed", "params": AccountLoginCompletedNotification } | { "method": "authStatusChange", "params": AuthStatusChangeNotification } | { "method": "loginChatGptComplete", "params": LoginChatGptCompleteNotification } | { "method": "sessionConfigured", "params": SessionConfiguredNotification }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeAudioChunk.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeAudioChunk.ts new file mode 100644 index 0000000000..078f642247 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeAudioChunk.ts @@ -0,0 +1,8 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +/** + * EXPERIMENTAL - thread realtime audio chunk. + */ +export type ThreadRealtimeAudioChunk = { data: string, sampleRate: number, numChannels: number, samplesPerChannel: number | null, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeClosedNotification.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeClosedNotification.ts new file mode 100644 index 0000000000..a39cd71ed7 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeClosedNotification.ts @@ -0,0 +1,8 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +/** + * EXPERIMENTAL - emitted when thread realtime transport closes. + */ +export type ThreadRealtimeClosedNotification = { threadId: string, reason: string | null, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeErrorNotification.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeErrorNotification.ts new file mode 100644 index 0000000000..0b24879ecd --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeErrorNotification.ts @@ -0,0 +1,8 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +/** + * EXPERIMENTAL - emitted when thread realtime encounters an error. + */ +export type ThreadRealtimeErrorNotification = { threadId: string, message: string, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeItemAddedNotification.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeItemAddedNotification.ts new file mode 100644 index 0000000000..f996e77cd8 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeItemAddedNotification.ts @@ -0,0 +1,9 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { JsonValue } from "../serde_json/JsonValue"; + +/** + * EXPERIMENTAL - raw non-audio thread realtime item emitted by the backend. + */ +export type ThreadRealtimeItemAddedNotification = { threadId: string, item: JsonValue, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeOutputAudioDeltaNotification.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeOutputAudioDeltaNotification.ts new file mode 100644 index 0000000000..1d03fd89cf --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeOutputAudioDeltaNotification.ts @@ -0,0 +1,9 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { ThreadRealtimeAudioChunk } from "./ThreadRealtimeAudioChunk"; + +/** + * EXPERIMENTAL - streamed output audio emitted by thread realtime. + */ +export type ThreadRealtimeOutputAudioDeltaNotification = { threadId: string, audio: ThreadRealtimeAudioChunk, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeStartedNotification.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeStartedNotification.ts new file mode 100644 index 0000000000..736ecde1fe --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeStartedNotification.ts @@ -0,0 +1,8 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +/** + * EXPERIMENTAL - emitted when thread realtime startup is accepted. + */ +export type ThreadRealtimeStartedNotification = { threadId: string, sessionId: string | null, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/index.ts b/codex-rs/app-server-protocol/schema/typescript/v2/index.ts index 5d033d3636..3f9041dcfe 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/index.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/index.ts @@ -179,6 +179,12 @@ export type { ThreadLoadedListResponse } from "./ThreadLoadedListResponse"; export type { ThreadNameUpdatedNotification } from "./ThreadNameUpdatedNotification"; export type { ThreadReadParams } from "./ThreadReadParams"; export type { ThreadReadResponse } from "./ThreadReadResponse"; +export type { ThreadRealtimeAudioChunk } from "./ThreadRealtimeAudioChunk"; +export type { ThreadRealtimeClosedNotification } from "./ThreadRealtimeClosedNotification"; +export type { ThreadRealtimeErrorNotification } from "./ThreadRealtimeErrorNotification"; +export type { ThreadRealtimeItemAddedNotification } from "./ThreadRealtimeItemAddedNotification"; +export type { ThreadRealtimeOutputAudioDeltaNotification } from "./ThreadRealtimeOutputAudioDeltaNotification"; +export type { ThreadRealtimeStartedNotification } from "./ThreadRealtimeStartedNotification"; export type { ThreadResumeParams } from "./ThreadResumeParams"; export type { ThreadResumeResponse } from "./ThreadResumeResponse"; export type { ThreadRollbackParams } from "./ThreadRollbackParams"; diff --git a/codex-rs/app-server-protocol/src/protocol/common.rs b/codex-rs/app-server-protocol/src/protocol/common.rs index b7df2593be..7a918ada09 100644 --- a/codex-rs/app-server-protocol/src/protocol/common.rs +++ b/codex-rs/app-server-protocol/src/protocol/common.rs @@ -7,6 +7,7 @@ use crate::export::GeneratedSchema; use crate::export::write_json_schema; use crate::protocol::v1; use crate::protocol::v2; +use codex_experimental_api_macros::ExperimentalApi; use schemars::JsonSchema; use serde::Deserialize; use serde::Serialize; @@ -268,6 +269,26 @@ client_request_definitions! { params: v2::TurnInterruptParams, response: v2::TurnInterruptResponse, }, + #[experimental("thread/realtime/start")] + ThreadRealtimeStart => "thread/realtime/start" { + params: v2::ThreadRealtimeStartParams, + response: v2::ThreadRealtimeStartResponse, + }, + #[experimental("thread/realtime/appendAudio")] + ThreadRealtimeAppendAudio => "thread/realtime/appendAudio" { + params: v2::ThreadRealtimeAppendAudioParams, + response: v2::ThreadRealtimeAppendAudioResponse, + }, + #[experimental("thread/realtime/appendText")] + ThreadRealtimeAppendText => "thread/realtime/appendText" { + params: v2::ThreadRealtimeAppendTextParams, + response: v2::ThreadRealtimeAppendTextResponse, + }, + #[experimental("thread/realtime/stop")] + ThreadRealtimeStop => "thread/realtime/stop" { + params: v2::ThreadRealtimeStopParams, + response: v2::ThreadRealtimeStopResponse, + }, ReviewStart => "review/start" { params: v2::ReviewStartParams, response: v2::ReviewStartResponse, @@ -586,7 +607,16 @@ macro_rules! server_notification_definitions { ),* $(,)? ) => { /// Notification sent from the server to the client. - #[derive(Serialize, Deserialize, Debug, Clone, JsonSchema, TS, Display)] + #[derive( + Serialize, + Deserialize, + Debug, + Clone, + JsonSchema, + TS, + Display, + ExperimentalApi, + )] #[serde(tag = "method", content = "params", rename_all = "camelCase")] #[strum(serialize_all = "camelCase")] pub enum ServerNotification { @@ -823,6 +853,16 @@ server_notification_definitions! { ConfigWarning => "configWarning" (v2::ConfigWarningNotification), FuzzyFileSearchSessionUpdated => "fuzzyFileSearch/sessionUpdated" (FuzzyFileSearchSessionUpdatedNotification), FuzzyFileSearchSessionCompleted => "fuzzyFileSearch/sessionCompleted" (FuzzyFileSearchSessionCompletedNotification), + #[experimental("thread/realtime/started")] + ThreadRealtimeStarted => "thread/realtime/started" (v2::ThreadRealtimeStartedNotification), + #[experimental("thread/realtime/itemAdded")] + ThreadRealtimeItemAdded => "thread/realtime/itemAdded" (v2::ThreadRealtimeItemAddedNotification), + #[experimental("thread/realtime/outputAudio/delta")] + ThreadRealtimeOutputAudioDelta => "thread/realtime/outputAudio/delta" (v2::ThreadRealtimeOutputAudioDeltaNotification), + #[experimental("thread/realtime/error")] + ThreadRealtimeError => "thread/realtime/error" (v2::ThreadRealtimeErrorNotification), + #[experimental("thread/realtime/closed")] + ThreadRealtimeClosed => "thread/realtime/closed" (v2::ThreadRealtimeClosedNotification), /// Notifies the user of world-writable directories on Windows, which cannot be protected by the sandbox. WindowsWorldWritableWarning => "windows/worldWritableWarning" (v2::WindowsWorldWritableWarningNotification), @@ -1350,6 +1390,31 @@ mod tests { Ok(()) } + #[test] + fn serialize_thread_realtime_start() -> Result<()> { + let request = ClientRequest::ThreadRealtimeStart { + request_id: RequestId::Integer(9), + params: v2::ThreadRealtimeStartParams { + thread_id: "thr_123".to_string(), + prompt: "You are on a call".to_string(), + session_id: Some("sess_456".to_string()), + }, + }; + assert_eq!( + json!({ + "method": "thread/realtime/start", + "id": 9, + "params": { + "threadId": "thr_123", + "prompt": "You are on a call", + "sessionId": "sess_456" + } + }), + serde_json::to_value(&request)?, + ); + Ok(()) + } + #[test] fn serialize_thread_status_changed_notification() -> Result<()> { let notification = @@ -1372,6 +1437,37 @@ mod tests { Ok(()) } + #[test] + fn serialize_thread_realtime_output_audio_delta_notification() -> Result<()> { + let notification = ServerNotification::ThreadRealtimeOutputAudioDelta( + v2::ThreadRealtimeOutputAudioDeltaNotification { + thread_id: "thr_123".to_string(), + audio: v2::ThreadRealtimeAudioChunk { + data: "AQID".to_string(), + sample_rate: 24_000, + num_channels: 1, + samples_per_channel: Some(512), + }, + }, + ); + assert_eq!( + json!({ + "method": "thread/realtime/outputAudio/delta", + "params": { + "threadId": "thr_123", + "audio": { + "data": "AQID", + "sampleRate": 24000, + "numChannels": 1, + "samplesPerChannel": 512 + } + } + }), + serde_json::to_value(¬ification)?, + ); + Ok(()) + } + #[test] fn mock_experimental_method_is_marked_experimental() { let request = ClientRequest::MockExperimentalMethod { @@ -1381,6 +1477,46 @@ mod tests { let reason = crate::experimental_api::ExperimentalApi::experimental_reason(&request); assert_eq!(reason, Some("mock/experimentalMethod")); } + #[test] + fn thread_realtime_start_is_marked_experimental() { + let request = ClientRequest::ThreadRealtimeStart { + request_id: RequestId::Integer(1), + params: v2::ThreadRealtimeStartParams { + thread_id: "thr_123".to_string(), + prompt: "You are on a call".to_string(), + session_id: None, + }, + }; + let reason = crate::experimental_api::ExperimentalApi::experimental_reason(&request); + assert_eq!(reason, Some("thread/realtime/start")); + } + #[test] + fn thread_realtime_started_notification_is_marked_experimental() { + let notification = + ServerNotification::ThreadRealtimeStarted(v2::ThreadRealtimeStartedNotification { + thread_id: "thr_123".to_string(), + session_id: Some("sess_456".to_string()), + }); + let reason = crate::experimental_api::ExperimentalApi::experimental_reason(¬ification); + assert_eq!(reason, Some("thread/realtime/started")); + } + + #[test] + fn thread_realtime_output_audio_delta_notification_is_marked_experimental() { + let notification = ServerNotification::ThreadRealtimeOutputAudioDelta( + v2::ThreadRealtimeOutputAudioDeltaNotification { + thread_id: "thr_123".to_string(), + audio: v2::ThreadRealtimeAudioChunk { + data: "AQID".to_string(), + sample_rate: 24_000, + num_channels: 1, + samples_per_channel: Some(512), + }, + }, + ); + let reason = crate::experimental_api::ExperimentalApi::experimental_reason(¬ification); + assert_eq!(reason, Some("thread/realtime/outputAudio/delta")); + } #[test] fn command_execution_request_approval_additional_permissions_is_marked_experimental() { diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index 22b7fe05cf..1e4e5d0c31 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -46,6 +46,7 @@ use codex_protocol::protocol::PatchApplyStatus as CorePatchApplyStatus; use codex_protocol::protocol::RateLimitSnapshot as CoreRateLimitSnapshot; use codex_protocol::protocol::RateLimitWindow as CoreRateLimitWindow; use codex_protocol::protocol::ReadOnlyAccess as CoreReadOnlyAccess; +use codex_protocol::protocol::RealtimeAudioFrame as CoreRealtimeAudioFrame; use codex_protocol::protocol::RejectConfig as CoreRejectConfig; use codex_protocol::protocol::SessionSource as CoreSessionSource; use codex_protocol::protocol::SkillDependencies as CoreSkillDependencies; @@ -2551,6 +2552,157 @@ pub struct ErrorNotification { pub turn_id: String, } +/// EXPERIMENTAL - thread realtime audio chunk. +#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadRealtimeAudioChunk { + pub data: String, + pub sample_rate: u32, + pub num_channels: u16, + pub samples_per_channel: Option, +} + +impl From for ThreadRealtimeAudioChunk { + fn from(value: CoreRealtimeAudioFrame) -> Self { + let CoreRealtimeAudioFrame { + data, + sample_rate, + num_channels, + samples_per_channel, + } = value; + Self { + data, + sample_rate, + num_channels, + samples_per_channel, + } + } +} + +impl From for CoreRealtimeAudioFrame { + fn from(value: ThreadRealtimeAudioChunk) -> Self { + let ThreadRealtimeAudioChunk { + data, + sample_rate, + num_channels, + samples_per_channel, + } = value; + Self { + data, + sample_rate, + num_channels, + samples_per_channel, + } + } +} + +/// EXPERIMENTAL - start a thread-scoped realtime session. +#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadRealtimeStartParams { + pub thread_id: String, + pub prompt: String, + #[ts(optional = nullable)] + pub session_id: Option, +} + +/// EXPERIMENTAL - response for starting thread realtime. +#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadRealtimeStartResponse {} + +/// EXPERIMENTAL - append audio input to thread realtime. +#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadRealtimeAppendAudioParams { + pub thread_id: String, + pub audio: ThreadRealtimeAudioChunk, +} + +/// EXPERIMENTAL - response for appending realtime audio input. +#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadRealtimeAppendAudioResponse {} + +/// EXPERIMENTAL - append text input to thread realtime. +#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadRealtimeAppendTextParams { + pub thread_id: String, + pub text: String, +} + +/// EXPERIMENTAL - response for appending realtime text input. +#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadRealtimeAppendTextResponse {} + +/// EXPERIMENTAL - stop thread realtime. +#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadRealtimeStopParams { + pub thread_id: String, +} + +/// EXPERIMENTAL - response for stopping thread realtime. +#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadRealtimeStopResponse {} + +/// EXPERIMENTAL - emitted when thread realtime startup is accepted. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadRealtimeStartedNotification { + pub thread_id: String, + pub session_id: Option, +} + +/// EXPERIMENTAL - raw non-audio thread realtime item emitted by the backend. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadRealtimeItemAddedNotification { + pub thread_id: String, + pub item: JsonValue, +} + +/// EXPERIMENTAL - streamed output audio emitted by thread realtime. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadRealtimeOutputAudioDeltaNotification { + pub thread_id: String, + pub audio: ThreadRealtimeAudioChunk, +} + +/// EXPERIMENTAL - emitted when thread realtime encounters an error. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadRealtimeErrorNotification { + pub thread_id: String, + pub message: String, +} + +/// EXPERIMENTAL - emitted when thread realtime transport closes. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct ThreadRealtimeClosedNotification { + pub thread_id: String, + pub reason: Option, +} + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] #[serde(rename_all = "camelCase")] #[ts(export_to = "v2/")] diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index 0be424a400..08b7c8df0d 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -135,6 +135,10 @@ Example with notification opt-out: - `turn/start` — add user input to a thread and begin Codex generation; responds with the initial `turn` object and streams `turn/started`, `item/*`, and `turn/completed` notifications. For `collaborationMode`, `settings.developer_instructions: null` means "use built-in instructions for the selected mode". - `turn/steer` — add user input to an already in-flight turn without starting a new turn; returns the active `turnId` that accepted the input. - `turn/interrupt` — request cancellation of an in-flight turn by `(thread_id, turn_id)`; success is an empty `{}` response and the turn finishes with `status: "interrupted"`. +- `thread/realtime/start` — start a thread-scoped realtime session (experimental); returns `{}` and streams `thread/realtime/*` notifications. +- `thread/realtime/appendAudio` — append an input audio chunk to the active realtime session (experimental); returns `{}`. +- `thread/realtime/appendText` — append text input to the active realtime session (experimental); returns `{}`. +- `thread/realtime/stop` — stop the active realtime session for the thread (experimental); returns `{}`. - `review/start` — kick off Codex’s automated reviewer for a thread; responds like `turn/start` and emits `item/started`/`item/completed` notifications with `enteredReviewMode` and `exitedReviewMode` items, plus a final assistant `agentMessage` containing the review. - `command/exec` — run a single command under the server sandbox without starting a thread/turn (handy for utilities and validation). - `model/list` — list available models (set `includeHidden: true` to include entries with `hidden: true`), with reasoning effort options and optional `upgrade` model ids. @@ -553,6 +557,8 @@ Notes: Event notifications are the server-initiated event stream for thread lifecycles, turn lifecycles, and the items within them. After you start or resume a thread, keep reading stdout for `thread/started`, `thread/archived`, `thread/unarchived`, `turn/*`, and `item/*` notifications. +Thread realtime uses a separate thread-scoped notification surface. `thread/realtime/*` notifications are ephemeral transport events, not `ThreadItem`s, and are not returned by `thread/read`, `thread/resume`, or `thread/fork`. + ### Notification opt-out Clients can suppress specific notifications per connection by sending exact method names in `initialize.params.capabilities.optOutNotificationMethods`. @@ -574,6 +580,18 @@ The fuzzy file search session API emits per-query notifications: - `fuzzyFileSearch/sessionUpdated` — `{ sessionId, query, files }` with the current matching files for the active query. - `fuzzyFileSearch/sessionCompleted` — `{ sessionId, query }` once indexing/matching for that query has completed. +### Thread realtime events (experimental) + +The thread realtime API emits thread-scoped notifications for session lifecycle and streaming media: + +- `thread/realtime/started` — `{ threadId, sessionId }` once realtime starts for the thread (experimental). +- `thread/realtime/itemAdded` — `{ threadId, item }` for non-audio realtime items (experimental). `item` is forwarded as raw JSON while the upstream websocket item schema remains unstable. +- `thread/realtime/outputAudio/delta` — `{ threadId, audio }` for streamed output audio chunks (experimental). `audio` uses camelCase fields (`data`, `sampleRate`, `numChannels`, `samplesPerChannel`). +- `thread/realtime/error` — `{ threadId, message }` when realtime encounters a transport or backend error (experimental). +- `thread/realtime/closed` — `{ threadId, reason }` when the realtime transport closes (experimental). + +Because audio is intentionally separate from `ThreadItem`, clients can opt out of `thread/realtime/outputAudio/delta` independently with `optOutNotificationMethods`. + ### Windows sandbox setup events - `windowsSandbox/setupCompleted` — `{ mode, success, error }` after a `windowsSandbox/setupStart` request finishes. diff --git a/codex-rs/app-server/src/bespoke_event_handling.rs b/codex-rs/app-server/src/bespoke_event_handling.rs index 0b81a4c898..56e55823c8 100644 --- a/codex-rs/app-server/src/bespoke_event_handling.rs +++ b/codex-rs/app-server/src/bespoke_event_handling.rs @@ -62,6 +62,11 @@ use codex_app_server_protocol::SkillRequestApprovalResponse; use codex_app_server_protocol::TerminalInteractionNotification; use codex_app_server_protocol::ThreadItem; use codex_app_server_protocol::ThreadNameUpdatedNotification; +use codex_app_server_protocol::ThreadRealtimeClosedNotification; +use codex_app_server_protocol::ThreadRealtimeErrorNotification; +use codex_app_server_protocol::ThreadRealtimeItemAddedNotification; +use codex_app_server_protocol::ThreadRealtimeOutputAudioDeltaNotification; +use codex_app_server_protocol::ThreadRealtimeStartedNotification; use codex_app_server_protocol::ThreadRollbackResponse; use codex_app_server_protocol::ThreadTokenUsage; use codex_app_server_protocol::ThreadTokenUsageUpdatedNotification; @@ -97,6 +102,7 @@ use codex_protocol::protocol::ExecCommandEndEvent; use codex_protocol::protocol::McpToolCallBeginEvent; use codex_protocol::protocol::McpToolCallEndEvent; use codex_protocol::protocol::Op; +use codex_protocol::protocol::RealtimeEvent; use codex_protocol::protocol::ReviewDecision; use codex_protocol::protocol::ReviewOutputEvent; use codex_protocol::protocol::TokenCountEvent; @@ -173,6 +179,73 @@ pub(crate) async fn apply_bespoke_event_handling( .await; } } + EventMsg::RealtimeConversationStarted(event) => { + if let ApiVersion::V2 = api_version { + let notification = ThreadRealtimeStartedNotification { + thread_id: conversation_id.to_string(), + session_id: event.session_id, + }; + outgoing + .send_server_notification(ServerNotification::ThreadRealtimeStarted( + notification, + )) + .await; + } + } + EventMsg::RealtimeConversationRealtime(event) => { + if let ApiVersion::V2 = api_version { + match event.payload { + RealtimeEvent::SessionCreated { .. } => {} + RealtimeEvent::SessionUpdated { .. } => {} + RealtimeEvent::AudioOut(audio) => { + let notification = ThreadRealtimeOutputAudioDeltaNotification { + thread_id: conversation_id.to_string(), + audio: audio.into(), + }; + outgoing + .send_server_notification( + ServerNotification::ThreadRealtimeOutputAudioDelta(notification), + ) + .await; + } + RealtimeEvent::ConversationItemAdded(item) => { + let notification = ThreadRealtimeItemAddedNotification { + thread_id: conversation_id.to_string(), + item, + }; + outgoing + .send_server_notification(ServerNotification::ThreadRealtimeItemAdded( + notification, + )) + .await; + } + RealtimeEvent::Error(message) => { + let notification = ThreadRealtimeErrorNotification { + thread_id: conversation_id.to_string(), + message, + }; + outgoing + .send_server_notification(ServerNotification::ThreadRealtimeError( + notification, + )) + .await; + } + } + } + } + EventMsg::RealtimeConversationClosed(event) => { + if let ApiVersion::V2 = api_version { + let notification = ThreadRealtimeClosedNotification { + thread_id: conversation_id.to_string(), + reason: event.reason, + }; + outgoing + .send_server_notification(ServerNotification::ThreadRealtimeClosed( + notification, + )) + .await; + } + } EventMsg::ApplyPatchApprovalRequest(ApplyPatchApprovalRequestEvent { call_id, turn_id, diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index ebe1b77ed6..1897898718 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -138,6 +138,14 @@ use codex_app_server_protocol::ThreadLoadedListParams; use codex_app_server_protocol::ThreadLoadedListResponse; use codex_app_server_protocol::ThreadReadParams; use codex_app_server_protocol::ThreadReadResponse; +use codex_app_server_protocol::ThreadRealtimeAppendAudioParams; +use codex_app_server_protocol::ThreadRealtimeAppendAudioResponse; +use codex_app_server_protocol::ThreadRealtimeAppendTextParams; +use codex_app_server_protocol::ThreadRealtimeAppendTextResponse; +use codex_app_server_protocol::ThreadRealtimeStartParams; +use codex_app_server_protocol::ThreadRealtimeStartResponse; +use codex_app_server_protocol::ThreadRealtimeStopParams; +use codex_app_server_protocol::ThreadRealtimeStopResponse; use codex_app_server_protocol::ThreadResumeParams; use codex_app_server_protocol::ThreadResumeResponse; use codex_app_server_protocol::ThreadRollbackParams; @@ -235,6 +243,9 @@ use codex_protocol::dynamic_tools::DynamicToolSpec as CoreDynamicToolSpec; use codex_protocol::items::TurnItem; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::AgentStatus; +use codex_protocol::protocol::ConversationAudioParams; +use codex_protocol::protocol::ConversationStartParams; +use codex_protocol::protocol::ConversationTextParams; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::GitInfo as CoreGitInfo; use codex_protocol::protocol::InitialHistory; @@ -625,6 +636,22 @@ impl CodexMessageProcessor { self.turn_interrupt(to_connection_request_id(request_id), params) .await; } + ClientRequest::ThreadRealtimeStart { request_id, params } => { + self.thread_realtime_start(to_connection_request_id(request_id), params) + .await; + } + ClientRequest::ThreadRealtimeAppendAudio { request_id, params } => { + self.thread_realtime_append_audio(to_connection_request_id(request_id), params) + .await; + } + ClientRequest::ThreadRealtimeAppendText { request_id, params } => { + self.thread_realtime_append_text(to_connection_request_id(request_id), params) + .await; + } + ClientRequest::ThreadRealtimeStop { request_id, params } => { + self.thread_realtime_stop(to_connection_request_id(request_id), params) + .await; + } ClientRequest::ReviewStart { request_id, params } => { self.review_start(to_connection_request_id(request_id), params) .await; @@ -5518,6 +5545,177 @@ impl CodexMessageProcessor { } } + async fn prepare_realtime_conversation_thread( + &mut self, + request_id: ConnectionRequestId, + thread_id: &str, + ) -> Option<(ThreadId, Arc)> { + let (thread_id, thread) = match self.load_thread(thread_id).await { + Ok(v) => v, + Err(error) => { + self.outgoing.send_error(request_id, error).await; + return None; + } + }; + + if let Err(error) = self + .ensure_conversation_listener( + thread_id, + request_id.connection_id, + false, + ApiVersion::V2, + ) + .await + { + self.outgoing.send_error(request_id, error).await; + return None; + } + + if !thread.enabled(Feature::RealtimeConversation) { + self.send_invalid_request_error( + request_id, + format!("thread {thread_id} does not support realtime conversation"), + ) + .await; + return None; + } + + Some((thread_id, thread)) + } + + async fn thread_realtime_start( + &mut self, + request_id: ConnectionRequestId, + params: ThreadRealtimeStartParams, + ) { + let Some((_, thread)) = self + .prepare_realtime_conversation_thread(request_id.clone(), ¶ms.thread_id) + .await + else { + return; + }; + + let submit = thread + .submit(Op::RealtimeConversationStart(ConversationStartParams { + prompt: params.prompt, + session_id: params.session_id, + })) + .await; + + match submit { + Ok(_) => { + self.outgoing + .send_response(request_id, ThreadRealtimeStartResponse::default()) + .await; + } + Err(err) => { + self.send_internal_error( + request_id, + format!("failed to start realtime conversation: {err}"), + ) + .await; + } + } + } + + async fn thread_realtime_append_audio( + &mut self, + request_id: ConnectionRequestId, + params: ThreadRealtimeAppendAudioParams, + ) { + let Some((_, thread)) = self + .prepare_realtime_conversation_thread(request_id.clone(), ¶ms.thread_id) + .await + else { + return; + }; + + let submit = thread + .submit(Op::RealtimeConversationAudio(ConversationAudioParams { + frame: params.audio.into(), + })) + .await; + + match submit { + Ok(_) => { + self.outgoing + .send_response(request_id, ThreadRealtimeAppendAudioResponse::default()) + .await; + } + Err(err) => { + self.send_internal_error( + request_id, + format!("failed to append realtime conversation audio: {err}"), + ) + .await; + } + } + } + + async fn thread_realtime_append_text( + &mut self, + request_id: ConnectionRequestId, + params: ThreadRealtimeAppendTextParams, + ) { + let Some((_, thread)) = self + .prepare_realtime_conversation_thread(request_id.clone(), ¶ms.thread_id) + .await + else { + return; + }; + + let submit = thread + .submit(Op::RealtimeConversationText(ConversationTextParams { + text: params.text, + })) + .await; + + match submit { + Ok(_) => { + self.outgoing + .send_response(request_id, ThreadRealtimeAppendTextResponse::default()) + .await; + } + Err(err) => { + self.send_internal_error( + request_id, + format!("failed to append realtime conversation text: {err}"), + ) + .await; + } + } + } + + async fn thread_realtime_stop( + &mut self, + request_id: ConnectionRequestId, + params: ThreadRealtimeStopParams, + ) { + let Some((_, thread)) = self + .prepare_realtime_conversation_thread(request_id.clone(), ¶ms.thread_id) + .await + else { + return; + }; + + let submit = thread.submit(Op::RealtimeConversationClose).await; + + match submit { + Ok(_) => { + self.outgoing + .send_response(request_id, ThreadRealtimeStopResponse::default()) + .await; + } + Err(err) => { + self.send_internal_error( + request_id, + format!("failed to stop realtime conversation: {err}"), + ) + .await; + } + } + } + fn build_review_turn(turn_id: String, display_text: &str) -> Turn { let items = if display_text.is_empty() { Vec::new() diff --git a/codex-rs/app-server/tests/common/mcp_process.rs b/codex-rs/app-server/tests/common/mcp_process.rs index bb35c2c0ab..f1917add85 100644 --- a/codex-rs/app-server/tests/common/mcp_process.rs +++ b/codex-rs/app-server/tests/common/mcp_process.rs @@ -57,6 +57,10 @@ use codex_app_server_protocol::ThreadForkParams; use codex_app_server_protocol::ThreadListParams; use codex_app_server_protocol::ThreadLoadedListParams; use codex_app_server_protocol::ThreadReadParams; +use codex_app_server_protocol::ThreadRealtimeAppendAudioParams; +use codex_app_server_protocol::ThreadRealtimeAppendTextParams; +use codex_app_server_protocol::ThreadRealtimeStartParams; +use codex_app_server_protocol::ThreadRealtimeStopParams; use codex_app_server_protocol::ThreadResumeParams; use codex_app_server_protocol::ThreadRollbackParams; use codex_app_server_protocol::ThreadSetNameParams; @@ -584,6 +588,44 @@ impl McpProcess { self.send_request("turn/interrupt", params).await } + /// Send a `thread/realtime/start` JSON-RPC request (v2). + pub async fn send_thread_realtime_start_request( + &mut self, + params: ThreadRealtimeStartParams, + ) -> anyhow::Result { + let params = Some(serde_json::to_value(params)?); + self.send_request("thread/realtime/start", params).await + } + + /// Send a `thread/realtime/appendAudio` JSON-RPC request (v2). + pub async fn send_thread_realtime_append_audio_request( + &mut self, + params: ThreadRealtimeAppendAudioParams, + ) -> anyhow::Result { + let params = Some(serde_json::to_value(params)?); + self.send_request("thread/realtime/appendAudio", params) + .await + } + + /// Send a `thread/realtime/appendText` JSON-RPC request (v2). + pub async fn send_thread_realtime_append_text_request( + &mut self, + params: ThreadRealtimeAppendTextParams, + ) -> anyhow::Result { + let params = Some(serde_json::to_value(params)?); + self.send_request("thread/realtime/appendText", params) + .await + } + + /// Send a `thread/realtime/stop` JSON-RPC request (v2). + pub async fn send_thread_realtime_stop_request( + &mut self, + params: ThreadRealtimeStopParams, + ) -> anyhow::Result { + let params = Some(serde_json::to_value(params)?); + self.send_request("thread/realtime/stop", params).await + } + /// Deterministically clean up an intentionally in-flight turn. /// /// Some tests assert behavior while a turn is still running. Returning from those tests diff --git a/codex-rs/app-server/tests/suite/v2/experimental_api.rs b/codex-rs/app-server/tests/suite/v2/experimental_api.rs index 798d52abfd..9af3aa4c7a 100644 --- a/codex-rs/app-server/tests/suite/v2/experimental_api.rs +++ b/codex-rs/app-server/tests/suite/v2/experimental_api.rs @@ -10,6 +10,7 @@ use codex_app_server_protocol::JSONRPCMessage; use codex_app_server_protocol::JSONRPCResponse; use codex_app_server_protocol::MockExperimentalMethodParams; use codex_app_server_protocol::RequestId; +use codex_app_server_protocol::ThreadRealtimeStartParams; use codex_app_server_protocol::ThreadStartParams; use codex_app_server_protocol::ThreadStartResponse; use pretty_assertions::assert_eq; @@ -50,6 +51,40 @@ async fn mock_experimental_method_requires_experimental_api_capability() -> Resu Ok(()) } +#[tokio::test] +async fn realtime_conversation_start_requires_experimental_api_capability() -> Result<()> { + let codex_home = TempDir::new()?; + let mut mcp = McpProcess::new(codex_home.path()).await?; + + let init = mcp + .initialize_with_capabilities( + default_client_info(), + Some(InitializeCapabilities { + experimental_api: false, + opt_out_notification_methods: None, + }), + ) + .await?; + let JSONRPCMessage::Response(_) = init else { + anyhow::bail!("expected initialize response, got {init:?}"); + }; + + let request_id = mcp + .send_thread_realtime_start_request(ThreadRealtimeStartParams { + thread_id: "thr_123".to_string(), + prompt: "hello".to_string(), + session_id: None, + }) + .await?; + let error = timeout( + DEFAULT_TIMEOUT, + mcp.read_stream_until_error_message(RequestId::Integer(request_id)), + ) + .await??; + assert_experimental_capability_error(error, "thread/realtime/start"); + Ok(()) +} + #[tokio::test] async fn thread_start_mock_field_requires_experimental_api_capability() -> Result<()> { let server = create_mock_responses_server_sequence_unchecked(Vec::new()).await; diff --git a/codex-rs/app-server/tests/suite/v2/mod.rs b/codex-rs/app-server/tests/suite/v2/mod.rs index 67e7ae3ea6..8afb1e3f40 100644 --- a/codex-rs/app-server/tests/suite/v2/mod.rs +++ b/codex-rs/app-server/tests/suite/v2/mod.rs @@ -15,6 +15,7 @@ mod model_list; mod output_schema; mod plan_item; mod rate_limits; +mod realtime_conversation; mod request_user_input; mod review; mod safety_check_downgrade; diff --git a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs new file mode 100644 index 0000000000..f2664848ca --- /dev/null +++ b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs @@ -0,0 +1,392 @@ +use anyhow::Context; +use anyhow::Result; +use app_test_support::McpProcess; +use app_test_support::create_mock_responses_server_sequence_unchecked; +use app_test_support::to_response; +use codex_app_server_protocol::JSONRPCError; +use codex_app_server_protocol::JSONRPCResponse; +use codex_app_server_protocol::RequestId; +use codex_app_server_protocol::ThreadRealtimeAppendAudioParams; +use codex_app_server_protocol::ThreadRealtimeAppendAudioResponse; +use codex_app_server_protocol::ThreadRealtimeAppendTextParams; +use codex_app_server_protocol::ThreadRealtimeAppendTextResponse; +use codex_app_server_protocol::ThreadRealtimeAudioChunk; +use codex_app_server_protocol::ThreadRealtimeClosedNotification; +use codex_app_server_protocol::ThreadRealtimeErrorNotification; +use codex_app_server_protocol::ThreadRealtimeItemAddedNotification; +use codex_app_server_protocol::ThreadRealtimeOutputAudioDeltaNotification; +use codex_app_server_protocol::ThreadRealtimeStartParams; +use codex_app_server_protocol::ThreadRealtimeStartResponse; +use codex_app_server_protocol::ThreadRealtimeStartedNotification; +use codex_app_server_protocol::ThreadRealtimeStopParams; +use codex_app_server_protocol::ThreadRealtimeStopResponse; +use codex_app_server_protocol::ThreadStartParams; +use codex_app_server_protocol::ThreadStartResponse; +use codex_core::features::FEATURES; +use codex_core::features::Feature; +use core_test_support::responses::start_websocket_server; +use core_test_support::skip_if_no_network; +use pretty_assertions::assert_eq; +use serde::de::DeserializeOwned; +use serde_json::json; +use std::path::Path; +use std::time::Duration; +use tempfile::TempDir; +use tokio::time::timeout; + +const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10); + +#[tokio::test] +async fn realtime_conversation_streams_v2_notifications() -> Result<()> { + skip_if_no_network!(Ok(())); + + let responses_server = create_mock_responses_server_sequence_unchecked(Vec::new()).await; + let realtime_server = start_websocket_server(vec![vec![ + vec![json!({ + "type": "session.created", + "session": { "id": "sess_backend" } + })], + vec![json!({ + "type": "session.updated", + "session": { "backend_prompt": "backend prompt" } + })], + vec![ + json!({ + "type": "response.output_audio.delta", + "delta": "AQID", + "sample_rate": 24_000, + "num_channels": 1, + "samples_per_channel": 512 + }), + json!({ + "type": "conversation.item.added", + "item": { + "type": "message", + "role": "assistant", + "content": [{ "type": "text", "text": "hi" }] + } + }), + json!({ + "type": "error", + "message": "upstream boom" + }), + ], + ]]) + .await; + + let codex_home = TempDir::new()?; + create_config_toml( + codex_home.path(), + &responses_server.uri(), + realtime_server.uri(), + true, + )?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + mcp.initialize().await?; + + let thread_start_request_id = mcp + .send_thread_start_request(ThreadStartParams::default()) + .await?; + let thread_start_response: JSONRPCResponse = timeout( + DEFAULT_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(thread_start_request_id)), + ) + .await??; + let thread_start: ThreadStartResponse = to_response(thread_start_response)?; + + let start_request_id = mcp + .send_thread_realtime_start_request(ThreadRealtimeStartParams { + thread_id: thread_start.thread.id.clone(), + prompt: "backend prompt".to_string(), + session_id: None, + }) + .await?; + let start_response: JSONRPCResponse = timeout( + DEFAULT_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(start_request_id)), + ) + .await??; + let _: ThreadRealtimeStartResponse = to_response(start_response)?; + + let started = + read_notification::(&mut mcp, "thread/realtime/started") + .await?; + assert_eq!(started.thread_id, thread_start.thread.id); + assert!(started.session_id.is_some()); + + let audio_append_request_id = mcp + .send_thread_realtime_append_audio_request(ThreadRealtimeAppendAudioParams { + thread_id: started.thread_id.clone(), + audio: ThreadRealtimeAudioChunk { + data: "BQYH".to_string(), + sample_rate: 24_000, + num_channels: 1, + samples_per_channel: Some(480), + }, + }) + .await?; + let audio_append_response: JSONRPCResponse = timeout( + DEFAULT_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(audio_append_request_id)), + ) + .await??; + let _: ThreadRealtimeAppendAudioResponse = to_response(audio_append_response)?; + + let text_append_request_id = mcp + .send_thread_realtime_append_text_request(ThreadRealtimeAppendTextParams { + thread_id: started.thread_id.clone(), + text: "hello".to_string(), + }) + .await?; + let text_append_response: JSONRPCResponse = timeout( + DEFAULT_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(text_append_request_id)), + ) + .await??; + let _: ThreadRealtimeAppendTextResponse = to_response(text_append_response)?; + + let output_audio = read_notification::( + &mut mcp, + "thread/realtime/outputAudio/delta", + ) + .await?; + assert_eq!(output_audio.audio.data, "AQID"); + assert_eq!(output_audio.audio.sample_rate, 24_000); + assert_eq!(output_audio.audio.num_channels, 1); + assert_eq!(output_audio.audio.samples_per_channel, Some(512)); + + let item_added = read_notification::( + &mut mcp, + "thread/realtime/itemAdded", + ) + .await?; + assert_eq!(item_added.thread_id, output_audio.thread_id); + assert_eq!(item_added.item["type"], json!("message")); + + let realtime_error = + read_notification::(&mut mcp, "thread/realtime/error") + .await?; + assert_eq!(realtime_error.thread_id, output_audio.thread_id); + assert_eq!(realtime_error.message, "upstream boom"); + + let closed = + read_notification::(&mut mcp, "thread/realtime/closed") + .await?; + assert_eq!(closed.thread_id, output_audio.thread_id); + assert_eq!(closed.reason.as_deref(), Some("transport_closed")); + + let connections = realtime_server.connections(); + assert_eq!(connections.len(), 1); + let connection = &connections[0]; + assert_eq!(connection.len(), 3); + assert_eq!( + connection[0].body_json()["type"].as_str(), + Some("session.create") + ); + let mut request_types = [ + connection[1].body_json()["type"] + .as_str() + .context("expected websocket request type")? + .to_string(), + connection[2].body_json()["type"] + .as_str() + .context("expected websocket request type")? + .to_string(), + ]; + request_types.sort(); + assert_eq!( + request_types, + [ + "conversation.item.create".to_string(), + "response.input_audio.delta".to_string(), + ] + ); + + realtime_server.shutdown().await; + Ok(()) +} + +#[tokio::test] +async fn realtime_conversation_stop_emits_closed_notification() -> Result<()> { + skip_if_no_network!(Ok(())); + + let responses_server = create_mock_responses_server_sequence_unchecked(Vec::new()).await; + let realtime_server = start_websocket_server(vec![vec![ + vec![json!({ + "type": "session.created", + "session": { "id": "sess_backend" } + })], + vec![], + ]]) + .await; + + let codex_home = TempDir::new()?; + create_config_toml( + codex_home.path(), + &responses_server.uri(), + realtime_server.uri(), + true, + )?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + mcp.initialize().await?; + + let thread_start_request_id = mcp + .send_thread_start_request(ThreadStartParams::default()) + .await?; + let thread_start_response: JSONRPCResponse = timeout( + DEFAULT_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(thread_start_request_id)), + ) + .await??; + let thread_start: ThreadStartResponse = to_response(thread_start_response)?; + + let start_request_id = mcp + .send_thread_realtime_start_request(ThreadRealtimeStartParams { + thread_id: thread_start.thread.id.clone(), + prompt: "backend prompt".to_string(), + session_id: None, + }) + .await?; + let start_response: JSONRPCResponse = timeout( + DEFAULT_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(start_request_id)), + ) + .await??; + let _: ThreadRealtimeStartResponse = to_response(start_response)?; + + let started = + read_notification::(&mut mcp, "thread/realtime/started") + .await?; + + let stop_request_id = mcp + .send_thread_realtime_stop_request(ThreadRealtimeStopParams { + thread_id: started.thread_id.clone(), + }) + .await?; + let stop_response: JSONRPCResponse = timeout( + DEFAULT_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(stop_request_id)), + ) + .await??; + let _: ThreadRealtimeStopResponse = to_response(stop_response)?; + + let closed = + read_notification::(&mut mcp, "thread/realtime/closed") + .await?; + assert_eq!(closed.thread_id, started.thread_id); + assert!(matches!( + closed.reason.as_deref(), + Some("requested" | "transport_closed") + )); + + realtime_server.shutdown().await; + Ok(()) +} + +#[tokio::test] +async fn realtime_conversation_requires_feature_flag() -> Result<()> { + skip_if_no_network!(Ok(())); + + let responses_server = create_mock_responses_server_sequence_unchecked(Vec::new()).await; + let realtime_server = start_websocket_server(vec![vec![]]).await; + + let codex_home = TempDir::new()?; + create_config_toml( + codex_home.path(), + &responses_server.uri(), + realtime_server.uri(), + false, + )?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + mcp.initialize().await?; + + let thread_start_request_id = mcp + .send_thread_start_request(ThreadStartParams::default()) + .await?; + let thread_start_response: JSONRPCResponse = timeout( + DEFAULT_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(thread_start_request_id)), + ) + .await??; + let thread_start: ThreadStartResponse = to_response(thread_start_response)?; + + let start_request_id = mcp + .send_thread_realtime_start_request(ThreadRealtimeStartParams { + thread_id: thread_start.thread.id.clone(), + prompt: "backend prompt".to_string(), + session_id: None, + }) + .await?; + let error = timeout( + DEFAULT_TIMEOUT, + mcp.read_stream_until_error_message(RequestId::Integer(start_request_id)), + ) + .await??; + assert_invalid_request( + error, + format!( + "thread {} does not support realtime conversation", + thread_start.thread.id + ), + ); + + realtime_server.shutdown().await; + Ok(()) +} + +async fn read_notification(mcp: &mut McpProcess, method: &str) -> Result { + let notification = timeout( + DEFAULT_TIMEOUT, + mcp.read_stream_until_notification_message(method), + ) + .await??; + let params = notification + .params + .context("expected notification params to be present")?; + Ok(serde_json::from_value(params)?) +} + +fn create_config_toml( + codex_home: &Path, + responses_server_uri: &str, + realtime_server_uri: &str, + realtime_enabled: bool, +) -> std::io::Result<()> { + let realtime_feature_key = FEATURES + .iter() + .find(|spec| spec.id == Feature::RealtimeConversation) + .map(|spec| spec.key) + .unwrap_or("realtime_conversation"); + + std::fs::write( + codex_home.join("config.toml"), + format!( + r#" +model = "mock-model" +approval_policy = "never" +sandbox_mode = "read-only" +model_provider = "mock_provider" +experimental_realtime_ws_base_url = "{realtime_server_uri}" + +[features] +{realtime_feature_key} = {realtime_enabled} + +[model_providers.mock_provider] +name = "Mock provider for test" +base_url = "{responses_server_uri}/v1" +wire_api = "responses" +request_max_retries = 0 +stream_max_retries = 0 +"# + ), + ) +} + +fn assert_invalid_request(error: JSONRPCError, message: String) { + assert_eq!(error.error.code, -32600); + assert_eq!(error.error.message, message); + assert_eq!(error.error.data, None); +} From 7b39e76a668b43ceca16941556ca633a3951958b Mon Sep 17 00:00:00 2001 From: jif-oai Date: Wed, 25 Feb 2026 18:06:17 +0000 Subject: [PATCH 3/4] Revert "fix(bazel): replace askama templates with include_str! in memories" (#12795) Reverts openai/codex#11778 --- MODULE.bazel.lock | 4 ++ codex-rs/Cargo.lock | 53 +++++++++++++++++++++++++++ codex-rs/Cargo.toml | 1 + codex-rs/core/BUILD.bazel | 6 +++ codex-rs/core/Cargo.toml | 1 + codex-rs/core/src/memories/prompts.rs | 53 ++++++++++++++++++++------- 6 files changed, 105 insertions(+), 13 deletions(-) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 5777c6a36b..d3dfd0346e 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -617,6 +617,10 @@ "arrayvec_0.7.6": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bencher\",\"req\":\"^0.1.4\"},{\"default_features\":false,\"name\":\"borsh\",\"optional\":true,\"req\":\"^1.2.0\"},{\"kind\":\"dev\",\"name\":\"matches\",\"req\":\"^0.1\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"zeroize\",\"optional\":true,\"req\":\"^1.4\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", "ascii-canvas_3.0.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"diff\",\"req\":\"^0.1\"},{\"name\":\"term\",\"req\":\"^0.7\"}],\"features\":{}}", "ascii_1.1.0": "{\"dependencies\":[{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.25\"},{\"name\":\"serde_test\",\"optional\":true,\"req\":\"^1.0\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[\"alloc\"]}}", + "askama_0.15.4": "{\"dependencies\":[{\"default_features\":false,\"name\":\"askama_macros\",\"optional\":true,\"req\":\"=0.15.4\"},{\"kind\":\"dev\",\"name\":\"assert_matches\",\"req\":\"^1.5.0\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.8\"},{\"name\":\"itoa\",\"req\":\"^1.0.11\"},{\"default_features\":false,\"name\":\"percent-encoding\",\"optional\":true,\"req\":\"^2.1.0\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde_json\",\"optional\":true,\"req\":\"^1.0\"}],\"features\":{\"alloc\":[\"askama_macros?/alloc\",\"serde?/alloc\",\"serde_json?/alloc\",\"percent-encoding?/alloc\"],\"code-in-doc\":[\"askama_macros?/code-in-doc\"],\"config\":[\"askama_macros?/config\"],\"default\":[\"config\",\"derive\",\"std\",\"urlencode\"],\"derive\":[\"dep:askama_macros\",\"dep:askama_macros\"],\"full\":[\"default\",\"code-in-doc\",\"serde_json\"],\"nightly-spans\":[\"askama_macros/nightly-spans\"],\"serde_json\":[\"std\",\"askama_macros?/serde_json\",\"dep:serde\",\"dep:serde_json\"],\"std\":[\"alloc\",\"askama_macros?/std\",\"serde?/std\",\"serde_json?/std\",\"percent-encoding?/std\"],\"urlencode\":[\"askama_macros?/urlencode\",\"dep:percent-encoding\"]}}", + "askama_derive_0.15.4": "{\"dependencies\":[{\"name\":\"basic-toml\",\"optional\":true,\"req\":\"^0.1.1\"},{\"kind\":\"dev\",\"name\":\"console\",\"req\":\"^0.16.0\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.8\"},{\"name\":\"memchr\",\"req\":\"^2\"},{\"name\":\"parser\",\"package\":\"askama_parser\",\"req\":\"=0.15.4\"},{\"kind\":\"dev\",\"name\":\"prettyplease\",\"req\":\"^0.2.20\"},{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"pulldown-cmark\",\"optional\":true,\"req\":\"^0.13.0\"},{\"default_features\":false,\"name\":\"quote\",\"req\":\"^1\"},{\"name\":\"rustc-hash\",\"req\":\"^2.0.0\"},{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"serde_derive\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"similar\",\"req\":\"^2.6.0\"},{\"default_features\":false,\"features\":[\"clone-impls\",\"derive\",\"full\",\"parsing\",\"printing\"],\"name\":\"syn\",\"req\":\"^2.0.3\"}],\"features\":{\"alloc\":[],\"code-in-doc\":[\"dep:pulldown-cmark\"],\"config\":[\"external-sources\",\"dep:basic-toml\",\"dep:serde\",\"dep:serde_derive\",\"parser/config\"],\"default\":[\"alloc\",\"code-in-doc\",\"config\",\"external-sources\",\"proc-macro\",\"serde_json\",\"std\",\"urlencode\"],\"external-sources\":[],\"nightly-spans\":[],\"proc-macro\":[\"proc-macro2/proc-macro\"],\"serde_json\":[],\"std\":[\"alloc\"],\"urlencode\":[]}}", + "askama_macros_0.15.4": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"external-sources\",\"proc-macro\"],\"name\":\"askama_derive\",\"package\":\"askama_derive\",\"req\":\"=0.15.4\"}],\"features\":{\"alloc\":[\"askama_derive/alloc\"],\"code-in-doc\":[\"askama_derive/code-in-doc\"],\"config\":[\"askama_derive/config\"],\"default\":[\"config\",\"derive\",\"std\",\"urlencode\"],\"derive\":[],\"full\":[\"default\",\"code-in-doc\",\"serde_json\"],\"nightly-spans\":[\"askama_derive/nightly-spans\"],\"serde_json\":[\"askama_derive/serde_json\"],\"std\":[\"askama_derive/std\"],\"urlencode\":[\"askama_derive/urlencode\"]}}", + "askama_parser_0.15.4": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.8\"},{\"name\":\"rustc-hash\",\"req\":\"^2.0.0\"},{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"serde_derive\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"unicode-ident\",\"req\":\"^1.0.12\"},{\"features\":[\"simd\"],\"name\":\"winnow\",\"req\":\"^0.7.0\"}],\"features\":{\"config\":[\"dep:serde\",\"dep:serde_derive\"]}}", "asn1-rs-derive_0.6.0": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0\"},{\"name\":\"quote\",\"req\":\"^1.0\"},{\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0\"},{\"name\":\"synstructure\",\"req\":\"^0.13\"}],\"features\":{}}", "asn1-rs-impl_0.2.0": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1\"},{\"name\":\"quote\",\"req\":\"^1\"},{\"name\":\"syn\",\"req\":\"^2.0\"}],\"features\":{}}", "asn1-rs_0.7.1": "{\"dependencies\":[{\"name\":\"asn1-rs-derive\",\"req\":\"^0.6\"},{\"name\":\"asn1-rs-impl\",\"req\":\"^0.2\"},{\"name\":\"bitvec\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"colored\",\"optional\":true,\"req\":\"^3.0\"},{\"kind\":\"dev\",\"name\":\"colored\",\"req\":\"^3.0\"},{\"name\":\"cookie-factory\",\"optional\":true,\"req\":\"^0.3.0\"},{\"name\":\"displaydoc\",\"req\":\"^0.2.2\"},{\"kind\":\"dev\",\"name\":\"hex-literal\",\"req\":\"^0.4\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"nom\",\"req\":\"^7.0\"},{\"name\":\"num-bigint\",\"optional\":true,\"req\":\"^0.4\"},{\"name\":\"num-traits\",\"req\":\"^0.2.14\"},{\"kind\":\"dev\",\"name\":\"pem\",\"req\":\"^3.0\"},{\"name\":\"rusticata-macros\",\"req\":\"^4.0\"},{\"name\":\"thiserror\",\"req\":\"^2.0.0\"},{\"features\":[\"macros\",\"parsing\",\"formatting\"],\"name\":\"time\",\"optional\":true,\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0\"}],\"features\":{\"bigint\":[\"num-bigint\"],\"bits\":[\"bitvec\"],\"datetime\":[\"time\"],\"debug\":[\"std\",\"colored\"],\"default\":[\"std\"],\"serialize\":[\"cookie-factory\"],\"std\":[],\"trace\":[\"debug\"]}}", diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 21f17fc691..c888f9368d 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -480,6 +480,58 @@ dependencies = [ "term", ] +[[package]] +name = "askama" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08e1676b346cadfec169374f949d7490fd80a24193d37d2afce0c047cf695e57" +dependencies = [ + "askama_macros", + "itoa", + "percent-encoding", + "serde", + "serde_json", +] + +[[package]] +name = "askama_derive" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7661ff56517787343f376f75db037426facd7c8d3049cef8911f1e75016f3a37" +dependencies = [ + "askama_parser", + "basic-toml", + "memchr", + "proc-macro2", + "quote", + "rustc-hash 2.1.1", + "serde", + "serde_derive", + "syn 2.0.114", +] + +[[package]] +name = "askama_macros" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "713ee4dbfd1eb719c2dab859465b01fa1d21cb566684614a713a6b7a99a4e47b" +dependencies = [ + "askama_derive", +] + +[[package]] +name = "askama_parser" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d62d674238a526418b30c0def480d5beadb9d8964e7f38d635b03bf639c704c" +dependencies = [ + "rustc-hash 2.1.1", + "serde", + "serde_derive", + "unicode-ident", + "winnow", +] + [[package]] name = "asn1-rs" version = "0.7.1" @@ -1688,6 +1740,7 @@ version = "0.0.0" dependencies = [ "anyhow", "arc-swap", + "askama", "assert_cmd", "assert_matches", "async-channel", diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index 3bb436ef07..c72bb907f3 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -149,6 +149,7 @@ allocative = "0.3.3" ansi-to-tui = "7.0.0" anyhow = "1" arboard = { version = "3", features = ["wayland-data-control"] } +askama = "0.15.4" assert_cmd = "2" assert_matches = "1.5.0" async-channel = "2.3.1" diff --git a/codex-rs/core/BUILD.bazel b/codex-rs/core/BUILD.bazel index 5269148bc2..015fb87e37 100644 --- a/codex-rs/core/BUILD.bazel +++ b/codex-rs/core/BUILD.bazel @@ -14,6 +14,12 @@ codex_rust_crate( ) + [ "//codex-rs:node-version.txt", ], + rustc_env = { + # Askama resolves template paths relative to CARGO_MANIFEST_DIR. In + # Bazel, the Cargo-provided absolute source path points outside the + # sandbox, so keep the manifest root anchored inside the execroot. + "CARGO_MANIFEST_DIR": "codex-rs/core", + }, integration_compile_data_extra = [ "//codex-rs/apply-patch:apply_patch_tool_instructions.md", "models.json", diff --git a/codex-rs/core/Cargo.toml b/codex-rs/core/Cargo.toml index c79e905883..47731291a4 100644 --- a/codex-rs/core/Cargo.toml +++ b/codex-rs/core/Cargo.toml @@ -21,6 +21,7 @@ anyhow = { workspace = true } arc-swap = "1.8.2" async-channel = { workspace = true } async-trait = { workspace = true } +askama = { workspace = true } base64 = { workspace = true } bm25 = { workspace = true } chardetng = { workspace = true } diff --git a/codex-rs/core/src/memories/prompts.rs b/codex-rs/core/src/memories/prompts.rs index 9a3ea6a92d..9b341c1ffe 100644 --- a/codex-rs/core/src/memories/prompts.rs +++ b/codex-rs/core/src/memories/prompts.rs @@ -2,18 +2,43 @@ use crate::memories::memory_root; use crate::memories::phase_one; use crate::truncate::TruncationPolicy; use crate::truncate::truncate_text; +use askama::Template; use codex_protocol::openai_models::ModelInfo; use std::path::Path; use tokio::fs; +use tracing::warn; -const CONSOLIDATION_TEMPLATE: &str = include_str!("../../templates/memories/consolidation.md"); -const STAGE_ONE_INPUT_TEMPLATE: &str = include_str!("../../templates/memories/stage_one_input.md"); -const READ_PATH_TEMPLATE: &str = include_str!("../../templates/memories/read_path.md"); +#[derive(Template)] +#[template(path = "memories/consolidation.md", escape = "none")] +struct ConsolidationPromptTemplate<'a> { + memory_root: &'a str, +} + +#[derive(Template)] +#[template(path = "memories/stage_one_input.md", escape = "none")] +struct StageOneInputTemplate<'a> { + rollout_path: &'a str, + rollout_cwd: &'a str, + rollout_contents: &'a str, +} + +#[derive(Template)] +#[template(path = "memories/read_path.md", escape = "none")] +struct MemoryToolDeveloperInstructionsTemplate<'a> { + base_path: &'a str, + memory_summary: &'a str, +} /// Builds the consolidation subagent prompt for a specific memory root. pub(super) fn build_consolidation_prompt(memory_root: &Path) -> String { let memory_root = memory_root.display().to_string(); - CONSOLIDATION_TEMPLATE.replace("{{ memory_root }}", &memory_root) + let template = ConsolidationPromptTemplate { + memory_root: &memory_root, + }; + template.render().unwrap_or_else(|err| { + warn!("failed to render memories consolidation prompt template: {err}"); + format!("## Memory Phase 2 (Consolidation)\nConsolidate Codex memories in: {memory_root}") + }) } /// Builds the stage-1 user message containing rollout metadata and content. @@ -40,10 +65,12 @@ pub(super) fn build_stage_one_input_message( let rollout_path = rollout_path.display().to_string(); let rollout_cwd = rollout_cwd.display().to_string(); - Ok(STAGE_ONE_INPUT_TEMPLATE - .replace("{{ rollout_path }}", &rollout_path) - .replace("{{ rollout_cwd }}", &rollout_cwd) - .replace("{{ rollout_contents }}", &truncated_rollout_contents)) + Ok(StageOneInputTemplate { + rollout_path: &rollout_path, + rollout_cwd: &rollout_cwd, + rollout_contents: &truncated_rollout_contents, + } + .render()?) } /// Build prompt used for read path. This prompt must be added to the developer instructions. In @@ -65,11 +92,11 @@ pub(crate) async fn build_memory_tool_developer_instructions(codex_home: &Path) return None; } let base_path = base_path.display().to_string(); - Some( - READ_PATH_TEMPLATE - .replace("{{ base_path }}", &base_path) - .replace("{{ memory_summary }}", &memory_summary), - ) + let template = MemoryToolDeveloperInstructionsTemplate { + base_path: &base_path, + memory_summary: &memory_summary, + }; + template.render().ok() } #[cfg(test)] From 520ae998012ff28548e32424b14df556bbec0371 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Wed, 25 Feb 2026 10:38:17 -0800 Subject: [PATCH 4/4] fix: zsh-shell-fork was not honoring sandbox --- .../tools/runtimes/shell/unix_escalation.rs | 45 +++++------ codex-rs/core/tests/suite/skill_approval.rs | 81 +++++++++++++++++++ 2 files changed, 102 insertions(+), 24 deletions(-) diff --git a/codex-rs/core/src/tools/runtimes/shell/unix_escalation.rs b/codex-rs/core/src/tools/runtimes/shell/unix_escalation.rs index 7485825ab1..68e5367aa8 100644 --- a/codex-rs/core/src/tools/runtimes/shell/unix_escalation.rs +++ b/codex-rs/core/src/tools/runtimes/shell/unix_escalation.rs @@ -72,14 +72,9 @@ pub(super) async fn try_run_zsh_fork( let sandbox_exec_request = attempt .env_for(spec, req.network.as_ref()) .map_err(|err| ToolError::Codex(err.into()))?; - // Keep env/network/sandbox metadata from `attempt.env_for()`, but build the - // script from the original shell argv. `attempt.env_for()` may wrap the - // command with `sandbox-exec` on macOS, and passing those wrapper flags - // (`-p`, `-D...`) through zsh breaks the zsh-fork path before subcommand - // approval runs. let crate::sandboxing::ExecRequest { - command: _sandbox_command, - cwd: _sandbox_cwd, + command, + cwd: sandbox_cwd, env: sandbox_env, network: sandbox_network, expiration: _sandbox_expiration, @@ -90,7 +85,7 @@ pub(super) async fn try_run_zsh_fork( justification, arg0, } = sandbox_exec_request; - let ParsedShellCommand { script, login } = extract_shell_script(command)?; + let ParsedShellCommand { script, login } = extract_shell_script(&command)?; let effective_timeout = Duration::from_millis( req.timeout_ms .unwrap_or(crate::exec::DEFAULT_EXEC_COMMAND_TIMEOUT_MS), @@ -99,6 +94,8 @@ pub(super) async fn try_run_zsh_fork( ctx.session.services.exec_policy.current().as_ref().clone(), )); let command_executor = CoreShellCommandExecutor { + command, + cwd: sandbox_cwd, sandbox_policy, sandbox, env: sandbox_env, @@ -438,6 +435,8 @@ impl EscalationPolicy for CoreShellActionProvider { } struct CoreShellCommandExecutor { + command: Vec, + cwd: PathBuf, sandbox_policy: SandboxPolicy, sandbox: SandboxType, env: HashMap, @@ -452,8 +451,8 @@ struct CoreShellCommandExecutor { impl ShellCommandExecutor for CoreShellCommandExecutor { async fn run( &self, - command: Vec, - cwd: PathBuf, + _command: Vec, + _cwd: PathBuf, env: HashMap, cancel_rx: CancellationToken, ) -> anyhow::Result { @@ -466,8 +465,8 @@ impl ShellCommandExecutor for CoreShellCommandExecutor { let result = crate::sandboxing::execute_env( crate::sandboxing::ExecRequest { - command, - cwd, + command: self.command.clone(), + cwd: self.cwd.clone(), env: exec_env, network: self.network.clone(), expiration: ExecExpiration::Cancellation(cancel_rx), @@ -500,19 +499,17 @@ struct ParsedShellCommand { } fn extract_shell_script(command: &[String]) -> Result { - match command { - [_, flag, script, ..] if flag == "-c" => Ok(ParsedShellCommand { - script: script.clone(), - login: false, - }), - [_, flag, script, ..] if flag == "-lc" => Ok(ParsedShellCommand { - script: script.clone(), - login: true, - }), - _ => Err(ToolError::Rejected( - "unexpected shell command format for zsh-fork execution".to_string(), - )), + if let Some((script, login)) = command.windows(3).find_map(|parts| match parts { + [_, flag, script] if flag == "-c" => Some((script.to_owned(), false)), + [_, flag, script] if flag == "-lc" => Some((script.to_owned(), true)), + _ => None, + }) { + return Ok(ParsedShellCommand { script, login }); } + + Err(ToolError::Rejected( + "unexpected shell command format for zsh-fork execution".to_string(), + )) } fn map_exec_result( diff --git a/codex-rs/core/tests/suite/skill_approval.rs b/codex-rs/core/tests/suite/skill_approval.rs index 147ecdd0c2..d1df0ae896 100644 --- a/codex-rs/core/tests/suite/skill_approval.rs +++ b/codex-rs/core/tests/suite/skill_approval.rs @@ -558,3 +558,84 @@ permissions: Ok(()) } + +#[cfg(unix)] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn shell_zsh_fork_still_enforces_workspace_write_sandbox() -> Result<()> { + use codex_config::Constrained; + use codex_protocol::protocol::AskForApproval; + + skip_if_no_network!(Ok(())); + + let Some(zsh_path) = find_test_zsh_path()? else { + return Ok(()); + }; + if !supports_exec_wrapper_intercept(&zsh_path) { + eprintln!( + "skipping zsh-fork sandbox test: zsh does not support EXEC_WRAPPER intercepts ({})", + zsh_path.display() + ); + return Ok(()); + } + let Ok(main_execve_wrapper_exe) = codex_utils_cargo_bin::cargo_bin("codex-execve-wrapper") + else { + eprintln!( + "skipping zsh-fork sandbox test: unable to resolve `codex-execve-wrapper` binary" + ); + return Ok(()); + }; + + let server = start_mock_server().await; + let tool_call_id = "zsh-fork-workspace-write-deny"; + let outside_path = "/var/tmp/codex-zsh-fork-workspace-write-deny.txt"; + let _ = fs::remove_file(outside_path); + let mut builder = test_codex() + .with_pre_build_hook(move |_| { + let _ = fs::remove_file(outside_path); + }) + .with_config(move |config| { + config.features.enable(Feature::ShellTool); + config.features.enable(Feature::ShellZshFork); + config.zsh_path = Some(zsh_path.clone()); + config.main_execve_wrapper_exe = Some(main_execve_wrapper_exe); + config.permissions.allow_login_shell = false; + config.permissions.approval_policy = Constrained::allow_any(AskForApproval::Never); + config.permissions.sandbox_policy = + Constrained::allow_any(SandboxPolicy::new_workspace_write_policy()); + }); + let test = builder.build(&server).await?; + + let command = format!("touch {outside_path}"); + let arguments = shell_command_arguments(&command)?; + let mocks = + mount_function_call_agent_response(&server, tool_call_id, &arguments, "shell_command") + .await; + + submit_turn_with_policies( + &test, + "write outside workspace with zsh fork", + AskForApproval::Never, + SandboxPolicy::new_workspace_write_policy(), + ) + .await?; + + wait_for_turn_complete_without_skill_approval(&test).await; + + let call_output = mocks + .completion + .single_request() + .function_call_output(tool_call_id); + let output = call_output["output"].as_str().unwrap_or_default(); + assert!( + output.contains("Permission denied") + || output.contains("Operation not permitted") + || output.contains("Read-only file system"), + "expected sandbox denial, got output: {output:?}" + ); + assert!( + !Path::new(outside_path).exists(), + "command should not write outside workspace under WorkspaceWrite policy" + ); + + Ok(()) +}