From db9cb04fb6c0eabdc8c8da7cd335980942139698 Mon Sep 17 00:00:00 2001 From: canvrno-oai Date: Tue, 26 May 2026 10:29:15 -0700 Subject: [PATCH 01/16] Move slash input logic out of chat composer (#23964) Recent composer cleanups split state ownership out of `ChatComposer`, but slash-command handling still mixed parsing, popup coordination, completion, submission validation, queue behavior, and argument element rebasing into the main composer file. Pending changes to slash command parsing and selection inspired this code move to prevent `chat_composer.rs` bloat. This is just a refactor, no functional or behavioral changes are intended. ## What changed - Move slash-command parsing and lookup helpers into `bottom_pane/chat_composer/slash_input.rs`. - Move slash popup key handling, command-name completion, and popup construction into the slash input helper module. - Centralize bare-command, inline-args, submission-validation, and queued-input action selection behind slash-specific helpers. - Move command argument text-element rebasing into the slash input module so inline command submission keeps the same element behavior with less composer-local logic. ## Verification - `just fmt` - `just test -p codex-tui` - `cargo insta pending-snapshots -p codex-tui` --- codex-rs/tui/src/bottom_pane/chat_composer.rs | 475 ++---------------- .../bottom_pane/chat_composer/slash_input.rs | 452 +++++++++++++++++ 2 files changed, 504 insertions(+), 423 deletions(-) create mode 100644 codex-rs/tui/src/bottom_pane/chat_composer/slash_input.rs diff --git a/codex-rs/tui/src/bottom_pane/chat_composer.rs b/codex-rs/tui/src/bottom_pane/chat_composer.rs index a3413dd3be..8fcda378fc 100644 --- a/codex-rs/tui/src/bottom_pane/chat_composer.rs +++ b/codex-rs/tui/src/bottom_pane/chat_composer.rs @@ -160,8 +160,6 @@ use super::chat_composer_history::ChatComposerHistory; use super::chat_composer_history::HistoryEntry; use super::chat_composer_history::HistoryEntryResponse; use super::command_popup::CommandItem; -use super::command_popup::CommandPopup; -use super::command_popup::CommandPopupFlags; use super::file_search_popup::FileSearchPopup; use super::footer::CollaborationModeIndicator; use super::footer::FooterKeyHints; @@ -197,10 +195,7 @@ use super::skill_popup::SkillPopup; use super::slash_commands::BuiltinCommandFlags; use super::slash_commands::ServiceTierCommand; use super::slash_commands::SlashCommandItem; -use super::slash_commands::find_slash_command; -use super::slash_commands::has_slash_command_prefix; use crate::bottom_pane::paste_burst::FlushResult; -use crate::bottom_pane::prompt_args::parse_slash_name; use crate::key_hint::KeyBindingListExt; use crate::keymap::EditorKeymap; use crate::keymap::RuntimeKeymap; @@ -222,6 +217,7 @@ mod draft_state; mod footer_state; mod history_search; mod popup_state; +mod slash_input; use self::attachment_state::AttachmentState; use self::draft_state::ComposerMentionBinding; @@ -230,6 +226,9 @@ use self::footer_state::FooterState; use self::history_search::HistorySearchSession; use self::popup_state::ActivePopup; use self::popup_state::PopupState; +use self::slash_input::SlashInput; +use self::slash_input::SlashValidation; +use self::slash_input::SubmissionValidation; use crate::app_event::AppEvent; use crate::app_event::ConnectorsSnapshot; use crate::app_event_sender::AppEventSender; @@ -411,12 +410,6 @@ pub(crate) struct ComposerDraftSnapshot { pub(crate) pending_pastes: Vec<(String, String)>, } -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum SlashValidation { - Immediate, - Deferred, -} - const FOOTER_SPACING_HEIGHT: u16 = 0; /// Builds the one-line nudge that replaces the ambient footer without adding layout height. @@ -433,6 +426,15 @@ fn plan_mode_nudge_line() -> Line<'static> { } impl ChatComposer { + fn slash_input(&self) -> SlashInput<'_> { + SlashInput::new( + self.slash_commands_enabled(), + self.draft.is_bash_mode, + self.builtin_command_flags(), + &self.service_tier_commands, + ) + } + fn builtin_command_flags(&self) -> BuiltinCommandFlags { BuiltinCommandFlags { collaboration_modes_enabled: self.collaboration_modes_enabled, @@ -1653,145 +1655,6 @@ impl ChatComposer { self.history_search.is_some() || self.popups.active() } - /// Handle key event when the slash-command popup is visible. - fn handle_key_event_with_slash_popup(&mut self, key_event: KeyEvent) -> (InputResult, bool) { - if self.handle_shortcut_overlay_key(&key_event) { - return (InputResult::None, true); - } - if key_event.code == KeyCode::Esc { - let next_mode = esc_hint_mode(self.footer.mode, self.is_task_running); - if next_mode != self.footer.mode { - self.footer.mode = next_mode; - return (InputResult::None, true); - } - } else { - self.footer.mode = reset_mode_after_activity(self.footer.mode); - } - let ActivePopup::Command(popup) = &mut self.popups.active else { - unreachable!(); - }; - - match key_event { - KeyEvent { - code: KeyCode::Up, .. - } - | KeyEvent { - code: KeyCode::Char('p'), - modifiers: KeyModifiers::CONTROL, - .. - } => { - popup.move_up(); - (InputResult::None, true) - } - KeyEvent { - code: KeyCode::Down, - .. - } - | KeyEvent { - code: KeyCode::Char('n'), - modifiers: KeyModifiers::CONTROL, - .. - } => { - popup.move_down(); - (InputResult::None, true) - } - KeyEvent { - code: KeyCode::Esc, .. - } => { - // Dismiss the slash popup; keep the current input untouched. - self.popups.active = ActivePopup::None; - (InputResult::None, true) - } - KeyEvent { - code: KeyCode::Tab, .. - } => { - // Ensure popup filtering/selection reflects the latest composer text - // before applying completion. - let first_line = self.draft.textarea.text().lines().next().unwrap_or(""); - popup.on_composer_text_change(first_line.to_string()); - if let Some(selected_cmd) = popup.selected_item() { - let selected_command_text = format!("/{}", selected_cmd.command()); - if let CommandItem::Builtin(cmd) = selected_cmd - && cmd == SlashCommand::Skills - { - self.stage_selected_slash_command_history(&CommandItem::Builtin(cmd)); - self.draft.textarea.set_text_clearing_elements(""); - self.draft.is_bash_mode = false; - return (InputResult::Command(cmd), true); - } - - let starts_with_cmd = - first_line.trim_start().starts_with(&selected_command_text); - if !starts_with_cmd { - self.draft - .textarea - .set_text_clearing_elements(&format!("{selected_command_text} ")); - if !self.draft.textarea.text().is_empty() { - self.draft - .textarea - .set_cursor(self.draft.textarea.text().len()); - } - return (InputResult::None, true); - } - } - if self.is_task_running { - return self.handle_submission(/*should_queue*/ true); - } - (InputResult::None, true) - } - KeyEvent { - code: KeyCode::Char('/'), - modifiers: KeyModifiers::NONE, - .. - } => { - // Treat "/" as accepting the highlighted command as text completion - // while the slash-command popup is active. - let first_line = self.draft.textarea.text().lines().next().unwrap_or(""); - popup.on_composer_text_change(first_line.to_string()); - if let Some(selected_cmd) = popup.selected_item() { - let selected_command_text = format!("/{}", selected_cmd.command()); - let starts_with_cmd = - first_line.trim_start().starts_with(&selected_command_text); - if !starts_with_cmd { - self.draft - .textarea - .set_text_clearing_elements(&format!("{selected_command_text} ")); - self.draft.is_bash_mode = false; - } - if !self.draft.textarea.text().is_empty() { - self.draft - .textarea - .set_cursor(self.draft.textarea.text().len()); - } - } - (InputResult::None, true) - } - KeyEvent { - code: KeyCode::Enter, - modifiers: KeyModifiers::NONE, - .. - } => { - if let Some(sel) = popup.selected_item() { - self.stage_selected_slash_command_history(&sel); - self.draft.textarea.set_text_clearing_elements(""); - self.draft.is_bash_mode = false; - return ( - match sel { - CommandItem::Builtin(cmd) => InputResult::Command(cmd), - CommandItem::ServiceTier(command) => { - InputResult::ServiceTierCommand(command) - } - }, - true, - ); - } - // Fallback to default newline handling if no command selected. - self.handle_key_event_without_popup(key_event) - } - input => self.handle_input_basic(input), - } - } - #[inline] fn clamp_to_char_boundary(text: &str, pos: usize) -> usize { let mut p = pos.min(text.len()); @@ -2721,37 +2584,27 @@ impl ChatComposer { text_elements = Self::trim_text_elements(&expanded_input, &text, text_elements); if slash_validation == SlashValidation::Immediate - && self.slash_commands_enabled() - && let Some((name, _rest, _rest_offset)) = parse_slash_name(&text) + && let SubmissionValidation::UnknownCommand(name) = self + .slash_input() + .validate_submission(&text, input_starts_with_space) { - let treat_as_plain_text = input_starts_with_space || name.contains('/'); - if !treat_as_plain_text { - let is_known = find_slash_command( - name, - self.builtin_command_flags(), - &self.service_tier_commands, - ) - .is_some(); - if !is_known { - let message = format!( - r#"Unrecognized command '/{name}'. Type "/" for a list of supported commands."# - ); - self.app_event_tx.send(AppEvent::InsertHistoryCell(Box::new( - history_cell::new_info_event(message, /*hint*/ None), - ))); - self.set_text_content_with_mention_bindings( - original_input.clone(), - original_text_elements, - original_local_image_paths, - original_mention_bindings, - ); - self.draft - .pending_pastes - .clone_from(&original_pending_pastes); - self.draft.textarea.set_cursor(original_input.len()); - return None; - } - } + let message = format!( + r#"Unrecognized command '/{name}'. Type "/" for a list of supported commands."# + ); + self.app_event_tx.send(AppEvent::InsertHistoryCell(Box::new( + history_cell::new_info_event(message, /*hint*/ None), + ))); + self.set_text_content_with_mention_bindings( + original_input.clone(), + original_text_elements, + original_local_image_paths, + original_mention_bindings, + ); + self.draft + .pending_pastes + .clone_from(&original_pending_pastes); + self.draft.textarea.set_cursor(original_input.len()); + return None; } let actual_chars = text.chars().count(); @@ -2823,8 +2676,7 @@ impl ChatComposer { self.handle_paste(pasted); } let raw_text = self.draft.textarea.text(); - let defer_slash_validation = - self.should_parse_as_slash_on_dequeue_from_raw_text(raw_text); + let defer_slash_validation = self.slash_input().should_parse_on_dequeue(raw_text); if let Some((text, text_elements)) = self.prepare_submission_text_with_options( /*record_history*/ true, if defer_slash_validation { @@ -2833,7 +2685,7 @@ impl ChatComposer { SlashValidation::Immediate }, ) { - let action = self.queued_input_action(&text, defer_slash_validation); + let action = slash_input::queued_input_action(&text, defer_slash_validation); return ( InputResult::Queued { text, @@ -2940,25 +2792,9 @@ impl ChatComposer { /// Check if the first line is a bare slash command (no args) and dispatch it. /// Returns Some(InputResult) if a command was dispatched, None otherwise. fn try_dispatch_bare_slash_command(&mut self) -> Option { - if !self.slash_commands_enabled() || self.draft.is_bash_mode { - return None; - } - let text = self.draft.textarea.text(); - let first_line = text.lines().next().unwrap_or(""); - let (name, rest, _rest_offset) = parse_slash_name(first_line)?; - if !rest.is_empty() { - return None; - } - let command = find_slash_command( - name, - self.builtin_command_flags(), - &self.service_tier_commands, - )?; - if command.supports_inline_args() - && parse_slash_name(text).is_some_and(|(_, full_rest, _)| !full_rest.is_empty()) - { - return None; - } + let command = self + .slash_input() + .bare_command(self.draft.textarea.text())?; if self.reject_slash_command_if_unavailable(&command) { self.stage_slash_command_history(&command); self.record_pending_slash_command_history(); @@ -2976,28 +2812,9 @@ impl ChatComposer { /// Check if the input is a slash command with args (e.g., /review args) and dispatch it. /// Returns Some(InputResult) if a command was dispatched, None otherwise. fn try_dispatch_slash_command_with_args(&mut self) -> Option { - if !self.slash_commands_enabled() || self.draft.is_bash_mode { - return None; - } let text = self.draft.textarea.text().to_string(); - if text.starts_with(' ') { - return None; - } - - let (name, rest, rest_offset) = parse_slash_name(&text)?; - if rest.is_empty() || name.contains('/') { - return None; - } - - let command = find_slash_command( - name, - self.builtin_command_flags(), - &self.service_tier_commands, - )?; - - if !command.supports_inline_args() { - return None; - } + let inline_command = self.slash_input().inline_command(&text)?; + let command = inline_command.command; if self.reject_slash_command_if_unavailable(&command) { self.stage_slash_command_history(&command); self.record_pending_slash_command_history(); @@ -3006,13 +2823,13 @@ impl ChatComposer { self.stage_slash_command_history(&command); - let mut args_elements = Self::slash_command_args_elements( - rest, - rest_offset, + let mut args_elements = slash_input::args_elements( + inline_command.rest, + inline_command.rest_offset, &self.draft.textarea.text_elements(), ); - let trimmed_rest = rest.trim(); - args_elements = Self::trim_text_elements(rest, trimmed_rest, args_elements); + let trimmed_rest = inline_command.rest.trim(); + args_elements = Self::trim_text_elements(inline_command.rest, trimmed_rest, args_elements); let SlashCommandItem::Builtin(cmd) = command else { return None; }; @@ -3038,12 +2855,9 @@ impl ChatComposer { record_history: bool, ) -> Option<(String, Vec)> { let (prepared_text, prepared_elements) = self.prepare_submission_text(record_history)?; - let (_, prepared_rest, prepared_rest_offset) = parse_slash_name(&prepared_text)?; - let mut args_elements = Self::slash_command_args_elements( - prepared_rest, - prepared_rest_offset, - &prepared_elements, - ); + let (prepared_rest, prepared_rest_offset) = slash_input::prepared_args(&prepared_text)?; + let mut args_elements = + slash_input::args_elements(prepared_rest, prepared_rest_offset, &prepared_elements); let trimmed_rest = prepared_rest.trim(); args_elements = Self::trim_text_elements(prepared_rest, trimmed_rest, args_elements); Some((trimmed_rest.to_string(), args_elements)) @@ -3063,24 +2877,6 @@ impl ChatComposer { true } - fn should_parse_as_slash_on_dequeue_from_raw_text(&self, text: &str) -> bool { - self.slash_commands_enabled() && !text.starts_with(' ') && text.trim().starts_with('/') - } - - fn queued_input_action( - &self, - prepared_text: &str, - defer_slash_validation: bool, - ) -> QueuedInputAction { - if defer_slash_validation && prepared_text.starts_with('/') { - QueuedInputAction::ParseSlash - } else if prepared_text.starts_with('!') { - QueuedInputAction::RunShell - } else { - QueuedInputAction::Plain - } - } - /// Stage the current slash-command text for later local recall. /// /// Staging snapshots the rich composer state before the textarea is cleared. `ChatWidget` @@ -3120,34 +2916,6 @@ impl ChatComposer { }); } - /// Translate full-text element ranges into command-argument ranges. - /// - /// `rest_offset` is the byte offset where `rest` begins in the full text. - fn slash_command_args_elements( - rest: &str, - rest_offset: usize, - text_elements: &[TextElement], - ) -> Vec { - if rest.is_empty() || text_elements.is_empty() { - return Vec::new(); - } - text_elements - .iter() - .filter_map(|elem| { - if elem.byte_range.end <= rest_offset { - return None; - } - let start = elem.byte_range.start.saturating_sub(rest_offset); - let mut end = elem.byte_range.end.saturating_sub(rest_offset); - if start >= rest.len() { - return None; - } - end = end.min(rest.len()); - (start < end).then_some(elem.map_range(|_| ByteRange { start, end })) - }) - .collect() - } - fn handle_remote_image_selection_key( &mut self, key_event: &KeyEvent, @@ -3715,123 +3483,6 @@ impl ChatComposer { } } - /// Keep slash command elements aligned with the current first line. - fn sync_slash_command_elements(&mut self) { - if !self.slash_commands_enabled() { - return; - } - let text = self.draft.textarea.text(); - let first_line_end = text.find('\n').unwrap_or(text.len()); - let first_line = &text[..first_line_end]; - let desired_range = self.slash_command_element_range(first_line); - // Slash commands are only valid at byte 0 of the first line. - // Any slash-shaped element not matching the current desired prefix is stale. - let mut has_desired = false; - let mut stale_ranges = Vec::new(); - for elem in self.draft.textarea.text_elements() { - let Some(payload) = elem.placeholder(text) else { - continue; - }; - if payload.strip_prefix('/').is_none() { - continue; - } - let range = elem.byte_range.start..elem.byte_range.end; - if desired_range.as_ref() == Some(&range) { - has_desired = true; - } else { - stale_ranges.push(range); - } - } - - for range in stale_ranges { - self.draft.textarea.remove_element_range(range); - } - - if let Some(range) = desired_range - && !has_desired - { - self.draft.textarea.add_element_range(range); - } - } - - fn slash_command_element_range(&self, first_line: &str) -> Option> { - if self.draft.is_bash_mode { - return None; - } - let (name, _rest, _rest_offset) = parse_slash_name(first_line)?; - if name.contains('/') { - return None; - } - let element_end = 1 + name.len(); - let has_space_after = first_line - .get(element_end..) - .and_then(|tail| tail.chars().next()) - .is_some_and(char::is_whitespace); - if !has_space_after { - return None; - } - if self.is_known_slash_name(name) { - Some(0..element_end) - } else { - None - } - } - - fn is_known_slash_name(&self, name: &str) -> bool { - find_slash_command( - name, - self.builtin_command_flags(), - &self.service_tier_commands, - ) - .is_some() - } - - /// If the cursor is currently within a slash command on the first line, - /// extract the command name and the rest of the line after it. - /// Returns None if the cursor is outside a slash command. - fn slash_command_under_cursor(first_line: &str, cursor: usize) -> Option<(&str, &str)> { - if !first_line.starts_with('/') { - return None; - } - - let name_start = 1usize; - let name_end = first_line[name_start..] - .find(char::is_whitespace) - .map(|idx| name_start + idx) - .unwrap_or_else(|| first_line.len()); - - if cursor > name_end { - return None; - } - - let name = &first_line[name_start..name_end]; - let rest_start = first_line[name_end..] - .find(|c: char| !c.is_whitespace()) - .map(|idx| name_end + idx) - .unwrap_or(name_end); - let rest = &first_line[rest_start..]; - - Some((name, rest)) - } - - /// Heuristic for whether the typed slash command looks like a valid - /// prefix for any known built-in command. - /// Empty names only count when there is no extra content after the '/'. - fn looks_like_slash_prefix(&self, name: &str, rest_after_name: &str) -> bool { - if !self.slash_commands_enabled() { - return false; - } - if name.is_empty() { - return rest_after_name.is_empty(); - } - - has_slash_command_prefix( - name, - self.builtin_command_flags(), - &self.service_tier_commands, - ) - } - /// Synchronize `self.command_popup` with the current text in the /// textarea. This must be called after every modification that can change /// the text so the popup is shown/updated/hidden as appropriate. @@ -3850,8 +3501,9 @@ impl ChatComposer { let caret_on_first_line = cursor <= first_line_end; let is_editing_slash_command_name = caret_on_first_line - && Self::slash_command_under_cursor(first_line, cursor) - .is_some_and(|(name, rest)| self.looks_like_slash_prefix(name, rest)); + && self + .slash_input() + .is_editing_command_name(first_line, cursor); // If the cursor is currently positioned within an `@token`, prefer the // file-search popup over the slash popup so users can insert a file path @@ -3872,30 +3524,7 @@ impl ChatComposer { } _ => { if is_editing_slash_command_name { - let collaboration_modes_enabled = self.collaboration_modes_enabled; - let connectors_enabled = self.connectors_enabled; - let plugins_command_enabled = self.plugins_command_enabled; - let service_tier_commands_enabled = self.service_tier_commands_enabled; - let goal_command_enabled = self.goal_command_enabled; - let personality_command_enabled = self.personality_command_enabled; - let realtime_conversation_enabled = self.realtime_conversation_enabled; - let audio_device_selection_enabled = self.audio_device_selection_enabled; - let mut command_popup = CommandPopup::new( - CommandPopupFlags { - collaboration_modes_enabled, - connectors_enabled, - plugins_command_enabled, - service_tier_commands_enabled, - goal_command_enabled, - personality_command_enabled, - realtime_conversation_enabled, - audio_device_selection_enabled, - windows_degraded_sandbox_active: self.windows_degraded_sandbox_active, - side_conversation_active: self.side_conversation_active, - }, - self.service_tier_commands.clone(), - ); - command_popup.on_composer_text_change(first_line.to_string()); + let command_popup = self.slash_input().command_popup(first_line); self.popups.active = ActivePopup::Command(command_popup); } } diff --git a/codex-rs/tui/src/bottom_pane/chat_composer/slash_input.rs b/codex-rs/tui/src/bottom_pane/chat_composer/slash_input.rs new file mode 100644 index 0000000000..a7205d237a --- /dev/null +++ b/codex-rs/tui/src/bottom_pane/chat_composer/slash_input.rs @@ -0,0 +1,452 @@ +//! Slash-command input parsing, cursor detection, and completion helpers. + +use std::ops::Range; + +use crossterm::event::KeyCode; +use crossterm::event::KeyEvent; +use crossterm::event::KeyModifiers; + +use crate::bottom_pane::command_popup::CommandItem; +use crate::bottom_pane::command_popup::CommandPopup; +use crate::bottom_pane::command_popup::CommandPopupFlags; +use crate::bottom_pane::prompt_args::parse_slash_name; +use crate::bottom_pane::slash_commands::BuiltinCommandFlags; +use crate::bottom_pane::slash_commands::ServiceTierCommand; +use crate::bottom_pane::slash_commands::SlashCommandItem; +use crate::bottom_pane::slash_commands::find_slash_command; +use crate::bottom_pane::slash_commands::has_slash_command_prefix; +use crate::slash_command::SlashCommand; +use codex_protocol::user_input::ByteRange; +use codex_protocol::user_input::TextElement; + +use super::super::footer::esc_hint_mode; +use super::super::footer::reset_mode_after_activity; +use super::ActivePopup; +use super::ChatComposer; +use super::InputResult; +use super::QueuedInputAction; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum SlashValidation { + Immediate, + Deferred, +} + +pub(super) enum SubmissionValidation { + Valid, + UnknownCommand(String), +} + +pub(super) struct InlineCommand<'a> { + pub(super) command: SlashCommandItem, + pub(super) rest: &'a str, + pub(super) rest_offset: usize, +} + +pub(super) struct SlashInput<'a> { + enabled: bool, + is_bash_mode: bool, + command_flags: BuiltinCommandFlags, + service_tier_commands: &'a [ServiceTierCommand], +} + +impl<'a> SlashInput<'a> { + pub(super) fn new( + enabled: bool, + is_bash_mode: bool, + command_flags: BuiltinCommandFlags, + service_tier_commands: &'a [ServiceTierCommand], + ) -> Self { + Self { + enabled, + is_bash_mode, + command_flags, + service_tier_commands, + } + } + + pub(super) fn validate_submission( + &self, + text: &str, + input_starts_with_space: bool, + ) -> SubmissionValidation { + if !self.enabled { + return SubmissionValidation::Valid; + } + let Some((name, _rest, _rest_offset)) = parse_slash_name(text) else { + return SubmissionValidation::Valid; + }; + if input_starts_with_space || name.contains('/') { + return SubmissionValidation::Valid; + } + if self.command(name).is_some() { + SubmissionValidation::Valid + } else { + SubmissionValidation::UnknownCommand(name.to_string()) + } + } + + pub(super) fn bare_command(&self, text: &str) -> Option { + if !self.enabled || self.is_bash_mode { + return None; + } + let first_line = text.lines().next().unwrap_or(""); + let (name, rest, _rest_offset) = parse_slash_name(first_line)?; + if !rest.is_empty() { + return None; + } + let command = self.command(name)?; + if command.supports_inline_args() + && parse_slash_name(text).is_some_and(|(_, full_rest, _)| !full_rest.is_empty()) + { + return None; + } + Some(command) + } + + pub(super) fn inline_command<'text>(&self, text: &'text str) -> Option> { + if !self.enabled || self.is_bash_mode || text.starts_with(' ') { + return None; + } + + let (name, rest, rest_offset) = parse_slash_name(text)?; + if rest.is_empty() || name.contains('/') { + return None; + } + + let command = self.command(name)?; + command.supports_inline_args().then_some(InlineCommand { + command, + rest, + rest_offset, + }) + } + + pub(super) fn should_parse_on_dequeue(&self, text: &str) -> bool { + self.enabled && !text.starts_with(' ') && text.trim().starts_with('/') + } + + pub(super) fn command_element_range(&self, first_line: &str) -> Option> { + if self.is_bash_mode { + return None; + } + let (name, _rest, _rest_offset) = parse_slash_name(first_line)?; + if name.contains('/') { + return None; + } + let element_end = 1 + name.len(); + let has_space_after = first_line + .get(element_end..) + .and_then(|tail| tail.chars().next()) + .is_some_and(char::is_whitespace); + if !has_space_after { + return None; + } + self.command(name).is_some().then_some(0..element_end) + } + + pub(super) fn is_editing_command_name(&self, first_line: &str, cursor: usize) -> bool { + let Some((name, rest)) = command_under_cursor(first_line, cursor) else { + return false; + }; + if !self.enabled { + return false; + } + if name.is_empty() { + return rest.is_empty(); + } + + has_slash_command_prefix(name, self.command_flags, self.service_tier_commands) + } + + pub(super) fn command_popup(&self, first_line: &str) -> CommandPopup { + let mut command_popup = CommandPopup::new( + CommandPopupFlags { + collaboration_modes_enabled: self.command_flags.collaboration_modes_enabled, + connectors_enabled: self.command_flags.connectors_enabled, + plugins_command_enabled: self.command_flags.plugins_command_enabled, + service_tier_commands_enabled: self.command_flags.service_tier_commands_enabled, + goal_command_enabled: self.command_flags.goal_command_enabled, + personality_command_enabled: self.command_flags.personality_command_enabled, + realtime_conversation_enabled: self.command_flags.realtime_conversation_enabled, + audio_device_selection_enabled: self.command_flags.audio_device_selection_enabled, + windows_degraded_sandbox_active: self.command_flags.allow_elevate_sandbox, + side_conversation_active: self.command_flags.side_conversation_active, + }, + self.service_tier_commands.to_vec(), + ); + command_popup.on_composer_text_change(first_line.to_string()); + command_popup + } + + fn command(&self, name: &str) -> Option { + find_slash_command(name, self.command_flags, self.service_tier_commands) + } +} + +pub(super) fn queued_input_action( + prepared_text: &str, + defer_slash_validation: bool, +) -> QueuedInputAction { + if defer_slash_validation && prepared_text.starts_with('/') { + QueuedInputAction::ParseSlash + } else if prepared_text.starts_with('!') { + QueuedInputAction::RunShell + } else { + QueuedInputAction::Plain + } +} + +impl ChatComposer { + /// Handle key event when the slash-command popup is visible. + pub(super) fn handle_key_event_with_slash_popup( + &mut self, + key_event: KeyEvent, + ) -> (InputResult, bool) { + if self.handle_shortcut_overlay_key(&key_event) { + return (InputResult::None, true); + } + if key_event.code == KeyCode::Esc { + let next_mode = esc_hint_mode(self.footer.mode, self.is_task_running); + if next_mode != self.footer.mode { + self.footer.mode = next_mode; + return (InputResult::None, true); + } + } else { + self.footer.mode = reset_mode_after_activity(self.footer.mode); + } + let ActivePopup::Command(popup) = &mut self.popups.active else { + unreachable!(); + }; + + match key_event { + KeyEvent { + code: KeyCode::Up, .. + } + | KeyEvent { + code: KeyCode::Char('p'), + modifiers: KeyModifiers::CONTROL, + .. + } => { + popup.move_up(); + (InputResult::None, true) + } + KeyEvent { + code: KeyCode::Down, + .. + } + | KeyEvent { + code: KeyCode::Char('n'), + modifiers: KeyModifiers::CONTROL, + .. + } => { + popup.move_down(); + (InputResult::None, true) + } + KeyEvent { + code: KeyCode::Esc, .. + } => { + // Dismiss the slash popup; keep the current input untouched. + self.popups.active = ActivePopup::None; + (InputResult::None, true) + } + KeyEvent { + code: KeyCode::Tab, .. + } => { + // Ensure popup filtering/selection reflects the latest composer text + // before applying completion. + let first_line = self.draft.textarea.text().lines().next().unwrap_or(""); + popup.on_composer_text_change(first_line.to_string()); + if let Some(selected_cmd) = popup.selected_item() { + if selected_command_dispatches_immediately_on_tab(&selected_cmd) + && let CommandItem::Builtin(cmd) = &selected_cmd + { + self.stage_selected_slash_command_history(&selected_cmd); + self.draft.textarea.set_text_clearing_elements(""); + self.draft.is_bash_mode = false; + return (InputResult::Command(*cmd), true); + } + + if let Some(completed_text) = + selected_command_completion(first_line, &selected_cmd) + { + self.draft + .textarea + .set_text_clearing_elements(&completed_text); + if !self.draft.textarea.text().is_empty() { + self.draft + .textarea + .set_cursor(self.draft.textarea.text().len()); + } + return (InputResult::None, true); + } + } + if self.is_task_running { + return self.handle_submission(/*should_queue*/ true); + } + (InputResult::None, true) + } + KeyEvent { + code: KeyCode::Char('/'), + modifiers: KeyModifiers::NONE, + .. + } => { + // Treat "/" as accepting the highlighted command as text completion + // while the slash-command popup is active. + let first_line = self.draft.textarea.text().lines().next().unwrap_or(""); + popup.on_composer_text_change(first_line.to_string()); + if let Some(selected_cmd) = popup.selected_item() { + if let Some(completed_text) = + selected_command_completion(first_line, &selected_cmd) + { + self.draft + .textarea + .set_text_clearing_elements(&completed_text); + self.draft.is_bash_mode = false; + } + if !self.draft.textarea.text().is_empty() { + self.draft + .textarea + .set_cursor(self.draft.textarea.text().len()); + } + } + (InputResult::None, true) + } + KeyEvent { + code: KeyCode::Enter, + modifiers: KeyModifiers::NONE, + .. + } => { + if let Some(sel) = popup.selected_item() { + self.stage_selected_slash_command_history(&sel); + self.draft.textarea.set_text_clearing_elements(""); + self.draft.is_bash_mode = false; + return ( + match sel { + CommandItem::Builtin(cmd) => InputResult::Command(cmd), + CommandItem::ServiceTier(command) => { + InputResult::ServiceTierCommand(command) + } + }, + true, + ); + } + // Fallback to default newline handling if no command selected. + self.handle_key_event_without_popup(key_event) + } + input => self.handle_input_basic(input), + } + } + + /// Keep slash command elements aligned with the current first line. + pub(super) fn sync_slash_command_elements(&mut self) { + if !self.slash_commands_enabled() { + return; + } + let text = self.draft.textarea.text(); + let first_line_end = text.find('\n').unwrap_or(text.len()); + let first_line = &text[..first_line_end]; + let desired_range = self.slash_input().command_element_range(first_line); + // Slash commands are only valid at byte 0 of the first line. + // Any slash-shaped element not matching the current desired prefix is stale. + let mut has_desired = false; + let mut stale_ranges = Vec::new(); + for elem in self.draft.textarea.text_elements() { + let Some(payload) = elem.placeholder(text) else { + continue; + }; + if payload.strip_prefix('/').is_none() { + continue; + } + let range = elem.byte_range.start..elem.byte_range.end; + if desired_range.as_ref() == Some(&range) { + has_desired = true; + } else { + stale_ranges.push(range); + } + } + + for range in stale_ranges { + self.draft.textarea.remove_element_range(range); + } + + if let Some(range) = desired_range + && !has_desired + { + self.draft.textarea.add_element_range(range); + } + } +} + +pub(super) fn selected_command_dispatches_immediately_on_tab(command: &CommandItem) -> bool { + matches!(command, CommandItem::Builtin(SlashCommand::Skills)) +} + +pub(super) fn selected_command_completion( + first_line: &str, + command: &CommandItem, +) -> Option { + let selected_command_text = format!("/{}", command.command()); + (!first_line.trim_start().starts_with(&selected_command_text)) + .then(|| format!("{selected_command_text} ")) +} + +pub(super) fn prepared_args(prepared_text: &str) -> Option<(&str, usize)> { + let (_, prepared_rest, prepared_rest_offset) = parse_slash_name(prepared_text)?; + Some((prepared_rest, prepared_rest_offset)) +} + +/// Translate full-text element ranges into command-argument ranges. +/// +/// `rest_offset` is the byte offset where `rest` begins in the full text. +pub(super) fn args_elements( + rest: &str, + rest_offset: usize, + text_elements: &[TextElement], +) -> Vec { + if rest.is_empty() || text_elements.is_empty() { + return Vec::new(); + } + text_elements + .iter() + .filter_map(|elem| { + if elem.byte_range.end <= rest_offset { + return None; + } + let start = elem.byte_range.start.saturating_sub(rest_offset); + let mut end = elem.byte_range.end.saturating_sub(rest_offset); + if start >= rest.len() { + return None; + } + end = end.min(rest.len()); + (start < end).then_some(elem.map_range(|_| ByteRange { start, end })) + }) + .collect() +} + +/// If the cursor is currently within a slash command on the first line, +/// extract the command name and the rest of the line after it. +fn command_under_cursor(first_line: &str, cursor: usize) -> Option<(&str, &str)> { + if !first_line.starts_with('/') { + return None; + } + + let name_start = 1usize; + let name_end = first_line[name_start..] + .find(char::is_whitespace) + .map(|idx| name_start + idx) + .unwrap_or_else(|| first_line.len()); + + if cursor > name_end { + return None; + } + + let name = &first_line[name_start..name_end]; + let rest_start = first_line[name_end..] + .find(|c: char| !c.is_whitespace()) + .map(|idx| name_end + idx) + .unwrap_or(name_end); + let rest = &first_line[rest_start..]; + + Some((name, rest)) +} From 08504e86fbbb3951796d9eb2f97d9c67b1a41b00 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Tue, 26 May 2026 19:48:32 +0200 Subject: [PATCH 02/16] Add goal extension telemetry parity (#24615) ## Why `core/src/goals.rs` already emits OTEL metrics for goal creation, resume, terminal transitions, token counts, and duration. As `/goal` moves into `ext/goal`, the extension needs to preserve that telemetry contract instead of only emitting app-visible `ThreadGoalUpdated` events. This keeps the existing `codex.goal.*` metric surface intact while goal lifecycle ownership shifts toward the extension. ## What changed - Added an extension-local `GoalMetrics` helper that records the existing `codex.goal.*` counters and histograms through `codex-otel`. - Threaded an optional `MetricsClient` through `install_with_backend`, `GoalExtension`, `GoalRuntimeHandle`, and `GoalToolExecutor`. - Emitted created, resumed, and terminal goal metrics from the extension paths that create goals, restore active goals on thread resume, account budget limits, complete or block goals, and handle external goal mutations. - Updated existing goal extension test setup callsites to pass `None` for metrics when instrumentation is not under test. ## Verification Not run locally. --- codex-rs/Cargo.lock | 1 + codex-rs/ext/goal/Cargo.toml | 1 + codex-rs/ext/goal/src/extension.rs | 46 ++++++++++ codex-rs/ext/goal/src/lib.rs | 1 + codex-rs/ext/goal/src/metrics.rs | 84 +++++++++++++++++++ codex-rs/ext/goal/src/runtime.rs | 49 +++++++++++ codex-rs/ext/goal/src/tool.rs | 42 +++++++++- .../ext/goal/tests/goal_extension_backend.rs | 16 +++- 8 files changed, 237 insertions(+), 3 deletions(-) create mode 100644 codex-rs/ext/goal/src/metrics.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index f95eb7a299..78acd3478d 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -2968,6 +2968,7 @@ dependencies = [ "chrono", "codex-core", "codex-extension-api", + "codex-otel", "codex-protocol", "codex-state", "codex-tools", diff --git a/codex-rs/ext/goal/Cargo.toml b/codex-rs/ext/goal/Cargo.toml index 80dbfa74b5..3bf0eca8d5 100644 --- a/codex-rs/ext/goal/Cargo.toml +++ b/codex-rs/ext/goal/Cargo.toml @@ -17,6 +17,7 @@ workspace = true async-trait = { workspace = true } codex-core = { workspace = true } codex-extension-api = { workspace = true } +codex-otel = { workspace = true } codex-protocol = { workspace = true } codex-state = { workspace = true } codex-tools = { workspace = true } diff --git a/codex-rs/ext/goal/src/extension.rs b/codex-rs/ext/goal/src/extension.rs index 82609285b7..aa0f658dae 100644 --- a/codex-rs/ext/goal/src/extension.rs +++ b/codex-rs/ext/goal/src/extension.rs @@ -8,6 +8,7 @@ use codex_extension_api::ExtensionData; use codex_extension_api::ExtensionEventSink; use codex_extension_api::ExtensionRegistryBuilder; use codex_extension_api::ThreadLifecycleContributor; +use codex_extension_api::ThreadResumeInput; use codex_extension_api::ThreadStartInput; use codex_extension_api::TokenUsageContributor; use codex_extension_api::ToolCallOutcome; @@ -19,6 +20,7 @@ use codex_extension_api::TurnAbortInput; use codex_extension_api::TurnLifecycleContributor; use codex_extension_api::TurnStartInput; use codex_extension_api::TurnStopInput; +use codex_otel::MetricsClient; use codex_protocol::ThreadId; use codex_protocol::protocol::ThreadGoalStatus; use codex_protocol::protocol::TokenUsageInfo; @@ -26,6 +28,7 @@ use codex_protocol::protocol::TokenUsageInfo; use crate::accounting::BudgetLimitedGoalDisposition; use crate::accounting::GoalAccountingState; use crate::events::GoalEventEmitter; +use crate::metrics::GoalMetrics; use crate::runtime::GoalRuntimeHandle; use crate::spec::UPDATE_GOAL_TOOL_NAME; use crate::steering::budget_limit_steering_item; @@ -46,6 +49,7 @@ impl GoalExtensionConfig { pub struct GoalExtension { state_dbs: Arc, event_emitter: GoalEventEmitter, + metrics: GoalMetrics, thread_manager: Weak, goals_enabled: Arc bool + Send + Sync>, } @@ -60,12 +64,14 @@ impl GoalExtension { pub(crate) fn new_with_host_capabilities( state_dbs: Arc, event_sink: Arc, + metrics_client: Option, thread_manager: Weak, goals_enabled: impl Fn(&C) -> bool + Send + Sync + 'static, ) -> Self { Self { state_dbs, event_emitter: GoalEventEmitter::new(event_sink), + metrics: GoalMetrics::new(metrics_client), thread_manager, goals_enabled: Arc::new(goals_enabled), } @@ -93,6 +99,7 @@ where thread_id, Arc::clone(&self.state_dbs), self.event_emitter.clone(), + self.metrics.clone(), self.thread_manager.clone(), accounting_state, enabled, @@ -100,6 +107,40 @@ where }); runtime.set_enabled(enabled); } + + async fn on_thread_resume(&self, input: ThreadResumeInput<'_>) { + let Some(runtime) = goal_runtime_handle(input.thread_store) else { + return; + }; + if !runtime.is_enabled() { + return; + } + + let goal = match self + .state_dbs + .thread_goals() + .get_thread_goal(runtime.thread_id()) + .await + { + Ok(goal) => goal, + Err(err) => { + tracing::warn!( + "failed to restore goal runtime after thread resume for {}: {err}", + runtime.thread_id() + ); + return; + } + }; + match goal { + Some(goal) if goal.status == codex_state::ThreadGoalStatus::Active => { + runtime + .accounting_state() + .mark_idle_goal_active(goal.goal_id); + self.metrics.record_resumed(); + } + Some(_) | None => runtime.accounting_state().clear_active_goal(), + } + } } impl ConfigContributor for GoalExtension @@ -320,18 +361,21 @@ where Arc::clone(&self.state_dbs), runtime.accounting_state(), self.event_emitter.clone(), + self.metrics.clone(), )), Arc::new(GoalToolExecutor::create( runtime.thread_id(), Arc::clone(&self.state_dbs), runtime.accounting_state(), self.event_emitter.clone(), + self.metrics.clone(), )), Arc::new(GoalToolExecutor::update( runtime.thread_id(), Arc::clone(&self.state_dbs), runtime.accounting_state(), self.event_emitter.clone(), + self.metrics.clone(), )), ] } @@ -340,6 +384,7 @@ where pub fn install_with_backend( registry: &mut ExtensionRegistryBuilder, state_dbs: Arc, + metrics_client: Option, thread_manager: Weak, goals_enabled: impl Fn(&C) -> bool + Send + Sync + 'static, ) where @@ -348,6 +393,7 @@ pub fn install_with_backend( let extension = Arc::new(GoalExtension::new_with_host_capabilities( state_dbs, registry.event_sink(), + metrics_client, thread_manager, goals_enabled, )); diff --git a/codex-rs/ext/goal/src/lib.rs b/codex-rs/ext/goal/src/lib.rs index c779f462e0..545d9e699a 100644 --- a/codex-rs/ext/goal/src/lib.rs +++ b/codex-rs/ext/goal/src/lib.rs @@ -7,6 +7,7 @@ mod accounting; mod events; mod extension; +mod metrics; mod runtime; mod spec; mod steering; diff --git a/codex-rs/ext/goal/src/metrics.rs b/codex-rs/ext/goal/src/metrics.rs new file mode 100644 index 0000000000..0bb227ef61 --- /dev/null +++ b/codex-rs/ext/goal/src/metrics.rs @@ -0,0 +1,84 @@ +use codex_otel::GOAL_BLOCKED_METRIC; +use codex_otel::GOAL_BUDGET_LIMITED_METRIC; +use codex_otel::GOAL_COMPLETED_METRIC; +use codex_otel::GOAL_CREATED_METRIC; +use codex_otel::GOAL_DURATION_SECONDS_METRIC; +use codex_otel::GOAL_RESUMED_METRIC; +use codex_otel::GOAL_TOKEN_COUNT_METRIC; +use codex_otel::GOAL_USAGE_LIMITED_METRIC; +use codex_otel::MetricsClient; + +#[derive(Clone, Default)] +pub(crate) struct GoalMetrics { + metrics_client: Option, +} + +impl GoalMetrics { + pub(crate) fn new(metrics_client: Option) -> Self { + Self { metrics_client } + } + + pub(crate) fn record_created(&self) { + let Some(metrics_client) = self.metrics_client.as_ref() else { + return; + }; + let _ = metrics_client.counter(GOAL_CREATED_METRIC, /*inc*/ 1, &[]); + } + + pub(crate) fn record_resumed(&self) { + let Some(metrics_client) = self.metrics_client.as_ref() else { + return; + }; + let _ = metrics_client.counter(GOAL_RESUMED_METRIC, /*inc*/ 1, &[]); + } + + pub(crate) fn record_resumed_if_status_changed( + &self, + previous_status: Option, + goal_status: codex_state::ThreadGoalStatus, + ) { + if goal_status == codex_state::ThreadGoalStatus::Active + && matches!( + previous_status, + Some( + codex_state::ThreadGoalStatus::Paused + | codex_state::ThreadGoalStatus::Blocked + | codex_state::ThreadGoalStatus::UsageLimited + ) + ) + { + self.record_resumed(); + } + } + + pub(crate) fn record_terminal_if_status_changed( + &self, + previous_status: Option, + goal: &codex_state::ThreadGoal, + ) { + if previous_status == Some(goal.status) { + return; + } + + let counter = match goal.status { + codex_state::ThreadGoalStatus::Blocked => GOAL_BLOCKED_METRIC, + codex_state::ThreadGoalStatus::UsageLimited => GOAL_USAGE_LIMITED_METRIC, + codex_state::ThreadGoalStatus::BudgetLimited => GOAL_BUDGET_LIMITED_METRIC, + codex_state::ThreadGoalStatus::Complete => GOAL_COMPLETED_METRIC, + codex_state::ThreadGoalStatus::Active | codex_state::ThreadGoalStatus::Paused => { + return; + } + }; + let Some(metrics_client) = self.metrics_client.as_ref() else { + return; + }; + let status_tag = [("status", goal.status.as_str())]; + let _ = metrics_client.counter(counter, /*inc*/ 1, &[]); + let _ = metrics_client.histogram(GOAL_TOKEN_COUNT_METRIC, goal.tokens_used, &status_tag); + let _ = metrics_client.histogram( + GOAL_DURATION_SECONDS_METRIC, + goal.time_used_seconds, + &status_tag, + ); + } +} diff --git a/codex-rs/ext/goal/src/runtime.rs b/codex-rs/ext/goal/src/runtime.rs index 074f78f7ec..689d0578cf 100644 --- a/codex-rs/ext/goal/src/runtime.rs +++ b/codex-rs/ext/goal/src/runtime.rs @@ -11,6 +11,7 @@ use codex_protocol::protocol::ThreadGoal; use crate::accounting::BudgetLimitedGoalDisposition; use crate::accounting::GoalAccountingState; use crate::events::GoalEventEmitter; +use crate::metrics::GoalMetrics; use crate::steering::objective_updated_steering_item; use crate::tool::protocol_goal_from_state; @@ -23,6 +24,7 @@ struct GoalRuntimeInner { thread_id: ThreadId, state_dbs: Arc, event_emitter: GoalEventEmitter, + metrics: GoalMetrics, thread_manager: Weak, accounting_state: Arc, enabled: AtomicBool, @@ -61,6 +63,7 @@ impl GoalRuntimeHandle { thread_id: ThreadId, state_dbs: Arc, event_emitter: GoalEventEmitter, + metrics: GoalMetrics, thread_manager: Weak, accounting_state: Arc, enabled: bool, @@ -70,6 +73,7 @@ impl GoalRuntimeHandle { thread_id, state_dbs, event_emitter, + metrics, thread_manager, accounting_state, enabled: AtomicBool::new(enabled), @@ -127,6 +131,21 @@ impl GoalRuntimeHandle { return Ok(()); } + let replaced_existing_goal = previous_goal + .as_ref() + .is_some_and(|previous_goal| previous_goal.goal_id != goal.goal_id); + if previous_goal.is_none() || replaced_existing_goal { + self.inner.metrics.record_created(); + } + let previous_status = previous_goal + .as_ref() + .and_then(|previous_goal| (!replaced_existing_goal).then_some(previous_goal.status)); + self.inner + .metrics + .record_resumed_if_status_changed(previous_status, goal.status); + self.inner + .metrics + .record_terminal_if_status_changed(previous_status, &goal); let should_steer_active_turn = previous_goal.as_ref().is_none_or(|previous_goal| { previous_goal.goal_id != goal.goal_id || previous_goal.status != codex_state::ThreadGoalStatus::Active @@ -202,6 +221,9 @@ impl GoalRuntimeHandle { let Some(snapshot) = accounting.progress_snapshot(turn_id) else { return Ok(None); }; + let previous_status = self + .current_goal_status_for_metrics(Some(snapshot.expected_goal_id.as_str())) + .await?; let outcome = self .inner .state_dbs @@ -218,6 +240,9 @@ impl GoalRuntimeHandle { Ok(match outcome { codex_state::GoalAccountingOutcome::Updated(goal) => { let goal_id = goal.goal_id.clone(); + self.inner + .metrics + .record_terminal_if_status_changed(previous_status, &goal); accounting.mark_progress_accounted_for_status( turn_id, &snapshot, @@ -246,6 +271,9 @@ impl GoalRuntimeHandle { let Some(snapshot) = accounting.idle_progress_snapshot() else { return Ok(None); }; + let previous_status = self + .current_goal_status_for_metrics(Some(snapshot.expected_goal_id.as_str())) + .await?; let outcome = self .inner .state_dbs @@ -262,6 +290,9 @@ impl GoalRuntimeHandle { Ok(match outcome { codex_state::GoalAccountingOutcome::Updated(goal) => { let goal_id = goal.goal_id.clone(); + self.inner + .metrics + .record_terminal_if_status_changed(previous_status, &goal); accounting.mark_idle_progress_accounted_for_status( &snapshot, goal.status, @@ -281,4 +312,22 @@ impl GoalRuntimeHandle { } }) } + + async fn current_goal_status_for_metrics( + &self, + expected_goal_id: Option<&str>, + ) -> Result, String> { + let goal = self + .inner + .state_dbs + .thread_goals() + .get_thread_goal(self.thread_id()) + .await + .map_err(|err| err.to_string())?; + Ok(goal.and_then(|goal| { + expected_goal_id + .is_none_or(|expected_goal_id| goal.goal_id == expected_goal_id) + .then_some(goal.status) + })) + } } diff --git a/codex-rs/ext/goal/src/tool.rs b/codex-rs/ext/goal/src/tool.rs index 39142cfe28..90ea0eb44b 100644 --- a/codex-rs/ext/goal/src/tool.rs +++ b/codex-rs/ext/goal/src/tool.rs @@ -18,6 +18,7 @@ use serde::Serialize; use crate::accounting::BudgetLimitedGoalDisposition; use crate::accounting::GoalAccountingState; use crate::events::GoalEventEmitter; +use crate::metrics::GoalMetrics; use crate::spec::CREATE_GOAL_TOOL_NAME; use crate::spec::GET_GOAL_TOOL_NAME; use crate::spec::UPDATE_GOAL_TOOL_NAME; @@ -32,6 +33,7 @@ pub(crate) struct GoalToolExecutor { state_db: Arc, accounting_state: Arc, event_emitter: GoalEventEmitter, + metrics: GoalMetrics, } #[derive(Clone, Copy)] @@ -74,6 +76,7 @@ impl GoalToolExecutor { state_db: Arc, accounting_state: Arc, event_emitter: GoalEventEmitter, + metrics: GoalMetrics, ) -> Self { Self { kind: GoalToolKind::Get, @@ -81,6 +84,7 @@ impl GoalToolExecutor { state_db, accounting_state, event_emitter, + metrics, } } @@ -89,6 +93,7 @@ impl GoalToolExecutor { state_db: Arc, accounting_state: Arc, event_emitter: GoalEventEmitter, + metrics: GoalMetrics, ) -> Self { Self { kind: GoalToolKind::Create, @@ -96,6 +101,7 @@ impl GoalToolExecutor { state_db, accounting_state, event_emitter, + metrics, } } @@ -104,6 +110,7 @@ impl GoalToolExecutor { state_db: Arc, accounting_state: Arc, event_emitter: GoalEventEmitter, + metrics: GoalMetrics, ) -> Self { Self { kind: GoalToolKind::Update, @@ -111,6 +118,7 @@ impl GoalToolExecutor { state_db, accounting_state, event_emitter, + metrics, } } } @@ -191,6 +199,7 @@ impl GoalToolExecutor { let turn_id = self .accounting_state .mark_current_turn_goal_active(goal.goal_id.clone()); + self.metrics.record_created(); let goal = protocol_goal_from_state(goal); self.emit_goal_updated_from_tool_call(&invocation, turn_id, goal.clone()); goal_response(Some(goal), CompletionBudgetReport::Omit) @@ -224,6 +233,9 @@ impl GoalToolExecutor { BudgetLimitedGoalDisposition::ClearActive, ) .await?; + let previous_status = self + .current_goal_status_for_metrics(/*expected_goal_id*/ None) + .await?; let goal = self .state_db .thread_goals() @@ -240,12 +252,14 @@ impl GoalToolExecutor { .map_err(|err| { FunctionCallError::RespondToModel(format!("failed to update goal: {err}")) })? - .map(protocol_goal_from_state) .ok_or_else(|| { FunctionCallError::RespondToModel( "cannot update goal because this thread has no goal".to_string(), ) })?; + self.metrics + .record_terminal_if_status_changed(previous_status, &goal); + let goal = protocol_goal_from_state(goal); let turn_id = self.accounting_state.clear_current_turn_goal(); self.emit_goal_updated_from_tool_call(&invocation, turn_id, goal.clone()); goal_response( @@ -280,6 +294,9 @@ impl GoalToolExecutor { let Some(snapshot) = self.accounting_state.progress_snapshot(turn_id.as_str()) else { return Ok(None); }; + let previous_status = self + .current_goal_status_for_metrics(Some(snapshot.expected_goal_id.as_str())) + .await?; let outcome = self .state_db .thread_goals() @@ -296,6 +313,8 @@ impl GoalToolExecutor { })?; Ok(match outcome { codex_state::GoalAccountingOutcome::Updated(goal) => { + self.metrics + .record_terminal_if_status_changed(previous_status, &goal); self.accounting_state.mark_progress_accounted_for_status( turn_id.as_str(), &snapshot, @@ -313,6 +332,27 @@ impl GoalToolExecutor { codex_state::GoalAccountingOutcome::Unchanged(_) => None, }) } + + async fn current_goal_status_for_metrics( + &self, + expected_goal_id: Option<&str>, + ) -> Result, FunctionCallError> { + let goal = self + .state_db + .thread_goals() + .get_thread_goal(self.thread_id) + .await + .map_err(|err| { + FunctionCallError::RespondToModel(format!( + "failed to read goal metrics status: {err}" + )) + })?; + Ok(goal.and_then(|goal| { + expected_goal_id + .is_none_or(|expected_goal_id| goal.goal_id == expected_goal_id) + .then_some(goal.status) + })) + } } fn parse_arguments(arguments: &str) -> Result diff --git a/codex-rs/ext/goal/tests/goal_extension_backend.rs b/codex-rs/ext/goal/tests/goal_extension_backend.rs index 85253345f6..0a3c768985 100644 --- a/codex-rs/ext/goal/tests/goal_extension_backend.rs +++ b/codex-rs/ext/goal/tests/goal_extension_backend.rs @@ -594,7 +594,13 @@ async fn installed_tools( thread_id: ThreadId, ) -> Vec>> { let mut builder = ExtensionRegistryBuilder::<()>::new(); - install_with_backend(&mut builder, runtime, Weak::new(), |_| true); + install_with_backend( + &mut builder, + runtime, + /*metrics_client*/ None, + Weak::new(), + |_| true, + ); let registry = builder.build(); let session_store = ExtensionData::new("session-1"); let thread_store = ExtensionData::new(thread_id.to_string()); @@ -629,7 +635,13 @@ impl GoalExtensionHarness { ) -> anyhow::Result { let sink = Arc::new(RecordingEventSink::default()); let mut builder = ExtensionRegistryBuilder::<()>::with_event_sink(sink.clone()); - install_with_backend(&mut builder, runtime, Weak::new(), |_| true); + install_with_backend( + &mut builder, + runtime, + /*metrics_client*/ None, + Weak::new(), + |_| true, + ); let registry = builder.build(); let session_store = ExtensionData::new("session-1"); let thread_store = ExtensionData::new(thread_id.to_string()); From 8a4a537e4471efbc62ec54b6a57c90a421b19b74 Mon Sep 17 00:00:00 2001 From: Felipe Coury Date: Tue, 26 May 2026 14:54:38 -0300 Subject: [PATCH 03/16] fix(tui): avoid modifyOtherKeys for unknown tmux formats (#24371) ## Why Codex 0.131 started enabling tmux `modifyOtherKeys` mode 2 when the active tmux session reported `extended-keys-format csi-u`, and also when that format could not be queried. The fallback was meant to help compatible tmux panes enter extended-key mode, but it breaks iTerm2 control-mode sessions on older tmux. Issue #23711 reproduces with: ```bash ssh -t ubuntu@192.168.68.149 'tmux -CC new -A -s main' ``` On tmux 3.2a, `extended-keys-format` is not available. With mode 2 enabled, `Ctrl-C` is delivered as `^[[27;5;99~` instead of the normal interrupt/control key path, so Codex does not handle it. Running with `CODEX_TUI_DISABLE_KEYBOARD_ENHANCEMENT=1` restores `Ctrl-C`, which points at keyboard mode setup rather than chat input routing. ## What Changed - Only request `modifyOtherKeys` mode 2 when tmux explicitly reports `extended-keys-format csi-u`. - Treat an unknown or unavailable tmux extended-key format as unsupported for this mode. - Update the keyboard mode unit coverage so `None` no longer opts into `modifyOtherKeys`. This preserves the explicit modern tmux `csi-u` path from #21943 while avoiding the unsafe fallback on older or unqueryable tmux setups. ## How to Test Regression path from #23711: 1. Start iTerm2 tmux integration against an older tmux host: ```bash ssh -t ubuntu@192.168.68.149 'tmux -CC new -A -s main' ``` 2. Start patched Codex. 3. Run `/keymap debug`, press a regular key, then press `Ctrl-C`. 4. Confirm `Ctrl-C` closes the inspector and Codex remains responsive without `CODEX_TUI_DISABLE_KEYBOARD_ENHANCEMENT=1`. 5. Confirm `Shift+Enter` still inserts a newline in the same session. Modern tmux compatibility path: 1. Start an ordinary tmux 3.6a server with explicit `csi-u`: ```bash tmux -L codex-csiu -f /dev/null new-session -d -s repro tmux -L codex-csiu set-option -g extended-keys on tmux -L codex-csiu set-option -g extended-keys-format csi-u tmux -L codex-csiu attach -t repro ``` 2. Start patched Codex. 3. From another terminal, confirm the Codex pane reports `mode=Ext 2`: ```bash tmux -L codex-csiu list-panes -a -F '#{pane_id} mode=#{pane_key_mode} cmd=#{pane_current_command}' ``` 4. Type `one`, press `Shift+Enter`, type `two`, and confirm the composer shows two lines without submitting. 5. Press `Ctrl-C` and confirm Codex handles it normally. Targeted tests: - `./tools/argument-comment-lint/run.py -p codex-tui -- --lib` - `just test -p codex-tui` runs the new keyboard mode test successfully; the full run currently reports two unrelated guardian feature-flag test failures: - `app::tests::update_feature_flags_disabling_guardian_clears_manual_review_policy_without_history` - `app::tests::update_feature_flags_disabling_guardian_clears_review_policy_and_restores_default` No documentation update is needed. --- codex-rs/tui/src/tui/keyboard_modes.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/codex-rs/tui/src/tui/keyboard_modes.rs b/codex-rs/tui/src/tui/keyboard_modes.rs index 4e0702bcbd..ca4ae3bce2 100644 --- a/codex-rs/tui/src/tui/keyboard_modes.rs +++ b/codex-rs/tui/src/tui/keyboard_modes.rs @@ -160,10 +160,10 @@ fn tmux_should_enable_modify_other_keys_for( running_in_tmux_session: bool, extended_keys_format: Option<&str>, ) -> bool { - // If tmux cannot be queried, still request mode 2; otherwise csi-u panes - // can stay in VT10x. Explicit xterm format is avoided because crossterm - // does not parse tmux's xterm-style extended-key sequences as Enter. - running_in_tmux_session && matches!(extended_keys_format, Some("csi-u") | None) + // Only request mode 2 when tmux confirms csi-u formatting. Older tmux + // versions do not expose this option and may emit xterm-style sequences, + // which crossterm does not parse consistently for modified keys. + running_in_tmux_session && matches!(extended_keys_format, Some("csi-u")) } fn read_tmux_extended_keys_format() -> Option { @@ -371,12 +371,12 @@ mod tests { } #[test] - fn tmux_modify_other_keys_requests_csi_u_or_unknown_format() { + fn tmux_modify_other_keys_only_requests_confirmed_csi_u_format() { assert!(tmux_should_enable_modify_other_keys_for( /*running_in_tmux_session*/ true, Some("csi-u") )); - assert!(tmux_should_enable_modify_other_keys_for( + assert!(!tmux_should_enable_modify_other_keys_for( /*running_in_tmux_session*/ true, /*extended_keys_format*/ None )); assert!(!tmux_should_enable_modify_other_keys_for( From 823381e867ae21babe8a41c1b070fc21b5ddd81d Mon Sep 17 00:00:00 2001 From: jif-oai Date: Tue, 26 May 2026 20:01:13 +0200 Subject: [PATCH 04/16] fix: restore goal accounting after thread resume (#24626) ## Why Goal idle accounting is supposed to survive a thread resume. Previously, the resume hook restored the active goal state inline from the extension lifecycle contributor, which left the runtime handle without a reusable restoration path and made the behavior hard to cover directly. When a thread with an active goal was resumed, goal accounting could lose track of the active idle goal instead of continuing to accrue elapsed time. ## What changed - Moved thread-resume restoration into `GoalRuntimeHandle::restore_after_resume()` so the runtime owns rehydrating active goal accounting from persisted thread goal state. - Kept disabled goal runtimes as a no-op and preserved the existing warning path when persisted goal state cannot be loaded. - Added a backend regression test that seeds an active goal, resumes the thread, waits briefly, and verifies elapsed idle time is reflected on the next external goal mutation. ## Testing - Not run locally; this metadata update only rewrote the PR title/body. --- codex-rs/ext/goal/src/extension.rs | 31 ++-------- codex-rs/ext/goal/src/runtime.rs | 24 ++++++++ .../ext/goal/tests/goal_extension_backend.rs | 59 +++++++++++++++++++ 3 files changed, 88 insertions(+), 26 deletions(-) diff --git a/codex-rs/ext/goal/src/extension.rs b/codex-rs/ext/goal/src/extension.rs index aa0f658dae..68068acbb4 100644 --- a/codex-rs/ext/goal/src/extension.rs +++ b/codex-rs/ext/goal/src/extension.rs @@ -112,33 +112,12 @@ where let Some(runtime) = goal_runtime_handle(input.thread_store) else { return; }; - if !runtime.is_enabled() { - return; - } - let goal = match self - .state_dbs - .thread_goals() - .get_thread_goal(runtime.thread_id()) - .await - { - Ok(goal) => goal, - Err(err) => { - tracing::warn!( - "failed to restore goal runtime after thread resume for {}: {err}", - runtime.thread_id() - ); - return; - } - }; - match goal { - Some(goal) if goal.status == codex_state::ThreadGoalStatus::Active => { - runtime - .accounting_state() - .mark_idle_goal_active(goal.goal_id); - self.metrics.record_resumed(); - } - Some(_) | None => runtime.accounting_state().clear_active_goal(), + if let Err(err) = runtime.restore_after_resume().await { + tracing::warn!( + "failed to restore goal runtime after thread resume for {}: {err}", + runtime.thread_id() + ); } } } diff --git a/codex-rs/ext/goal/src/runtime.rs b/codex-rs/ext/goal/src/runtime.rs index 689d0578cf..0139cc2c52 100644 --- a/codex-rs/ext/goal/src/runtime.rs +++ b/codex-rs/ext/goal/src/runtime.rs @@ -192,6 +192,30 @@ impl GoalRuntimeHandle { Ok(()) } + pub async fn restore_after_resume(&self) -> Result<(), String> { + if !self.is_enabled() { + return Ok(()); + } + + let goal = self + .inner + .state_dbs + .thread_goals() + .get_thread_goal(self.thread_id()) + .await + .map_err(|err| err.to_string())?; + match goal { + Some(goal) if goal.status == codex_state::ThreadGoalStatus::Active => { + self.inner + .accounting_state + .mark_idle_goal_active(goal.goal_id); + self.inner.metrics.record_resumed(); + } + Some(_) | None => self.inner.accounting_state.clear_active_goal(), + } + Ok(()) + } + pub(crate) async fn inject_active_turn_steering(&self, item: ResponseInputItem) { let Some(thread_manager) = self.inner.thread_manager.upgrade() else { tracing::debug!("skipping goal steering because thread manager is unavailable"); diff --git a/codex-rs/ext/goal/tests/goal_extension_backend.rs b/codex-rs/ext/goal/tests/goal_extension_backend.rs index 0a3c768985..0a36439b25 100644 --- a/codex-rs/ext/goal/tests/goal_extension_backend.rs +++ b/codex-rs/ext/goal/tests/goal_extension_backend.rs @@ -2,11 +2,13 @@ use std::sync::Arc; use std::sync::Mutex; use std::sync::PoisonError; use std::sync::Weak; +use std::time::Duration; use codex_extension_api::ExtensionData; use codex_extension_api::ExtensionEventSink; use codex_extension_api::ExtensionRegistryBuilder; use codex_extension_api::FunctionCallError; +use codex_extension_api::ThreadResumeInput; use codex_extension_api::ThreadStartInput; use codex_extension_api::ToolCall; use codex_extension_api::ToolCallOutcome; @@ -589,6 +591,52 @@ async fn external_goal_set_active_resets_baseline_without_live_thread() -> anyho Ok(()) } +#[tokio::test] +async fn thread_resume_rehydrates_active_goal_idle_accounting() -> anyhow::Result<()> { + let runtime = test_runtime().await?; + let thread_id = test_thread_id()?; + seed_thread_metadata(runtime.as_ref(), thread_id).await?; + runtime + .thread_goals() + .replace_thread_goal( + thread_id, + "ship goal extension backend", + codex_state::ThreadGoalStatus::Active, + /*token_budget*/ None, + ) + .await?; + let harness = GoalExtensionHarness::new(runtime.clone(), thread_id).await?; + + harness.resume_thread().await; + tokio::time::sleep(Duration::from_millis(1_100)).await; + harness + .runtime_handle() + .prepare_external_goal_mutation() + .await + .map_err(anyhow::Error::msg)?; + + let goal = runtime + .thread_goals() + .get_thread_goal(thread_id) + .await? + .ok_or_else(|| anyhow::anyhow!("goal should exist"))?; + assert_eq!(ThreadGoalStatus::Active, protocol_status(goal.status)); + assert!( + goal.time_used_seconds >= 1, + "resumed idle accounting should add elapsed wall-clock time" + ); + assert_eq!( + vec![CapturedGoalEvent { + event_id: format!("{thread_id}:external-goal-mutation"), + turn_id: None, + status: ThreadGoalStatus::Active, + tokens_used: 0, + }], + harness.sink.goal_events() + ); + Ok(()) +} + async fn installed_tools( runtime: Arc, thread_id: ThreadId, @@ -719,6 +767,17 @@ impl GoalExtensionHarness { } } + async fn resume_thread(&self) { + for contributor in self.registry.thread_lifecycle_contributors() { + contributor + .on_thread_resume(ThreadResumeInput { + session_store: &self.session_store, + thread_store: &self.thread_store, + }) + .await; + } + } + async fn notify_tool_finish(&self, turn_id: &str, call_id: &str, tool_name: &str) { let turn_store = ExtensionData::new(turn_id); let tool_name = codex_extension_api::ToolName::plain(tool_name); From aad59a0916baea1875f9a609a4b1d0a9062f24f6 Mon Sep 17 00:00:00 2001 From: jif-oai Date: Tue, 26 May 2026 20:07:25 +0200 Subject: [PATCH 05/16] Move memory state to a dedicated SQLite DB (#24591) ## Summary Generated memory rows and their stage-one/stage-two job state currently live in `state_5.sqlite` alongside thread metadata. That makes memory cleanup and regeneration share the main state schema even though those rows are memory-pipeline data and can be rebuilt independently from the durable thread records. This PR moves the memory-owned tables into a dedicated `memories_1.sqlite` runtime database while keeping thread metadata in `state_5.sqlite`. ## Changes - Adds a separate memories DB runtime, migrator, path helpers, telemetry kind, and Bazel compile data for `state/memory_migrations`. - Introduces `MemoryStore` behind `StateRuntime::memories()` and moves memory table/job operations onto that store. - Drops the old memory tables from the state DB and recreates their schema in `state/memory_migrations/0001_memories.sql`. - Updates memory startup, citation usage tracking, rollout pollution handling, `debug clear-memories`, and app-server `memory/reset` to operate through the memories DB. - Preserves cross-DB behavior by hydrating thread metadata from the state DB when selecting visible memory outputs and checking stage-one staleness. ## Verification - Added/updated `codex-state` tests for deleted-thread memory visibility and already-polluted phase-two enqueue behavior. - Updated `debug clear-memories`, app-server `memory/reset`, and memories startup tests to seed and assert memory rows through `memories_1.sqlite`. --- .../request_processors/thread_processor.rs | 10 +- .../app-server/tests/suite/v2/memory_reset.rs | 8 +- codex-rs/cli/src/doctor/output.rs | 32 + codex-rs/cli/src/doctor/output/detail.rs | 3 + codex-rs/cli/src/main.rs | 20 +- codex-rs/cli/tests/debug_clear_memories.rs | 62 +- codex-rs/core/src/stream_events_utils.rs | 2 +- codex-rs/memories/write/src/phase1.rs | 11 +- codex-rs/memories/write/src/phase2.rs | 21 +- codex-rs/memories/write/src/startup_tests.rs | 23 +- codex-rs/rollout/src/state_db.rs | 8 +- codex-rs/state/BUILD.bazel | 7 +- .../state/memory_migrations/0001_memories.sql | 35 + .../migrations/0035_drop_memory_tables.sql | 2 + codex-rs/state/src/lib.rs | 4 + codex-rs/state/src/migrations.rs | 5 + codex-rs/state/src/model/memories.rs | 55 -- codex-rs/state/src/model/mod.rs | 1 - codex-rs/state/src/runtime.rs | 73 +- codex-rs/state/src/runtime/memories.rs | 909 ++++++++++++++---- codex-rs/state/src/runtime/threads.rs | 1 + codex-rs/state/src/telemetry.rs | 2 + 22 files changed, 1014 insertions(+), 280 deletions(-) create mode 100644 codex-rs/state/memory_migrations/0001_memories.sql create mode 100644 codex-rs/state/migrations/0035_drop_memory_tables.sql diff --git a/codex-rs/app-server/src/request_processors/thread_processor.rs b/codex-rs/app-server/src/request_processors/thread_processor.rs index 1766c971b1..c9a42d8359 100644 --- a/codex-rs/app-server/src/request_processors/thread_processor.rs +++ b/codex-rs/app-server/src/request_processors/thread_processor.rs @@ -1513,9 +1513,13 @@ impl ThreadRequestProcessor { .clone() .ok_or_else(|| internal_error("sqlite state db unavailable for memory reset"))?; - state_db.clear_memory_data().await.map_err(|err| { - internal_error(format!("failed to clear memory rows in state db: {err}")) - })?; + state_db + .memories() + .clear_memory_data() + .await + .map_err(|err| { + internal_error(format!("failed to clear memory rows in memories db: {err}")) + })?; clear_memory_roots_contents(&self.config.codex_home) .await diff --git a/codex-rs/app-server/tests/suite/v2/memory_reset.rs b/codex-rs/app-server/tests/suite/v2/memory_reset.rs index 3c7ae38671..7b2eddeb53 100644 --- a/codex-rs/app-server/tests/suite/v2/memory_reset.rs +++ b/codex-rs/app-server/tests/suite/v2/memory_reset.rs @@ -49,7 +49,10 @@ async fn memory_reset_clears_memory_files_and_rows_preserves_threads() -> Result .await??; let _: MemoryResetResponse = to_response::(response)?; - let stage1_outputs = state_db.list_stage1_outputs_for_global(/*n*/ 10).await?; + let stage1_outputs = state_db + .memories() + .list_stage1_outputs_for_global(/*n*/ 10) + .await?; assert_eq!(stage1_outputs, Vec::new()); assert_eq!( state_db.get_thread_memory_mode(thread_id).await?.as_deref(), @@ -81,6 +84,7 @@ async fn seed_stage1_output(state_db: &Arc, codex_home: &Path) -> state_db.upsert_thread(&metadata).await?; let claim = state_db + .memories() .try_claim_stage1_job( thread_id, worker_id, @@ -94,6 +98,7 @@ async fn seed_stage1_output(state_db: &Arc, codex_home: &Path) -> }; assert!( state_db + .memories() .mark_stage1_job_succeeded( thread_id, ownership_token.as_str(), @@ -106,6 +111,7 @@ async fn seed_stage1_output(state_db: &Arc, codex_home: &Path) -> "stage1 success should be recorded" ); state_db + .memories() .enqueue_global_consolidation(now.timestamp()) .await?; diff --git a/codex-rs/cli/src/doctor/output.rs b/codex-rs/cli/src/doctor/output.rs index ec09a4d69f..3f6e2f05d2 100644 --- a/codex-rs/cli/src/doctor/output.rs +++ b/codex-rs/cli/src/doctor/output.rs @@ -702,6 +702,7 @@ fn state_summary(check: &DoctorCheck) -> String { "state DB integrity", "log DB integrity", "goals DB integrity", + "memories DB integrity", ] .into_iter() .all(|label| detail::detail_value(check, label).is_some_and(|value| value == "ok")); @@ -1363,6 +1364,37 @@ Run codex doctor without --summary for detailed diagnostics. ); } + #[test] + fn render_human_report_includes_memories_db_in_state_health_summary() { + let report = DoctorReport { + schema_version: 1, + generated_at: "0s since unix epoch".to_string(), + overall_status: CheckStatus::Ok, + codex_version: "0.0.0".to_string(), + checks: vec![ + DoctorCheck::new( + "state.paths", + "state", + CheckStatus::Ok, + "state paths inspectable", + ) + .detail("state DB: /tmp/state.sqlite") + .detail("state DB integrity: ok") + .detail("log DB: /tmp/logs.sqlite") + .detail("log DB integrity: ok") + .detail("goals DB: /tmp/goals.sqlite") + .detail("goals DB integrity: ok") + .detail("memories DB: /tmp/memories.sqlite") + .detail("memories DB integrity: ok"), + ], + }; + + let rendered = render_human_report(&report, detailed_no_color_unicode_options()); + + assert!(rendered.contains("✓ state databases healthy")); + assert!(rendered.contains("memories DB /tmp/memories.sqlite · integrity ok")); + } + #[test] fn render_human_report_supports_ascii_output() { let rendered = render_human_report( diff --git a/codex-rs/cli/src/doctor/output/detail.rs b/codex-rs/cli/src/doctor/output/detail.rs index 86cbf2a83f..6c6cf8633c 100644 --- a/codex-rs/cli/src/doctor/output/detail.rs +++ b/codex-rs/cli/src/doctor/output/detail.rs @@ -413,6 +413,7 @@ fn state_details(parsed: &[ParsedDetail]) -> Vec { push_database_row(&mut out, parsed, "state DB"); push_database_row(&mut out, parsed, "log DB"); push_database_row(&mut out, parsed, "goals DB"); + push_database_row(&mut out, parsed, "memories DB"); for (source, label) in [ ("active rollout files", "active rollouts"), @@ -440,6 +441,8 @@ fn state_details(parsed: &[ParsedDetail]) -> Vec { "state DB integrity", "log DB integrity", "goals DB integrity", + "memories DB", + "memories DB integrity", "active rollout files", "archived rollout files", ], diff --git a/codex-rs/cli/src/main.rs b/codex-rs/cli/src/main.rs index 4bb7ef74a0..856273354c 100644 --- a/codex-rs/cli/src/main.rs +++ b/codex-rs/cli/src/main.rs @@ -27,7 +27,7 @@ use codex_responses_api_proxy::Args as ResponsesApiProxyArgs; use codex_rollout_trace::REDUCED_STATE_FILE_NAME; use codex_rollout_trace::replay_bundle; use codex_state::StateRuntime; -use codex_state::state_db_path; +use codex_state::memories_db_path; use codex_tui::AppExitInfo; use codex_tui::Cli as TuiCli; use codex_tui::ExitReason; @@ -1751,22 +1751,16 @@ async fn run_debug_clear_memories_command( .build() .await?; - let state_path = state_db_path(config.sqlite_home.as_path()); - let mut cleared_state_db = false; - if tokio::fs::try_exists(&state_path).await? { - let state_db = - StateRuntime::init(config.sqlite_home.clone(), config.model_provider_id.clone()) - .await?; - state_db.clear_memory_data().await?; - cleared_state_db = true; - } + let memories_path = memories_db_path(config.sqlite_home.as_path()); + let cleared_memories_db = + StateRuntime::clear_memory_data_in_sqlite_home(config.sqlite_home.as_path()).await?; clear_memory_roots_contents(&config.codex_home).await?; - let mut message = if cleared_state_db { - format!("Cleared memory state from {}.", state_path.display()) + let mut message = if cleared_memories_db { + format!("Cleared memory state from {}.", memories_path.display()) } else { - format!("No state db found at {}.", state_path.display()) + format!("No memories db found at {}.", memories_path.display()) }; message.push_str(&format!( " Cleared memory directories under {}.", diff --git a/codex-rs/cli/tests/debug_clear_memories.rs b/codex-rs/cli/tests/debug_clear_memories.rs index 9d5e114dbf..e39d736ae0 100644 --- a/codex-rs/cli/tests/debug_clear_memories.rs +++ b/codex-rs/cli/tests/debug_clear_memories.rs @@ -2,6 +2,7 @@ use std::path::Path; use anyhow::Result; use codex_state::StateRuntime; +use codex_state::memories_db_path; use codex_state::state_db_path; use predicates::str::contains; use sqlx::SqlitePool; @@ -23,6 +24,9 @@ async fn debug_clear_memories_resets_state_and_removes_memory_dir() -> Result<() let thread_id = "00000000-0000-0000-0000-000000000123"; let db_path = state_db_path(codex_home.path()); let pool = SqlitePool::connect(&format!("sqlite://{}", db_path.display())).await?; + let memories_db_path = memories_db_path(codex_home.path()); + let memories_pool = + SqlitePool::connect(&format!("sqlite://{}", memories_db_path.display())).await?; sqlx::query( r#" @@ -74,7 +78,7 @@ INSERT INTO stage1_outputs ( "#, ) .bind(thread_id) - .execute(&pool) + .execute(&memories_pool) .await?; sqlx::query( @@ -99,13 +103,14 @@ INSERT INTO jobs ( "#, ) .bind(thread_id) - .execute(&pool) + .execute(&memories_pool) .await?; let memory_root = codex_home.path().join("memories"); std::fs::create_dir_all(&memory_root)?; std::fs::write(memory_root.join("memory_summary.md"), "stale memory")?; pool.close().await; + memories_pool.close().await; let mut cmd = codex_command(codex_home.path())?; cmd.args(["debug", "clear-memories"]) @@ -113,7 +118,7 @@ INSERT INTO jobs ( .success() .stdout(contains("Cleared memory state")); - let pool = SqlitePool::connect(&format!("sqlite://{}", db_path.display())).await?; + let pool = SqlitePool::connect(&format!("sqlite://{}", memories_db_path.display())).await?; let stage1_outputs_count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM stage1_outputs") .fetch_one(&pool) .await?; @@ -131,3 +136,54 @@ INSERT INTO jobs ( Ok(()) } + +#[tokio::test] +async fn debug_clear_memories_resets_memories_db_without_state_db() -> Result<()> { + let codex_home = TempDir::new()?; + let runtime = + StateRuntime::init(codex_home.path().to_path_buf(), "test-provider".to_string()).await?; + drop(runtime); + + let db_path = state_db_path(codex_home.path()); + let memories_db_path = memories_db_path(codex_home.path()); + let memories_pool = + SqlitePool::connect(&format!("sqlite://{}", memories_db_path.display())).await?; + + sqlx::query( + r#" +INSERT INTO stage1_outputs ( + thread_id, + source_updated_at, + raw_memory, + rollout_summary, + generated_at, + rollout_slug, + usage_count, + last_usage, + selected_for_phase2, + selected_for_phase2_source_updated_at +) VALUES ('00000000-0000-0000-0000-000000000123', 1, 'raw', 'summary', 1, NULL, 0, NULL, 0, NULL) + "#, + ) + .execute(&memories_pool) + .await?; + + memories_pool.close().await; + std::fs::remove_file(&db_path)?; + + let mut cmd = codex_command(codex_home.path())?; + cmd.args(["debug", "clear-memories"]) + .assert() + .success() + .stdout(contains("Cleared memory state")); + + let pool = SqlitePool::connect(&format!("sqlite://{}", memories_db_path.display())).await?; + let stage1_outputs_count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM stage1_outputs") + .fetch_one(&pool) + .await?; + assert_eq!(stage1_outputs_count, 0); + pool.close().await; + assert!(!db_path.exists()); + + Ok(()) +} diff --git a/codex-rs/core/src/stream_events_utils.rs b/codex-rs/core/src/stream_events_utils.rs index 0e32ccb9fb..1ee4b4b58f 100644 --- a/codex-rs/core/src/stream_events_utils.rs +++ b/codex-rs/core/src/stream_events_utils.rs @@ -233,7 +233,7 @@ async fn record_stage1_output_usage_for_memory_citation( } if let Some(db) = state_db_ctx { - let _ = db.record_stage1_output_usage(&thread_ids).await; + let _ = db.memories().record_stage1_output_usage(&thread_ids).await; } true } diff --git a/codex-rs/memories/write/src/phase1.rs b/codex-rs/memories/write/src/phase1.rs index 9c6c2561de..1844ab3b52 100644 --- a/codex-rs/memories/write/src/phase1.rs +++ b/codex-rs/memories/write/src/phase1.rs @@ -112,6 +112,7 @@ pub async fn prune(context: &MemoryStartupContext, config: &Config) { if let Some(db) = context.state_db() { let max_unused_days = config.memories.max_unused_days; match db + .memories() .prune_stage1_outputs_for_retention(max_unused_days, crate::stage_one::PRUNE_BATCH_SIZE) .await { @@ -124,7 +125,7 @@ pub async fn prune(context: &MemoryStartupContext, config: &Config) { } Err(err) => { warn!( - "state db prune_stage1_outputs_for_retention failed during memories startup: {err}" + "memories db prune_stage1_outputs_for_retention failed during memories startup: {err}" ); } } @@ -161,6 +162,7 @@ async fn claim_startup_jobs( .collect::>(); match state_db + .memories() .claim_stage1_jobs_for_startup( context.thread_id(), codex_state::Stage1StartupClaimParams { @@ -176,7 +178,9 @@ async fn claim_startup_jobs( { Ok(claims) => Some(claims), Err(err) => { - warn!("state db claim_stage1_jobs_for_startup failed during memories startup: {err}"); + warn!( + "memories db claim_stage1_jobs_for_startup failed during memories startup: {err}" + ); None } } @@ -329,6 +333,7 @@ mod job { tracing::warn!("Phase 1 job failed for thread {thread_id}: {reason}"); if let Some(state_db) = context.state_db() { let _ = state_db + .memories() .mark_stage1_job_failed( thread_id, ownership_token, @@ -349,6 +354,7 @@ mod job { }; if state_db + .memories() .mark_stage1_job_succeeded_no_output(thread_id, ownership_token) .await .unwrap_or(false) @@ -373,6 +379,7 @@ mod job { }; if state_db + .memories() .mark_stage1_job_succeeded( thread_id, ownership_token, diff --git a/codex-rs/memories/write/src/phase2.rs b/codex-rs/memories/write/src/phase2.rs index 7a3841f8c3..c78032d9c2 100644 --- a/codex-rs/memories/write/src/phase2.rs +++ b/codex-rs/memories/write/src/phase2.rs @@ -91,6 +91,7 @@ pub async fn run(context: Arc, config: Arc) { // 4. Load current DB-backed Phase 2 inputs. let raw_memories = match db + .memories() .get_phase2_input_selection(max_raw_memories, max_unused_days) .await { @@ -217,6 +218,7 @@ mod job { db: &StateRuntime, ) -> Result { let claim = db + .memories() .try_claim_global_phase2_job(context.thread_id(), crate::stage_two::JOB_LEASE_SECONDS) .await .map_err(|e| { @@ -255,15 +257,17 @@ mod job { ) { context.counter(MEMORY_PHASE_TWO_JOBS, /*inc*/ 1, &[("status", reason)]); if matches!( - db.mark_global_phase2_job_failed( - &claim.token, - reason, - crate::stage_two::JOB_RETRY_DELAY_SECONDS, - ) - .await, + db.memories() + .mark_global_phase2_job_failed( + &claim.token, + reason, + crate::stage_two::JOB_RETRY_DELAY_SECONDS, + ) + .await, Ok(false) ) { let _ = db + .memories() .mark_global_phase2_job_failed_if_unowned( &claim.token, reason, @@ -282,7 +286,8 @@ mod job { reason: &'static str, ) -> bool { context.counter(MEMORY_PHASE_TWO_JOBS, /*inc*/ 1, &[("status", reason)]); - db.mark_global_phase2_job_succeeded(&claim.token, completion_watermark, selected_outputs) + db.memories() + .mark_global_phase2_job_succeeded(&claim.token, completion_watermark, selected_outputs) .await .unwrap_or(false) } @@ -382,6 +387,7 @@ mod agent { } // Do not reset the workspace baseline if we lost the lock. let still_owns_lock = match db + .memories() .heartbeat_global_phase2_job( &claim.token, crate::stage_two::JOB_LEASE_SECONDS, @@ -479,6 +485,7 @@ mod agent { } _ = heartbeat_interval.tick() => { match db + .memories() .heartbeat_global_phase2_job( &token, crate::stage_two::JOB_LEASE_SECONDS, diff --git a/codex-rs/memories/write/src/startup_tests.rs b/codex-rs/memories/write/src/startup_tests.rs index 30240f04ca..4af8560e67 100644 --- a/codex-rs/memories/write/src/startup_tests.rs +++ b/codex-rs/memories/write/src/startup_tests.rs @@ -190,7 +190,8 @@ async fn memories_startup_phase2_prunes_old_extension_resources_without_stage1_i let server = start_mock_server().await; let home = Arc::new(TempDir::new()?); let db = init_state_db(&home).await?; - db.enqueue_global_consolidation(/*input_watermark*/ 1) + db.memories() + .enqueue_global_consolidation(/*input_watermark*/ 1) .await?; let now = chrono::Utc::now(); @@ -445,6 +446,7 @@ async fn seed_stage1_output_for_existing_thread( ) -> anyhow::Result<()> { let owner = ThreadId::new(); let claim = db + .memories() .try_claim_stage1_job( thread_id, owner, updated_at, /*lease_seconds*/ 3_600, /*max_running_jobs*/ 64, @@ -456,15 +458,16 @@ async fn seed_stage1_output_for_existing_thread( }; assert!( - db.mark_stage1_job_succeeded( - thread_id, - &ownership_token, - updated_at, - raw_memory, - rollout_summary, - rollout_slug, - ) - .await?, + db.memories() + .mark_stage1_job_succeeded( + thread_id, + &ownership_token, + updated_at, + raw_memory, + rollout_summary, + rollout_slug, + ) + .await?, "stage-1 success should enqueue global consolidation" ); diff --git a/codex-rs/rollout/src/state_db.rs b/codex-rs/rollout/src/state_db.rs index 3bd8dd8e01..ea087e5c6d 100644 --- a/codex-rs/rollout/src/state_db.rs +++ b/codex-rs/rollout/src/state_db.rs @@ -493,8 +493,12 @@ pub async fn mark_thread_memory_mode_polluted( let Some(ctx) = context else { return; }; - if let Err(err) = ctx.mark_thread_memory_mode_polluted(thread_id).await { - warn!("state db mark_thread_memory_mode_polluted failed during {stage}: {err}"); + if let Err(err) = ctx + .memories() + .mark_thread_memory_mode_polluted(thread_id) + .await + { + warn!("memories db mark_thread_memory_mode_polluted failed during {stage}: {err}"); } } diff --git a/codex-rs/state/BUILD.bazel b/codex-rs/state/BUILD.bazel index b3f0fecab7..9225b6c126 100644 --- a/codex-rs/state/BUILD.bazel +++ b/codex-rs/state/BUILD.bazel @@ -3,5 +3,10 @@ load("//:defs.bzl", "codex_rust_crate") codex_rust_crate( name = "state", crate_name = "codex_state", - compile_data = glob(["goals_migrations/**", "logs_migrations/**", "migrations/**"]), + compile_data = glob([ + "goals_migrations/**", + "logs_migrations/**", + "memory_migrations/**", + "migrations/**", + ]), ) diff --git a/codex-rs/state/memory_migrations/0001_memories.sql b/codex-rs/state/memory_migrations/0001_memories.sql new file mode 100644 index 0000000000..4e5b3a1e9f --- /dev/null +++ b/codex-rs/state/memory_migrations/0001_memories.sql @@ -0,0 +1,35 @@ +CREATE TABLE stage1_outputs ( + thread_id TEXT PRIMARY KEY, + source_updated_at INTEGER NOT NULL, + raw_memory TEXT NOT NULL, + rollout_summary TEXT NOT NULL, + rollout_slug TEXT, + generated_at INTEGER NOT NULL, + usage_count INTEGER, + last_usage INTEGER, + selected_for_phase2 INTEGER NOT NULL DEFAULT 0, + selected_for_phase2_source_updated_at INTEGER +); + +CREATE INDEX idx_stage1_outputs_source_updated_at + ON stage1_outputs(source_updated_at DESC, thread_id DESC); + +CREATE TABLE jobs ( + kind TEXT NOT NULL, + job_key TEXT NOT NULL, + status TEXT NOT NULL, + worker_id TEXT, + ownership_token TEXT, + started_at INTEGER, + finished_at INTEGER, + lease_until INTEGER, + retry_at INTEGER, + retry_remaining INTEGER NOT NULL, + last_error TEXT, + input_watermark INTEGER, + last_success_watermark INTEGER, + PRIMARY KEY (kind, job_key) +); + +CREATE INDEX idx_jobs_kind_status_retry_lease + ON jobs(kind, status, retry_at, lease_until); diff --git a/codex-rs/state/migrations/0035_drop_memory_tables.sql b/codex-rs/state/migrations/0035_drop_memory_tables.sql new file mode 100644 index 0000000000..49721e3e54 --- /dev/null +++ b/codex-rs/state/migrations/0035_drop_memory_tables.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS jobs; +DROP TABLE IF EXISTS stage1_outputs; diff --git a/codex-rs/state/src/lib.rs b/codex-rs/state/src/lib.rs index 678ee11eee..cb8c711e65 100644 --- a/codex-rs/state/src/lib.rs +++ b/codex-rs/state/src/lib.rs @@ -55,6 +55,7 @@ pub use runtime::GoalAccountingMode; pub use runtime::GoalAccountingOutcome; pub use runtime::GoalStore; pub use runtime::GoalUpdate; +pub use runtime::MemoryStore; pub use runtime::RemoteControlEnrollmentRecord; pub use runtime::RuntimeDbPath; pub use runtime::ThreadFilterOptions; @@ -62,6 +63,8 @@ pub use runtime::goals_db_filename; pub use runtime::goals_db_path; pub use runtime::logs_db_filename; pub use runtime::logs_db_path; +pub use runtime::memories_db_filename; +pub use runtime::memories_db_path; pub use runtime::runtime_db_paths; pub use runtime::sqlite_integrity_check; pub use runtime::state_db_filename; @@ -77,6 +80,7 @@ pub const SQLITE_HOME_ENV: &str = "CODEX_SQLITE_HOME"; pub const LOGS_DB_FILENAME: &str = "logs_2.sqlite"; pub const GOALS_DB_FILENAME: &str = "goals_1.sqlite"; +pub const MEMORIES_DB_FILENAME: &str = "memories_1.sqlite"; pub const STATE_DB_FILENAME: &str = "state_5.sqlite"; /// Errors encountered during DB operations. Tags: [stage] diff --git a/codex-rs/state/src/migrations.rs b/codex-rs/state/src/migrations.rs index 526e958b5a..641d584530 100644 --- a/codex-rs/state/src/migrations.rs +++ b/codex-rs/state/src/migrations.rs @@ -5,6 +5,7 @@ use sqlx::migrate::Migrator; pub(crate) static STATE_MIGRATOR: Migrator = sqlx::migrate!("./migrations"); pub(crate) static LOGS_MIGRATOR: Migrator = sqlx::migrate!("./logs_migrations"); pub(crate) static GOALS_MIGRATOR: Migrator = sqlx::migrate!("./goals_migrations"); +pub(crate) static MEMORIES_MIGRATOR: Migrator = sqlx::migrate!("./memory_migrations"); /// Allow an older Codex binary to open a database that has already been /// migrated by a newer binary running in parallel. @@ -32,3 +33,7 @@ pub(crate) fn runtime_logs_migrator() -> Migrator { pub(crate) fn runtime_goals_migrator() -> Migrator { runtime_migrator(&GOALS_MIGRATOR) } + +pub(crate) fn runtime_memories_migrator() -> Migrator { + runtime_migrator(&MEMORIES_MIGRATOR) +} diff --git a/codex-rs/state/src/model/memories.rs b/codex-rs/state/src/model/memories.rs index 9bb34405ae..650ede08da 100644 --- a/codex-rs/state/src/model/memories.rs +++ b/codex-rs/state/src/model/memories.rs @@ -1,9 +1,6 @@ -use anyhow::Result; use chrono::DateTime; use chrono::Utc; use codex_protocol::ThreadId; -use sqlx::Row; -use sqlx::sqlite::SqliteRow; use std::path::PathBuf; use super::ThreadMetadata; @@ -22,58 +19,6 @@ pub struct Stage1Output { pub generated_at: DateTime, } -#[derive(Debug)] -pub(crate) struct Stage1OutputRow { - thread_id: String, - rollout_path: String, - source_updated_at: i64, - raw_memory: String, - rollout_summary: String, - rollout_slug: Option, - cwd: String, - git_branch: Option, - generated_at: i64, -} - -impl Stage1OutputRow { - pub(crate) fn try_from_row(row: &SqliteRow) -> Result { - Ok(Self { - thread_id: row.try_get("thread_id")?, - rollout_path: row.try_get("rollout_path")?, - source_updated_at: row.try_get("source_updated_at")?, - raw_memory: row.try_get("raw_memory")?, - rollout_summary: row.try_get("rollout_summary")?, - rollout_slug: row.try_get("rollout_slug")?, - cwd: row.try_get("cwd")?, - git_branch: row.try_get("git_branch")?, - generated_at: row.try_get("generated_at")?, - }) - } -} - -impl TryFrom for Stage1Output { - type Error = anyhow::Error; - - fn try_from(row: Stage1OutputRow) -> std::result::Result { - Ok(Self { - thread_id: ThreadId::try_from(row.thread_id)?, - rollout_path: PathBuf::from(row.rollout_path), - source_updated_at: epoch_seconds_to_datetime(row.source_updated_at)?, - raw_memory: row.raw_memory, - rollout_summary: row.rollout_summary, - rollout_slug: row.rollout_slug, - cwd: PathBuf::from(row.cwd), - git_branch: row.git_branch, - generated_at: epoch_seconds_to_datetime(row.generated_at)?, - }) - } -} - -fn epoch_seconds_to_datetime(secs: i64) -> Result> { - DateTime::::from_timestamp(secs, 0) - .ok_or_else(|| anyhow::anyhow!("invalid unix timestamp: {secs}")) -} - /// Result of trying to claim a stage-1 memory extraction job. #[derive(Debug, Clone, PartialEq, Eq)] pub enum Stage1JobClaimOutcome { diff --git a/codex-rs/state/src/model/mod.rs b/codex-rs/state/src/model/mod.rs index a431bc64c0..2c725e50ab 100644 --- a/codex-rs/state/src/model/mod.rs +++ b/codex-rs/state/src/model/mod.rs @@ -37,7 +37,6 @@ pub use thread_metadata::ThreadsPage; pub(crate) use agent_job::AgentJobItemRow; pub(crate) use agent_job::AgentJobRow; -pub(crate) use memories::Stage1OutputRow; pub(crate) use thread_goal::ThreadGoalRow; pub(crate) use thread_metadata::ThreadRow; pub(crate) use thread_metadata::anchor_from_item; diff --git a/codex-rs/state/src/runtime.rs b/codex-rs/state/src/runtime.rs index 8c9c6d0267..ba41021290 100644 --- a/codex-rs/state/src/runtime.rs +++ b/codex-rs/state/src/runtime.rs @@ -10,6 +10,7 @@ use crate::LOGS_DB_FILENAME; use crate::LogEntry; use crate::LogQuery; use crate::LogRow; +use crate::MEMORIES_DB_FILENAME; use crate::STATE_DB_FILENAME; use crate::SortKey; use crate::ThreadMetadata; @@ -18,6 +19,7 @@ use crate::ThreadsPage; use crate::apply_rollout_item; use crate::migrations::runtime_goals_migrator; use crate::migrations::runtime_logs_migrator; +use crate::migrations::runtime_memories_migrator; use crate::migrations::runtime_state_migrator; use crate::model::AgentJobRow; use crate::model::ThreadRow; @@ -70,6 +72,7 @@ pub use goals::GoalAccountingMode; pub use goals::GoalAccountingOutcome; pub use goals::GoalStore; pub use goals::GoalUpdate; +pub use memories::MemoryStore; pub use remote_control::RemoteControlEnrollmentRecord; pub use threads::ThreadFilterOptions; @@ -121,7 +124,15 @@ const GOALS_DB: RuntimeDbSpec = RuntimeDbSpec { migrate_phase: "migrate_goals", }; -const RUNTIME_DBS: [RuntimeDbSpec; 3] = [STATE_DB, LOGS_DB, GOALS_DB]; +const MEMORIES_DB: RuntimeDbSpec = RuntimeDbSpec { + label: "memories DB", + filename: MEMORIES_DB_FILENAME, + kind: DbKind::Memories, + open_phase: "open_memories", + migrate_phase: "migrate_memories", +}; + +const RUNTIME_DBS: [RuntimeDbSpec; 4] = [STATE_DB, LOGS_DB, GOALS_DB, MEMORIES_DB]; #[derive(Clone, Debug, Eq, PartialEq)] pub struct RuntimeDbPath { @@ -136,6 +147,7 @@ pub struct StateRuntime { pool: Arc, logs_pool: Arc, thread_goals: GoalStore, + memories: MemoryStore, thread_updated_at_millis: Arc, } @@ -172,9 +184,11 @@ impl StateRuntime { let state_migrator = runtime_state_migrator(); let logs_migrator = runtime_logs_migrator(); let goals_migrator = runtime_goals_migrator(); + let memories_migrator = runtime_memories_migrator(); let state_path = STATE_DB.path(codex_home.as_path()); let logs_path = LOGS_DB.path(codex_home.as_path()); let goals_path = GOALS_DB.path(codex_home.as_path()); + let memories_path = MEMORIES_DB.path(codex_home.as_path()); let pool = match open_state_sqlite(&state_path, &state_migrator, telemetry_override).await { Ok(db) => Arc::new(db), Err(err) => { @@ -198,6 +212,22 @@ impl StateRuntime { return Err(err); } }; + let memories_pool = match open_memories_sqlite( + &memories_path, + &memories_migrator, + telemetry_override, + ) + .await + { + Ok(db) => Arc::new(db), + Err(err) => { + warn!( + "failed to open memories db at {}: {err}", + memories_path.display() + ); + return Err(err); + } + }; let started = Instant::now(); let backfill_state_result = ensure_backfill_state_row_in_pool(pool.as_ref()).await; crate::telemetry::record_init_result( @@ -225,6 +255,7 @@ impl StateRuntime { let thread_updated_at_millis = thread_updated_at_millis.unwrap_or(0); let runtime = Arc::new(Self { thread_goals: GoalStore::new(Arc::clone(&goals_pool)), + memories: MemoryStore::new(Arc::clone(&memories_pool), Arc::clone(&pool)), pool, logs_pool, codex_home, @@ -248,6 +279,28 @@ impl StateRuntime { pub fn thread_goals(&self) -> &GoalStore { &self.thread_goals } + + pub fn memories(&self) -> &MemoryStore { + &self.memories + } + + pub async fn clear_memory_data_in_sqlite_home(sqlite_home: &Path) -> anyhow::Result { + let memories_path = MEMORIES_DB.path(sqlite_home); + if !tokio::fs::try_exists(&memories_path).await? { + return Ok(false); + } + + let memories_migrator = runtime_memories_migrator(); + let pool = open_memories_sqlite( + &memories_path, + &memories_migrator, + /*telemetry_override*/ None, + ) + .await?; + memories::clear_memory_data_in_pool(&pool).await?; + pool.close().await; + Ok(true) + } } fn base_sqlite_options(path: &Path) -> SqliteConnectOptions { @@ -287,6 +340,14 @@ async fn open_goals_sqlite( open_sqlite(path, migrator, GOALS_DB, telemetry_override).await } +async fn open_memories_sqlite( + path: &Path, + migrator: &Migrator, + telemetry_override: Option<&dyn DbTelemetry>, +) -> anyhow::Result { + open_sqlite(path, migrator, MEMORIES_DB, telemetry_override).await +} + async fn open_sqlite( path: &Path, migrator: &Migrator, @@ -363,6 +424,14 @@ pub fn goals_db_path(codex_home: &Path) -> PathBuf { GOALS_DB.path(codex_home) } +pub fn memories_db_filename() -> String { + MEMORIES_DB.filename.to_string() +} + +pub fn memories_db_path(codex_home: &Path) -> PathBuf { + MEMORIES_DB.path(codex_home) +} + pub fn runtime_db_paths(codex_home: &Path) -> Vec { RUNTIME_DBS .iter() @@ -579,6 +648,8 @@ mod tests { "migrate_logs", "open_goals", "migrate_goals", + "open_memories", + "migrate_memories", "ensure_backfill_state", "post_init_query", ] diff --git a/codex-rs/state/src/runtime/memories.rs b/codex-rs/state/src/runtime/memories.rs index aeafe62a77..350d22cc6d 100644 --- a/codex-rs/state/src/runtime/memories.rs +++ b/codex-rs/state/src/runtime/memories.rs @@ -1,15 +1,14 @@ use super::threads::ThreadFilterOptions; use super::threads::push_thread_filters; -use super::threads::push_thread_order_and_limit; use super::*; use crate::SortDirection; use crate::model::Phase2JobClaimOutcome; use crate::model::Stage1JobClaim; use crate::model::Stage1JobClaimOutcome; use crate::model::Stage1Output; -use crate::model::Stage1OutputRow; use crate::model::Stage1StartupClaimParams; use crate::model::ThreadRow; +use chrono::DateTime; use chrono::Duration; use sqlx::Executor; use sqlx::QueryBuilder; @@ -20,39 +19,29 @@ const JOB_KIND_MEMORY_STAGE1: &str = "memory_stage1"; const JOB_KIND_MEMORY_CONSOLIDATE_GLOBAL: &str = "memory_consolidate_global"; const MEMORY_CONSOLIDATION_JOB_KEY: &str = "global"; const PHASE2_SUCCESS_COOLDOWN_SECONDS: i64 = 6 * 60 * 60; +const PHASE2_INPUT_SELECTION_PAGE_SIZE: usize = 512; const DEFAULT_RETRY_REMAINING: i64 = 3; -impl StateRuntime { +/// Store for generated memory state and memory extraction/consolidation jobs. +#[derive(Clone)] +pub struct MemoryStore { + pool: Arc, + state_pool: Arc, +} + +impl MemoryStore { + pub(crate) fn new(pool: Arc, state_pool: Arc) -> Self { + Self { pool, state_pool } + } + /// Deletes all persisted memory state in one transaction. /// /// This removes every `stage1_outputs` row and all `jobs` rows for the /// stage-1 (`memory_stage1`) and phase-2 (`memory_consolidate_global`) /// memory pipelines. pub async fn clear_memory_data(&self) -> anyhow::Result<()> { - let mut tx = self.pool.begin().await?; - - sqlx::query( - r#" -DELETE FROM stage1_outputs - "#, - ) - .execute(&mut *tx) - .await?; - - sqlx::query( - r#" -DELETE FROM jobs -WHERE kind = ? OR kind = ? - "#, - ) - .bind(JOB_KIND_MEMORY_STAGE1) - .bind(JOB_KIND_MEMORY_CONSOLIDATE_GLOBAL) - .execute(&mut *tx) - .await?; - - tx.commit().await?; - Ok(()) + clear_memory_data_in_pool(self.pool.as_ref()).await } /// Record usage for cited stage-1 outputs. @@ -92,6 +81,51 @@ WHERE thread_id = ? Ok(updated_rows) } + async fn stage1_source_needs_update( + &self, + thread_id: ThreadId, + source_updated_at: i64, + ) -> anyhow::Result { + let thread_id = thread_id.to_string(); + let existing_output = sqlx::query( + r#" +SELECT source_updated_at +FROM stage1_outputs +WHERE thread_id = ? + "#, + ) + .bind(thread_id.as_str()) + .fetch_optional(self.pool.as_ref()) + .await?; + if let Some(existing_output) = existing_output { + let existing_source_updated_at: i64 = existing_output.try_get("source_updated_at")?; + if existing_source_updated_at >= source_updated_at { + return Ok(false); + } + } + + let existing_job = sqlx::query( + r#" +SELECT last_success_watermark +FROM jobs +WHERE kind = ? AND job_key = ? + "#, + ) + .bind(JOB_KIND_MEMORY_STAGE1) + .bind(thread_id.as_str()) + .fetch_optional(self.pool.as_ref()) + .await?; + if let Some(existing_job) = existing_job { + let last_success_watermark = + existing_job.try_get::, _>("last_success_watermark")?; + if last_success_watermark.is_some_and(|watermark| watermark >= source_updated_at) { + return Ok(false); + } + } + + Ok(true) + } + /// Selects and claims stage-1 startup jobs for stale threads. /// /// Query behavior: @@ -100,8 +134,9 @@ WHERE thread_id = ? /// - excludes threads with `memory_mode != 'enabled'` /// - excludes the current thread id /// - keeps only threads whose millisecond `updated_at` is in the age window - /// - keeps only threads whose memory is stale compared to millisecond `updated_at` - /// - orders by `updated_at_ms DESC` and applies `scan_limit` + /// - checks memory staleness against the memories DB + /// - orders by `updated_at_ms DESC` and applies `scan_limit` to bound + /// state-DB work before probing the memories DB /// /// For each selected thread, this function calls [`Self::try_claim_stage1_job`] /// with `source_updated_at = thread.updated_at.timestamp()` and returns up to @@ -157,16 +192,6 @@ SELECT threads.git_branch, threads.git_origin_url FROM threads -LEFT JOIN stage1_outputs - ON stage1_outputs.thread_id = threads.id -LEFT JOIN jobs - ON jobs.kind = - "#, - ); - builder.push_bind(JOB_KIND_MEMORY_STAGE1); - builder.push( - r#" - AND jobs.job_key = threads.id "#, ); push_thread_filters( @@ -196,32 +221,29 @@ LEFT JOIN jobs .push("threads.updated_at_ms") .push(" <= ") .push_bind(idle_cutoff); - let updated_at = "threads.updated_at_ms"; - builder.push(" AND ((COALESCE(stage1_outputs.source_updated_at, -1) + 1) * 1000) <= "); - builder.push(updated_at); - builder.push(" AND ((COALESCE(jobs.last_success_watermark, -1) + 1) * 1000) <= "); - builder.push(updated_at); - push_thread_order_and_limit( - &mut builder, - SortKey::UpdatedAt, - SortDirection::Desc, - scan_limit, - ); + let scan_limit_i64 = i64::try_from(scan_limit).unwrap_or(i64::MAX); + builder.push(" ORDER BY threads.updated_at_ms DESC LIMIT "); + builder.push_bind(scan_limit_i64); let items = builder .build() - .fetch_all(self.pool.as_ref()) + .fetch_all(self.state_pool.as_ref()) .await? .into_iter() .map(|row| ThreadRow::try_from_row(&row).and_then(ThreadMetadata::try_from)) .collect::, _>>()?; let mut claimed = Vec::new(); - for item in items { if claimed.len() >= max_claimed { break; } + if !self + .stage1_source_needs_update(item.id, item.updated_at.timestamp()) + .await? + { + continue; + } if let Stage1JobClaimOutcome::Claimed { ownership_token } = self .try_claim_stage1_job( @@ -243,13 +265,64 @@ LEFT JOIN jobs Ok(claimed) } + pub(super) async fn delete_thread_memory(&self, thread_id: ThreadId) -> anyhow::Result<()> { + let now = Utc::now().timestamp(); + let thread_id = thread_id.to_string(); + let mut tx = self.pool.begin().await?; + + let existing_output = sqlx::query( + r#" +SELECT selected_for_phase2 +FROM stage1_outputs +WHERE thread_id = ? + "#, + ) + .bind(thread_id.as_str()) + .fetch_optional(&mut *tx) + .await?; + let was_selected_for_phase2 = existing_output + .map(|row| row.try_get::("selected_for_phase2")) + .transpose()? + .is_some_and(|selected| selected != 0); + + let deleted_rows = sqlx::query( + r#" +DELETE FROM stage1_outputs +WHERE thread_id = ? + "#, + ) + .bind(thread_id.as_str()) + .execute(&mut *tx) + .await? + .rows_affected(); + + sqlx::query( + r#" +DELETE FROM jobs +WHERE kind = ? AND job_key = ? + "#, + ) + .bind(JOB_KIND_MEMORY_STAGE1) + .bind(thread_id.as_str()) + .execute(&mut *tx) + .await?; + + if deleted_rows > 0 && was_selected_for_phase2 { + enqueue_global_consolidation_with_executor(&mut *tx, now).await?; + } + + tx.commit().await?; + Ok(()) + } + /// Lists the most recent non-empty stage-1 outputs for global consolidation. /// /// Query behavior: /// - filters out rows where both `raw_memory` and `rollout_summary` are blank - /// - joins `threads` to include thread `cwd`, `rollout_path`, and `git_branch` + /// - hydrates thread `cwd`, `rollout_path`, and `git_branch` from the state DB + /// - filters out missing or non-enabled threads /// - orders by `source_updated_at DESC, thread_id DESC` - /// - applies `LIMIT n` + /// - returns the first `n` visible outputs pub async fn list_stage1_outputs_for_global( &self, n: usize, @@ -262,30 +335,30 @@ LEFT JOIN jobs r#" SELECT so.thread_id, - COALESCE(t.rollout_path, '') AS rollout_path, so.source_updated_at, so.raw_memory, so.rollout_summary, so.rollout_slug, - so.generated_at, - COALESCE(t.cwd, '') AS cwd, - t.git_branch AS git_branch + so.generated_at FROM stage1_outputs AS so -LEFT JOIN threads AS t - ON t.id = so.thread_id -WHERE t.memory_mode = 'enabled' - AND (length(trim(so.raw_memory)) > 0 OR length(trim(so.rollout_summary)) > 0) +WHERE length(trim(so.raw_memory)) > 0 OR length(trim(so.rollout_summary)) > 0 ORDER BY so.source_updated_at DESC, so.thread_id DESC -LIMIT ? "#, ) - .bind(n as i64) .fetch_all(self.pool.as_ref()) .await?; - rows.into_iter() - .map(|row| Stage1OutputRow::try_from_row(&row).and_then(Stage1Output::try_from)) - .collect::, _>>() + let mut outputs = Vec::new(); + for row in rows { + if let Some(output) = self.stage1_output_from_row_if_thread_enabled(&row).await? { + outputs.push(output); + if outputs.len() >= n { + break; + } + } + } + + Ok(outputs) } /// Prunes stale stage-1 outputs while preserving the latest phase-2 @@ -356,64 +429,150 @@ WHERE thread_id IN ( } let cutoff = (Utc::now() - Duration::days(max_unused_days.max(0))).timestamp(); - let current_rows = sqlx::query( - r#" -SELECT - selected.thread_id, - selected.rollout_path, - selected.source_updated_at, - selected.raw_memory, - selected.rollout_summary, - selected.rollout_slug, - selected.generated_at, - selected.cwd, - selected.git_branch -FROM ( - SELECT - so.thread_id, - COALESCE(t.rollout_path, '') AS rollout_path, - so.source_updated_at, - so.raw_memory, - so.rollout_summary, - so.rollout_slug, - so.generated_at, - COALESCE(t.cwd, '') AS cwd, - t.git_branch AS git_branch - FROM stage1_outputs AS so - LEFT JOIN threads AS t - ON t.id = so.thread_id - WHERE t.memory_mode = 'enabled' - AND (length(trim(so.raw_memory)) > 0 OR length(trim(so.rollout_summary)) > 0) - AND ( - (so.last_usage IS NOT NULL AND so.last_usage >= ?) - OR (so.last_usage IS NULL AND so.source_updated_at >= ?) - ) - ORDER BY - COALESCE(so.usage_count, 0) DESC, - COALESCE(so.last_usage, so.source_updated_at) DESC, - so.source_updated_at DESC, - so.thread_id DESC - LIMIT ? -) AS selected -ORDER BY selected.thread_id ASC - "#, - ) - .bind(cutoff) - .bind(cutoff) - .bind(n as i64) - .fetch_all(self.pool.as_ref()) - .await?; + let page_size = n.clamp(1, PHASE2_INPUT_SELECTION_PAGE_SIZE); + let page_size_i64 = i64::try_from(page_size).unwrap_or(i64::MAX); + let mut offset = 0_i64; + let mut selected_keys = Vec::with_capacity(n); - let mut selected = Vec::with_capacity(current_rows.len()); - for row in current_rows { - selected.push(Stage1Output::try_from(Stage1OutputRow::try_from_row( - &row, - )?)?); + while selected_keys.len() < n { + let candidate_rows = sqlx::query( + r#" +SELECT + so.thread_id, + so.source_updated_at +FROM stage1_outputs AS so +WHERE (length(trim(so.raw_memory)) > 0 OR length(trim(so.rollout_summary)) > 0) + AND ( + (so.last_usage IS NOT NULL AND so.last_usage >= ?) + OR (so.last_usage IS NULL AND so.source_updated_at >= ?) + ) +ORDER BY + COALESCE(so.usage_count, 0) DESC, + COALESCE(so.last_usage, so.source_updated_at) DESC, + so.source_updated_at DESC, + so.thread_id DESC +LIMIT ? OFFSET ? + "#, + ) + .bind(cutoff) + .bind(cutoff) + .bind(page_size_i64) + .bind(offset) + .fetch_all(self.pool.as_ref()) + .await?; + + if candidate_rows.is_empty() { + break; + } + + let candidate_count = i64::try_from(candidate_rows.len()).unwrap_or(i64::MAX); + for row in candidate_rows { + let thread_id: String = row.try_get("thread_id")?; + let source_updated_at: i64 = row.try_get("source_updated_at")?; + if self + .enabled_thread_metadata(ThreadId::try_from(thread_id.as_str())?) + .await? + .is_some() + { + selected_keys.push((thread_id, source_updated_at)); + if selected_keys.len() >= n { + break; + } + } + } + + offset = offset.saturating_add(candidate_count); } + let mut selected = Vec::with_capacity(selected_keys.len()); + for (thread_id, source_updated_at) in selected_keys { + let Some(row) = sqlx::query( + r#" +SELECT + so.thread_id, + so.source_updated_at, + so.raw_memory, + so.rollout_summary, + so.rollout_slug, + so.generated_at +FROM stage1_outputs AS so +WHERE so.thread_id = ? AND so.source_updated_at = ? + "#, + ) + .bind(thread_id.as_str()) + .bind(source_updated_at) + .fetch_optional(self.pool.as_ref()) + .await? + else { + continue; + }; + if let Some(output) = self.stage1_output_from_row_if_thread_enabled(&row).await? { + selected.push(output); + } + } + + selected.sort_by(|a, b| a.thread_id.to_string().cmp(&b.thread_id.to_string())); + Ok(selected) } + async fn stage1_output_from_row_if_thread_enabled( + &self, + row: &sqlx::sqlite::SqliteRow, + ) -> anyhow::Result> { + let thread_id: String = row.try_get("thread_id")?; + let Some(thread) = self + .enabled_thread_metadata(ThreadId::try_from(thread_id.as_str())?) + .await? + else { + return Ok(None); + }; + Ok(Some(stage1_output_from_row_and_thread(row, thread)?)) + } + + async fn enabled_thread_metadata( + &self, + thread_id: ThreadId, + ) -> anyhow::Result> { + let row = sqlx::query( + r#" +SELECT + threads.id, + threads.rollout_path, + threads.created_at_ms AS created_at, + threads.updated_at_ms AS updated_at, + threads.source, + threads.thread_source, + threads.agent_nickname, + threads.agent_role, + threads.agent_path, + threads.model_provider, + threads.model, + threads.reasoning_effort, + threads.cwd, + threads.cli_version, + threads.title, + threads.preview, + threads.sandbox_policy, + threads.approval_mode, + threads.tokens_used, + threads.first_user_message, + threads.archived_at, + threads.git_sha, + threads.git_branch, + threads.git_origin_url +FROM threads +WHERE threads.id = ? AND threads.memory_mode = 'enabled' + "#, + ) + .bind(thread_id.to_string()) + .fetch_optional(self.state_pool.as_ref()) + .await?; + + row.map(|row| ThreadRow::try_from_row(&row).and_then(ThreadMetadata::try_from)) + .transpose() + } + /// Marks a thread as polluted and enqueues phase-2 forgetting when the /// thread participated in the last successful phase-2 baseline. pub async fn mark_thread_memory_mode_polluted( @@ -422,24 +581,6 @@ ORDER BY selected.thread_id ASC ) -> anyhow::Result { let now = Utc::now().timestamp(); let thread_id = thread_id.to_string(); - let mut tx = self.pool.begin().await?; - let rows_affected = sqlx::query( - r#" -UPDATE threads -SET memory_mode = 'polluted' -WHERE id = ? AND memory_mode != 'polluted' - "#, - ) - .bind(thread_id.as_str()) - .execute(&mut *tx) - .await? - .rows_affected(); - - if rows_affected == 0 { - tx.commit().await?; - return Ok(false); - } - let selected_for_phase2 = sqlx::query_scalar::<_, i64>( r#" SELECT selected_for_phase2 @@ -448,15 +589,26 @@ WHERE thread_id = ? "#, ) .bind(thread_id.as_str()) - .fetch_optional(&mut *tx) + .fetch_optional(self.pool.as_ref()) .await? .unwrap_or(0); + let rows_affected = sqlx::query( + r#" +UPDATE threads +SET memory_mode = 'polluted' +WHERE id = ? AND memory_mode != 'polluted' + "#, + ) + .bind(thread_id.as_str()) + .execute(self.state_pool.as_ref()) + .await? + .rows_affected(); + if selected_for_phase2 != 0 { - enqueue_global_consolidation_with_executor(&mut *tx, now).await?; + self.enqueue_global_consolidation(now).await?; } - tx.commit().await?; - Ok(true) + Ok(rows_affected > 0) } /// Attempts to claim a stage-1 job for a thread at `source_updated_at`. @@ -1221,6 +1373,58 @@ WHERE kind = ? AND job_key = ? Ok(rows_affected) } +pub(super) async fn clear_memory_data_in_pool(pool: &SqlitePool) -> anyhow::Result<()> { + let mut tx = pool.begin().await?; + + sqlx::query( + r#" +DELETE FROM stage1_outputs + "#, + ) + .execute(&mut *tx) + .await?; + + sqlx::query( + r#" +DELETE FROM jobs +WHERE kind = ? OR kind = ? + "#, + ) + .bind(JOB_KIND_MEMORY_STAGE1) + .bind(JOB_KIND_MEMORY_CONSOLIDATE_GLOBAL) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + Ok(()) +} + +fn stage1_output_from_row_and_thread( + row: &sqlx::sqlite::SqliteRow, + thread: ThreadMetadata, +) -> anyhow::Result { + let source_updated_at: i64 = row.try_get("source_updated_at")?; + let generated_at: i64 = row.try_get("generated_at")?; + let source_updated_at = datetime_from_epoch_seconds(source_updated_at)?; + let generated_at = datetime_from_epoch_seconds(generated_at)?; + Ok(Stage1Output { + thread_id: thread.id, + rollout_path: thread.rollout_path, + source_updated_at, + raw_memory: row.try_get("raw_memory")?, + rollout_summary: row.try_get("rollout_summary")?, + rollout_slug: row.try_get("rollout_slug")?, + cwd: thread.cwd, + git_branch: thread.git_branch, + generated_at, + }) +} + +fn datetime_from_epoch_seconds(secs: i64) -> anyhow::Result> { + DateTime::::from_timestamp(secs, 0) + .ok_or_else(|| anyhow::anyhow!("invalid unix timestamp: {secs}")) +} + async fn enqueue_global_consolidation_with_executor<'e, E>( executor: E, input_watermark: i64, @@ -1272,6 +1476,181 @@ ON CONFLICT(kind, job_key) DO UPDATE SET Ok(()) } +#[cfg(test)] +impl StateRuntime { + async fn clear_memory_data(&self) -> anyhow::Result<()> { + self.memories.clear_memory_data().await + } + + async fn record_stage1_output_usage(&self, thread_ids: &[ThreadId]) -> anyhow::Result { + self.memories.record_stage1_output_usage(thread_ids).await + } + + async fn claim_stage1_jobs_for_startup( + &self, + current_thread_id: ThreadId, + params: Stage1StartupClaimParams<'_>, + ) -> anyhow::Result> { + self.memories + .claim_stage1_jobs_for_startup(current_thread_id, params) + .await + } + + async fn list_stage1_outputs_for_global(&self, n: usize) -> anyhow::Result> { + self.memories.list_stage1_outputs_for_global(n).await + } + + async fn prune_stage1_outputs_for_retention( + &self, + max_unused_days: i64, + limit: usize, + ) -> anyhow::Result { + self.memories + .prune_stage1_outputs_for_retention(max_unused_days, limit) + .await + } + + async fn get_phase2_input_selection( + &self, + n: usize, + max_unused_days: i64, + ) -> anyhow::Result> { + self.memories + .get_phase2_input_selection(n, max_unused_days) + .await + } + + async fn mark_thread_memory_mode_polluted(&self, thread_id: ThreadId) -> anyhow::Result { + self.memories + .mark_thread_memory_mode_polluted(thread_id) + .await + } + + async fn try_claim_stage1_job( + &self, + thread_id: ThreadId, + worker_id: ThreadId, + source_updated_at: i64, + lease_seconds: i64, + max_running_jobs: usize, + ) -> anyhow::Result { + self.memories + .try_claim_stage1_job( + thread_id, + worker_id, + source_updated_at, + lease_seconds, + max_running_jobs, + ) + .await + } + + async fn mark_stage1_job_succeeded( + &self, + thread_id: ThreadId, + ownership_token: &str, + source_updated_at: i64, + raw_memory: &str, + rollout_summary: &str, + rollout_slug: Option<&str>, + ) -> anyhow::Result { + self.memories + .mark_stage1_job_succeeded( + thread_id, + ownership_token, + source_updated_at, + raw_memory, + rollout_summary, + rollout_slug, + ) + .await + } + + async fn mark_stage1_job_succeeded_no_output( + &self, + thread_id: ThreadId, + ownership_token: &str, + ) -> anyhow::Result { + self.memories + .mark_stage1_job_succeeded_no_output(thread_id, ownership_token) + .await + } + + async fn mark_stage1_job_failed( + &self, + thread_id: ThreadId, + ownership_token: &str, + failure_reason: &str, + retry_delay_seconds: i64, + ) -> anyhow::Result { + self.memories + .mark_stage1_job_failed( + thread_id, + ownership_token, + failure_reason, + retry_delay_seconds, + ) + .await + } + + async fn enqueue_global_consolidation(&self, input_watermark: i64) -> anyhow::Result<()> { + self.memories + .enqueue_global_consolidation(input_watermark) + .await + } + + async fn try_claim_global_phase2_job( + &self, + worker_id: ThreadId, + lease_seconds: i64, + ) -> anyhow::Result { + self.memories + .try_claim_global_phase2_job(worker_id, lease_seconds) + .await + } + + async fn mark_global_phase2_job_succeeded( + &self, + ownership_token: &str, + completed_watermark: i64, + selected_outputs: &[Stage1Output], + ) -> anyhow::Result { + self.memories + .mark_global_phase2_job_succeeded( + ownership_token, + completed_watermark, + selected_outputs, + ) + .await + } + + async fn mark_global_phase2_job_failed( + &self, + ownership_token: &str, + failure_reason: &str, + retry_delay_seconds: i64, + ) -> anyhow::Result { + self.memories + .mark_global_phase2_job_failed(ownership_token, failure_reason, retry_delay_seconds) + .await + } + + async fn mark_global_phase2_job_failed_if_unowned( + &self, + ownership_token: &str, + failure_reason: &str, + retry_delay_seconds: i64, + ) -> anyhow::Result { + self.memories + .mark_global_phase2_job_failed_if_unowned( + ownership_token, + failure_reason, + retry_delay_seconds, + ) + .await + } +} + #[cfg(test)] mod tests { use super::JOB_KIND_MEMORY_CONSOLIDATE_GLOBAL; @@ -1296,12 +1675,16 @@ mod tests { ThreadId::from_string(value).expect("thread id") } + fn memory_pool(runtime: &StateRuntime) -> &sqlx::SqlitePool { + runtime.memories().pool.as_ref() + } + async fn age_phase2_success_beyond_cooldown(runtime: &StateRuntime) { sqlx::query("UPDATE jobs SET finished_at = ? WHERE kind = ? AND job_key = ?") .bind(Utc::now().timestamp() - PHASE2_SUCCESS_COOLDOWN_SECONDS - 1) .bind(JOB_KIND_MEMORY_CONSOLIDATE_GLOBAL) .bind(MEMORY_CONSOLIDATION_JOB_KEY) - .execute(runtime.pool.as_ref()) + .execute(memory_pool(runtime)) .await .expect("age phase2 success beyond cooldown"); } @@ -1410,7 +1793,7 @@ mod tests { sqlx::query("UPDATE jobs SET lease_until = 0 WHERE kind = 'memory_stage1' AND job_key = ?") .bind(thread_id.to_string()) - .execute(runtime.pool.as_ref()) + .execute(memory_pool(&runtime)) .await .expect("force stale lease"); @@ -1656,7 +2039,7 @@ mod tests { } #[tokio::test] - async fn claim_stage1_jobs_prefilters_threads_with_up_to_date_memory() { + async fn claim_stage1_jobs_bounds_state_scan_before_memory_probes() { let codex_home = unique_temp_dir(); let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string()) .await @@ -1734,7 +2117,7 @@ mod tests { .expect("upsert stale thread"); let allowed_sources = vec!["cli".to_string()]; - let claims = runtime + let claims_with_one_scanned_thread = runtime .claim_stage1_jobs_for_startup( current_thread_id, Stage1StartupClaimParams { @@ -1748,6 +2131,22 @@ mod tests { ) .await .expect("claim stage1 startup jobs"); + assert_eq!(claims_with_one_scanned_thread.len(), 0); + + let claims = runtime + .claim_stage1_jobs_for_startup( + current_thread_id, + Stage1StartupClaimParams { + scan_limit: 2, + max_claimed: 1, + max_age_days: 30, + min_rollout_idle_hours: 12, + allowed_sources: allowed_sources.as_slice(), + lease_seconds: 3600, + }, + ) + .await + .expect("claim stage1 startup jobs with wider scan"); assert_eq!(claims.len(), 1); assert_eq!(claims[0].thread.id, stale_thread_id); @@ -1901,7 +2300,7 @@ mod tests { .expect("clear memory data"); let stage1_outputs_count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM stage1_outputs") - .fetch_one(runtime.pool.as_ref()) + .fetch_one(memory_pool(&runtime)) .await .expect("count stage1 outputs"); assert_eq!(stage1_outputs_count, 0); @@ -1910,7 +2309,7 @@ mod tests { sqlx::query_scalar("SELECT COUNT(*) FROM jobs WHERE kind = ? OR kind = ?") .bind(JOB_KIND_MEMORY_STAGE1) .bind(JOB_KIND_MEMORY_CONSOLIDATE_GLOBAL) - .fetch_one(runtime.pool.as_ref()) + .fetch_one(memory_pool(&runtime)) .await .expect("count memory jobs"); assert_eq!(memory_jobs_count, 0); @@ -2001,7 +2400,7 @@ INSERT INTO jobs ( .bind(lease_until) .bind(3) .bind(metadata.updated_at.timestamp()) - .execute(runtime.pool.as_ref()) + .execute(memory_pool(&runtime)) .await .expect("seed running stage1 job"); } @@ -2035,7 +2434,7 @@ WHERE kind = 'memory_stage1' "#, ) .bind(Utc::now().timestamp()) - .fetch_one(runtime.pool.as_ref()) + .fetch_one(memory_pool(&runtime)) .await .expect("count running stage1 jobs") .try_get::("count") @@ -2149,7 +2548,7 @@ WHERE kind = 'memory_stage1' } #[tokio::test] - async fn stage1_output_cascades_on_thread_delete() { + async fn delete_thread_removes_stage1_output_and_enqueues_phase2_when_selected() { let codex_home = unique_temp_dir(); let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string()) .await @@ -2192,29 +2591,84 @@ WHERE kind = 'memory_stage1' let count_before = sqlx::query("SELECT COUNT(*) AS count FROM stage1_outputs WHERE thread_id = ?") .bind(thread_id.to_string()) - .fetch_one(runtime.pool.as_ref()) + .fetch_one(memory_pool(&runtime)) .await .expect("count before delete") .try_get::("count") .expect("count value"); assert_eq!(count_before, 1); - sqlx::query("DELETE FROM threads WHERE id = ?") - .bind(thread_id.to_string()) - .execute(runtime.pool.as_ref()) + let phase2_claim = runtime + .try_claim_global_phase2_job(owner, /*lease_seconds*/ 3600) .await - .expect("delete thread"); + .expect("claim phase2"); + let (phase2_token, input_watermark) = match phase2_claim { + Phase2JobClaimOutcome::Claimed { + ownership_token, + input_watermark, + } => (ownership_token, input_watermark), + other => panic!("unexpected phase2 claim outcome: {other:?}"), + }; + let selected_outputs = runtime + .list_stage1_outputs_for_global(/*n*/ 10) + .await + .expect("list stage1 outputs"); + assert!( + runtime + .mark_global_phase2_job_succeeded( + phase2_token.as_str(), + input_watermark, + &selected_outputs, + ) + .await + .expect("mark phase2 succeeded"), + "phase2 success should mark selected stage1 output" + ); + + let before_delete = Utc::now().timestamp(); + assert_eq!( + runtime + .delete_thread(thread_id) + .await + .expect("delete thread"), + 1 + ); let count_after = sqlx::query("SELECT COUNT(*) AS count FROM stage1_outputs WHERE thread_id = ?") .bind(thread_id.to_string()) - .fetch_one(runtime.pool.as_ref()) + .fetch_one(memory_pool(&runtime)) .await .expect("count after delete") .try_get::("count") .expect("count value"); assert_eq!(count_after, 0); + let phase2_job = sqlx::query( + r#" +SELECT status, input_watermark +FROM jobs +WHERE kind = ? AND job_key = ? + "#, + ) + .bind(JOB_KIND_MEMORY_CONSOLIDATE_GLOBAL) + .bind(MEMORY_CONSOLIDATION_JOB_KEY) + .fetch_one(memory_pool(&runtime)) + .await + .expect("load phase2 job after delete"); + let status: String = phase2_job.try_get("status").expect("status"); + let input_watermark: i64 = phase2_job + .try_get("input_watermark") + .expect("input watermark"); + assert_eq!(status, "pending"); + assert!(input_watermark >= before_delete); + + let visible_outputs = runtime + .list_stage1_outputs_for_global(/*n*/ 10) + .await + .expect("list stage1 outputs after thread delete"); + assert_eq!(visible_outputs.len(), 0); + let _ = tokio::fs::remove_dir_all(codex_home).await; } @@ -2259,7 +2713,7 @@ WHERE kind = 'memory_stage1' let output_row_count = sqlx::query("SELECT COUNT(*) AS count FROM stage1_outputs WHERE thread_id = ?") .bind(thread_id.to_string()) - .fetch_one(runtime.pool.as_ref()) + .fetch_one(memory_pool(&runtime)) .await .expect("load stage1 output count") .try_get::("count") @@ -2280,7 +2734,7 @@ WHERE kind = 'memory_stage1' let global_job_row_count = sqlx::query("SELECT COUNT(*) AS count FROM jobs WHERE kind = ?") .bind("memory_consolidate_global") - .fetch_one(runtime.pool.as_ref()) + .fetch_one(memory_pool(&runtime)) .await .expect("load phase2 job row count") .try_get::("count") @@ -2384,7 +2838,7 @@ WHERE kind = 'memory_stage1' let output_row_count = sqlx::query("SELECT COUNT(*) AS count FROM stage1_outputs WHERE thread_id = ?") .bind(thread_id.to_string()) - .fetch_one(runtime.pool.as_ref()) + .fetch_one(memory_pool(&runtime)) .await .expect("load stage1 output count after delete") .try_get::("count") @@ -2495,7 +2949,7 @@ WHERE kind = 'memory_stage1' ) .bind("memory_stage1") .bind(thread_id.to_string()) - .fetch_one(runtime.pool.as_ref()) + .fetch_one(memory_pool(&runtime)) .await .expect("load stage1 job row after newer-source claim"); assert_eq!( @@ -2621,7 +3075,7 @@ WHERE kind = 'memory_stage1' sqlx::query("SELECT retry_remaining FROM jobs WHERE kind = ? AND job_key = ?") .bind("memory_consolidate_global") .bind("global") - .fetch_one(runtime.pool.as_ref()) + .fetch_one(memory_pool(&runtime)) .await .expect("load phase2 job row after retry exhaustion"); assert_eq!( @@ -2788,7 +3242,7 @@ VALUES (?, ?, ?, ?, ?) .bind("raw memory") .bind("summary") .bind(100_i64) - .execute(runtime.pool.as_ref()) + .execute(memory_pool(&runtime)) .await .expect("insert non-empty stage1 output"); sqlx::query( @@ -2802,7 +3256,7 @@ VALUES (?, ?, ?, ?, ?) .bind("") .bind("") .bind(101_i64) - .execute(runtime.pool.as_ref()) + .execute(memory_pool(&runtime)) .await .expect("insert empty stage1 output"); @@ -3183,6 +3637,101 @@ VALUES (?, ?, ?, ?, ?) let _ = tokio::fs::remove_dir_all(codex_home).await; } + #[tokio::test] + async fn mark_thread_memory_mode_polluted_enqueues_phase2_when_already_polluted() { + let codex_home = unique_temp_dir(); + let runtime = StateRuntime::init(codex_home.clone(), "test-provider".to_string()) + .await + .expect("initialize runtime"); + + let thread_id = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("thread id"); + let owner = ThreadId::from_string(&Uuid::new_v4().to_string()).expect("owner id"); + runtime + .upsert_thread(&test_thread_metadata( + &codex_home, + thread_id, + codex_home.join("workspace"), + )) + .await + .expect("upsert thread"); + + let claim = runtime + .try_claim_stage1_job( + thread_id, owner, /*source_updated_at*/ 100, /*lease_seconds*/ 3600, + /*max_running_jobs*/ 64, + ) + .await + .expect("claim stage1"); + let ownership_token = match claim { + Stage1JobClaimOutcome::Claimed { ownership_token } => ownership_token, + other => panic!("unexpected stage1 claim outcome: {other:?}"), + }; + assert!( + runtime + .mark_stage1_job_succeeded( + thread_id, + ownership_token.as_str(), + /*source_updated_at*/ 100, + "raw", + "summary", + /*rollout_slug*/ None, + ) + .await + .expect("mark stage1 succeeded"), + "stage1 success should persist output" + ); + + let phase2_claim = runtime + .try_claim_global_phase2_job(owner, /*lease_seconds*/ 3600) + .await + .expect("claim phase2"); + let (phase2_token, input_watermark) = match phase2_claim { + Phase2JobClaimOutcome::Claimed { + ownership_token, + input_watermark, + } => (ownership_token, input_watermark), + other => panic!("unexpected phase2 claim outcome: {other:?}"), + }; + let selected_outputs = runtime + .list_stage1_outputs_for_global(/*n*/ 10) + .await + .expect("list stage1 outputs"); + assert!( + runtime + .mark_global_phase2_job_succeeded( + phase2_token.as_str(), + input_watermark, + &selected_outputs, + ) + .await + .expect("mark phase2 success"), + "phase2 success should persist selected rows" + ); + + sqlx::query("UPDATE threads SET memory_mode = 'polluted' WHERE id = ?") + .bind(thread_id.to_string()) + .execute(runtime.pool.as_ref()) + .await + .expect("mark thread polluted before memory enqueue"); + + assert!( + !runtime + .mark_thread_memory_mode_polluted(thread_id) + .await + .expect("mark already polluted thread"), + "already polluted thread should not report a state transition" + ); + + age_phase2_success_beyond_cooldown(&runtime).await; + let next_claim = runtime + .try_claim_global_phase2_job(owner, /*lease_seconds*/ 3600) + .await + .expect("claim phase2 after already-polluted enqueue"); + assert!(matches!(next_claim, Phase2JobClaimOutcome::Claimed { .. })); + + let _ = tokio::fs::remove_dir_all(codex_home).await; + } + #[tokio::test] async fn get_phase2_input_selection_returns_regenerated_selected_rows() { let codex_home = unique_temp_dir(); @@ -3293,7 +3842,7 @@ VALUES (?, ?, ?, ?, ?) "SELECT selected_for_phase2, selected_for_phase2_source_updated_at FROM stage1_outputs WHERE thread_id = ?", ) .bind(thread_id.to_string()) - .fetch_one(runtime.pool.as_ref()) + .fetch_one(memory_pool(&runtime)) .await .expect("load selected_for_phase2"); assert_eq!(selected_for_phase2, 1); @@ -3586,7 +4135,7 @@ VALUES (?, ?, ?, ?, ?) "SELECT selected_for_phase2, selected_for_phase2_source_updated_at FROM stage1_outputs WHERE thread_id = ?", ) .bind(thread_id.to_string()) - .fetch_one(runtime.pool.as_ref()) + .fetch_one(memory_pool(&runtime)) .await .expect("load selected snapshot after phase2"); assert_eq!(selected_for_phase2, 1); @@ -3699,7 +4248,7 @@ VALUES (?, ?, ?, ?, ?) "SELECT selected_for_phase2, selected_for_phase2_source_updated_at FROM stage1_outputs WHERE thread_id = ?", ) .bind(thread_id.to_string()) - .fetch_one(runtime.pool.as_ref()) + .fetch_one(memory_pool(&runtime)) .await .expect("load selected_for_phase2"); assert_eq!(selected_for_phase2, 0); @@ -3803,13 +4352,13 @@ VALUES (?, ?, ?, ?, ?) let row_a = sqlx::query("SELECT usage_count, last_usage FROM stage1_outputs WHERE thread_id = ?") .bind(thread_a.to_string()) - .fetch_one(runtime.pool.as_ref()) + .fetch_one(memory_pool(&runtime)) .await .expect("load stage1 usage row a"); let row_b = sqlx::query("SELECT usage_count, last_usage FROM stage1_outputs WHERE thread_id = ?") .bind(thread_b.to_string()) - .fetch_one(runtime.pool.as_ref()) + .fetch_one(memory_pool(&runtime)) .await .expect("load stage1 usage row b"); @@ -3909,7 +4458,7 @@ VALUES (?, ?, ?, ?, ?) .bind(usage_count) .bind(last_usage.timestamp()) .bind(thread_id.to_string()) - .execute(runtime.pool.as_ref()) + .execute(memory_pool(&runtime)) .await .expect("update usage metadata"); } @@ -4005,7 +4554,7 @@ VALUES (?, ?, ?, ?, ?) .bind(usage_count) .bind(last_usage.map(|value| value.timestamp())) .bind(thread_id.to_string()) - .execute(runtime.pool.as_ref()) + .execute(memory_pool(&runtime)) .await .expect("update usage metadata"); } @@ -4090,13 +4639,13 @@ VALUES (?, ?, ?, ?, ?) sqlx::query("UPDATE stage1_outputs SET generated_at = ? WHERE thread_id = ?") .bind(300_i64) .bind(older_thread.to_string()) - .execute(runtime.pool.as_ref()) + .execute(memory_pool(&runtime)) .await .expect("update older generated_at"); sqlx::query("UPDATE stage1_outputs SET generated_at = ? WHERE thread_id = ?") .bind(150_i64) .bind(newer_thread.to_string()) - .execute(runtime.pool.as_ref()) + .execute(memory_pool(&runtime)) .await .expect("update newer generated_at"); @@ -4202,14 +4751,14 @@ VALUES (?, ?, ?, ?, ?) .bind(3_i64) .bind(now - Duration::days(40).num_seconds()) .bind(stale_used.to_string()) - .execute(runtime.pool.as_ref()) + .execute(memory_pool(&runtime)) .await .expect("set stale used metadata"); sqlx::query( "UPDATE stage1_outputs SET selected_for_phase2 = 1, selected_for_phase2_source_updated_at = source_updated_at WHERE thread_id = ?", ) .bind(stale_selected.to_string()) - .execute(runtime.pool.as_ref()) + .execute(memory_pool(&runtime)) .await .expect("mark selected for phase2"); sqlx::query( @@ -4218,13 +4767,13 @@ VALUES (?, ?, ?, ?, ?) .bind(8_i64) .bind(now - Duration::days(2).num_seconds()) .bind(fresh_used.to_string()) - .execute(runtime.pool.as_ref()) + .execute(memory_pool(&runtime)) .await .expect("set fresh used metadata"); let before_jobs_count = sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM jobs WHERE kind = 'memory_stage1'") - .fetch_one(runtime.pool.as_ref()) + .fetch_one(memory_pool(&runtime)) .await .expect("count stage1 jobs before prune"); @@ -4237,7 +4786,7 @@ VALUES (?, ?, ?, ?, ?) let remaining = sqlx::query_scalar::<_, String>( "SELECT thread_id FROM stage1_outputs ORDER BY thread_id", ) - .fetch_all(runtime.pool.as_ref()) + .fetch_all(memory_pool(&runtime)) .await .expect("load remaining stage1 outputs"); let mut expected_remaining = vec![fresh_used.to_string(), stale_selected.to_string()]; @@ -4246,7 +4795,7 @@ VALUES (?, ?, ?, ?, ?) let after_jobs_count = sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM jobs WHERE kind = 'memory_stage1'") - .fetch_one(runtime.pool.as_ref()) + .fetch_one(memory_pool(&runtime)) .await .expect("count stage1 jobs after prune"); assert_eq!(after_jobs_count, before_jobs_count); @@ -4324,7 +4873,7 @@ VALUES (?, ?, ?, ?, ?) assert_eq!(pruned, 2); let remaining_count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM stage1_outputs") - .fetch_one(runtime.pool.as_ref()) + .fetch_one(memory_pool(&runtime)) .await .expect("count remaining stage1 outputs"); assert_eq!(remaining_count, 1); @@ -4540,7 +5089,7 @@ VALUES (?, ?, ?, ?, ?) .bind(Utc::now().timestamp() - 1) .bind("memory_consolidate_global") .bind("global") - .execute(runtime.pool.as_ref()) + .execute(memory_pool(&runtime)) .await .expect("expire global consolidation lease"); @@ -4676,7 +5225,7 @@ VALUES (?, ?, ?, ?, ?) sqlx::query("UPDATE jobs SET ownership_token = NULL WHERE kind = ? AND job_key = ?") .bind("memory_consolidate_global") .bind("global") - .execute(runtime.pool.as_ref()) + .execute(memory_pool(&runtime)) .await .expect("clear ownership token"); diff --git a/codex-rs/state/src/runtime/threads.rs b/codex-rs/state/src/runtime/threads.rs index c7030d7e44..78180e7c7d 100644 --- a/codex-rs/state/src/runtime/threads.rs +++ b/codex-rs/state/src/runtime/threads.rs @@ -971,6 +971,7 @@ ON CONFLICT(thread_id, position) DO NOTHING .execute(self.pool.as_ref()) .await?; let rows_affected = result.rows_affected(); + self.memories.delete_thread_memory(thread_id).await?; if rows_affected > 0 { self.thread_goals.delete_thread_goal(thread_id).await?; } diff --git a/codex-rs/state/src/telemetry.rs b/codex-rs/state/src/telemetry.rs index da2b7de7a8..1a9e4f9951 100644 --- a/codex-rs/state/src/telemetry.rs +++ b/codex-rs/state/src/telemetry.rs @@ -40,6 +40,7 @@ pub(crate) enum DbKind { State, Logs, Goals, + Memories, } impl DbKind { @@ -48,6 +49,7 @@ impl DbKind { Self::State => "state", Self::Logs => "logs", Self::Goals => "goals", + Self::Memories => "memories", } } } From a22706dfae2cedfe6056be800ff3d531468e999a Mon Sep 17 00:00:00 2001 From: sayan-oai Date: Tue, 26 May 2026 11:12:24 -0700 Subject: [PATCH 06/16] standalone websearch extension (#23823) ## Summary Add the extension-backed standalone `web.run` tool so Codex can call the standalone search endpoint through the `codex-api` search client and return its encrypted output to Responses. - gate the new tool behind `standalone_web_search` - install the extension in the app-server thread registry and hide hosted `web_search` when standalone search is enabled for OpenAI providers so the two paths stay mutually exclusive - build search context from persisted history using a small tail heuristic: previous user message, assistant text between the last two user turns capped at about 1k tokens, and current user message ## Test Plan - `cargo test -p codex-web-search-extension` - `cargo test -p codex-api` - `cargo test -p codex-core hosted_tools_follow_provider_auth_model_and_config_gates` --- codex-rs/Cargo.lock | 22 ++ codex-rs/Cargo.toml | 2 + codex-rs/app-server/Cargo.toml | 1 + codex-rs/app-server/src/extensions.rs | 3 + codex-rs/app-server/src/mcp_refresh.rs | 5 +- codex-rs/app-server/src/message_processor.rs | 1 + codex-rs/app-server/tests/suite/v2/mod.rs | 1 + .../app-server/tests/suite/v2/web_search.rs | 216 ++++++++++++++++++ codex-rs/codex-api/Cargo.toml | 1 + codex-rs/codex-api/src/search.rs | 69 ++++-- codex-rs/core/config.schema.json | 6 + codex-rs/core/src/tools/spec_plan.rs | 18 +- codex-rs/core/src/tools/spec_plan_tests.rs | 57 ++++- codex-rs/ext/web-search/BUILD.bazel | 9 + codex-rs/ext/web-search/Cargo.toml | 31 +++ codex-rs/ext/web-search/src/extension.rs | 173 ++++++++++++++ codex-rs/ext/web-search/src/history.rs | 170 ++++++++++++++ codex-rs/ext/web-search/src/lib.rs | 7 + codex-rs/ext/web-search/src/output.rs | 72 ++++++ codex-rs/ext/web-search/src/schema.rs | 36 +++ codex-rs/ext/web-search/src/tool.rs | 113 +++++++++ .../ext/web-search/web_run_description.md | 80 +++++++ codex-rs/features/src/lib.rs | 8 + codex-rs/protocol/src/models.rs | 7 + codex-rs/tools/src/lib.rs | 3 + codex-rs/tools/src/response_history.rs | 149 ++++++++++++ 26 files changed, 1238 insertions(+), 22 deletions(-) create mode 100644 codex-rs/app-server/tests/suite/v2/web_search.rs create mode 100644 codex-rs/ext/web-search/BUILD.bazel create mode 100644 codex-rs/ext/web-search/Cargo.toml create mode 100644 codex-rs/ext/web-search/src/extension.rs create mode 100644 codex-rs/ext/web-search/src/history.rs create mode 100644 codex-rs/ext/web-search/src/lib.rs create mode 100644 codex-rs/ext/web-search/src/output.rs create mode 100644 codex-rs/ext/web-search/src/schema.rs create mode 100644 codex-rs/ext/web-search/src/tool.rs create mode 100644 codex-rs/ext/web-search/web_run_description.md create mode 100644 codex-rs/tools/src/response_history.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 78acd3478d..a1884ba9e2 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1859,6 +1859,7 @@ dependencies = [ "pretty_assertions", "regex-lite", "reqwest", + "schemars 0.8.22", "serde", "serde_json", "tempfile", @@ -1927,6 +1928,7 @@ dependencies = [ "codex-utils-cli", "codex-utils-json-to-toml", "codex-utils-pty", + "codex-web-search-extension", "core_test_support", "flate2", "futures", @@ -4117,6 +4119,26 @@ dependencies = [ "v8", ] +[[package]] +name = "codex-web-search-extension" +version = "0.0.0" +dependencies = [ + "async-trait", + "codex-api", + "codex-core", + "codex-extension-api", + "codex-features", + "codex-login", + "codex-model-provider", + "codex-model-provider-info", + "codex-protocol", + "codex-tools", + "http 1.4.0", + "pretty_assertions", + "schemars 0.8.22", + "serde_json", +] + [[package]] name = "codex-windows-sandbox" version = "0.0.0" diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index 830e244082..6c9737e4ad 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -48,6 +48,7 @@ members = [ "ext/goal", "ext/guardian", "ext/memories", + "ext/web-search", "external-agent-migration", "external-agent-sessions", "keyring-store", @@ -180,6 +181,7 @@ codex-lmstudio = { path = "lmstudio" } codex-login = { path = "login" } codex-message-history = { path = "message-history" } codex-memories-extension = { path = "ext/memories" } +codex-web-search-extension = { path = "ext/web-search" } codex-memories-read = { path = "memories/read" } codex-memories-write = { path = "memories/write" } codex-mcp = { path = "codex-mcp" } diff --git a/codex-rs/app-server/Cargo.toml b/codex-rs/app-server/Cargo.toml index 95baac4e9e..17ecd7dc3f 100644 --- a/codex-rs/app-server/Cargo.toml +++ b/codex-rs/app-server/Cargo.toml @@ -55,6 +55,7 @@ codex-file-search = { workspace = true } codex-chatgpt = { workspace = true } codex-login = { workspace = true } codex-memories-extension = { workspace = true } +codex-web-search-extension = { workspace = true } codex-memories-write = { workspace = true } codex-mcp = { workspace = true } codex-model-provider = { workspace = true } diff --git a/codex-rs/app-server/src/extensions.rs b/codex-rs/app-server/src/extensions.rs index 9232cb758b..1246da7b37 100644 --- a/codex-rs/app-server/src/extensions.rs +++ b/codex-rs/app-server/src/extensions.rs @@ -12,6 +12,7 @@ use codex_extension_api::AgentSpawner; use codex_extension_api::ExtensionEventSink; use codex_extension_api::ExtensionRegistry; use codex_extension_api::ExtensionRegistryBuilder; +use codex_login::AuthManager; use codex_protocol::ThreadId; use codex_protocol::error::CodexErr; use codex_protocol::protocol::Event; @@ -22,6 +23,7 @@ use crate::outgoing_message::OutgoingMessageSender; pub(crate) fn thread_extensions( guardian_agent_spawner: S, event_sink: Arc, + auth_manager: Arc, ) -> Arc> where S: AgentSpawner + 'static, @@ -29,6 +31,7 @@ where let mut builder = ExtensionRegistryBuilder::::with_event_sink(event_sink); codex_guardian::install(&mut builder, guardian_agent_spawner); codex_memories_extension::install(&mut builder, codex_otel::global()); + codex_web_search_extension::install(&mut builder, auth_manager); Arc::new(builder.build()) } diff --git a/codex-rs/app-server/src/mcp_refresh.rs b/codex-rs/app-server/src/mcp_refresh.rs index d65b1d83e0..4ddfa34735 100644 --- a/codex-rs/app-server/src/mcp_refresh.rs +++ b/codex-rs/app-server/src/mcp_refresh.rs @@ -184,15 +184,16 @@ mod tests { let thread_manager = Arc::new_cyclic(|thread_manager| { ThreadManager::new( &good_config, - auth_manager, + auth_manager.clone(), SessionSource::Exec, Arc::new(EnvironmentManager::default_for_tests()), thread_extensions( guardian_agent_spawner(thread_manager.clone()), Arc::new(NoopExtensionEventSink), + auth_manager.clone(), ), /*analytics_events_client*/ None, - thread_store, + Arc::clone(&thread_store), Some(state_db.clone()), "11111111-1111-4111-8111-111111111111".to_string(), /*attestation_provider*/ None, diff --git a/codex-rs/app-server/src/message_processor.rs b/codex-rs/app-server/src/message_processor.rs index 2d0e8d085f..33b9c56d4a 100644 --- a/codex-rs/app-server/src/message_processor.rs +++ b/codex-rs/app-server/src/message_processor.rs @@ -314,6 +314,7 @@ impl MessageProcessor { thread_extensions( guardian_agent_spawner(thread_manager.clone()), app_server_extension_event_sink(outgoing.clone()), + auth_manager.clone(), ), Some(analytics_events_client.clone()), Arc::clone(&thread_store), diff --git a/codex-rs/app-server/tests/suite/v2/mod.rs b/codex-rs/app-server/tests/suite/v2/mod.rs index 92e0f126f3..1605983705 100644 --- a/codex-rs/app-server/tests/suite/v2/mod.rs +++ b/codex-rs/app-server/tests/suite/v2/mod.rs @@ -68,4 +68,5 @@ mod turn_interrupt; mod turn_start; mod turn_start_zsh_fork; mod turn_steer; +mod web_search; mod windows_sandbox_setup; diff --git a/codex-rs/app-server/tests/suite/v2/web_search.rs b/codex-rs/app-server/tests/suite/v2/web_search.rs new file mode 100644 index 0000000000..52c327a93f --- /dev/null +++ b/codex-rs/app-server/tests/suite/v2/web_search.rs @@ -0,0 +1,216 @@ +use std::path::Path; +use std::time::Duration; + +use anyhow::Context; +use anyhow::Result; +use app_test_support::ChatGptAuthFixture; +use app_test_support::McpProcess; +use app_test_support::to_response; +use app_test_support::write_chatgpt_auth; +use codex_app_server_protocol::JSONRPCResponse; +use codex_app_server_protocol::RequestId; +use codex_app_server_protocol::ThreadStartParams; +use codex_app_server_protocol::ThreadStartResponse; +use codex_app_server_protocol::TurnStartParams; +use codex_app_server_protocol::TurnStartResponse; +use codex_app_server_protocol::UserInput as V2UserInput; +use codex_config::types::AuthCredentialsStoreMode; +use core_test_support::responses; +use pretty_assertions::assert_eq; +use serde_json::Value; +use serde_json::json; +use tempfile::TempDir; +use tokio::time::timeout; +use wiremock::Mock; +use wiremock::MockServer; +use wiremock::ResponseTemplate; +use wiremock::matchers::method; +use wiremock::matchers::path; + +// macOS and Windows Bazel CI can spend tens of seconds starting app-server +// subprocesses or processing test RPCs under load. +#[cfg(any(target_os = "macos", windows))] +const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(60); +#[cfg(not(any(target_os = "macos", windows)))] +const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(10); + +#[tokio::test] +async fn standalone_web_search_round_trips_encrypted_output() -> Result<()> { + let call_id = "web-run-1"; + let server = responses::start_mock_server().await; + mount_search_response(&server).await; + + let response_mock = responses::mount_sse_sequence( + &server, + vec![ + responses::sse(vec![ + responses::ev_response_created("resp-1"), + responses::ev_function_call_with_namespace( + call_id, + "web", + "run", + &json!({ + "search_query": [{"q": "standalone web search"}], + }) + .to_string(), + ), + responses::ev_completed("resp-1"), + ]), + responses::sse(vec![ + responses::ev_assistant_message("msg-1", "Done"), + responses::ev_completed("resp-2"), + ]), + ], + ) + .await; + + let codex_home = TempDir::new()?; + create_config_toml(codex_home.path(), &server.uri())?; + write_chatgpt_auth( + codex_home.path(), + ChatGptAuthFixture::new("access-chatgpt"), + AuthCredentialsStoreMode::File, + )?; + + let mut mcp = McpProcess::new_with_env(codex_home.path(), &[("OPENAI_API_KEY", None)]).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let thread_req = mcp + .send_thread_start_request(ThreadStartParams::default()) + .await?; + let thread_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(thread_req)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response::(thread_resp)?; + + let turn_req = mcp + .send_turn_start_request(TurnStartParams { + thread_id: thread.id, + input: vec![V2UserInput::Text { + text: "Search the web".to_string(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + let turn_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(turn_req)), + ) + .await??; + let _turn: TurnStartResponse = to_response::(turn_resp)?; + + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("turn/completed"), + ) + .await??; + + let requests = response_mock.requests(); + assert_eq!(requests.len(), 2); + + let first_response = requests[0].body_json(); + assert!( + requests[0].tool_by_name("web", "run").is_some(), + "web.run should be sent to the model" + ); + assert!( + !has_hosted_web_search(&first_response), + "standalone web search should replace hosted web search" + ); + + let search_body = search_request_body(&server).await?; + assert_eq!( + search_body["commands"], + json!({ + "search_query": [{"q": "standalone web search"}], + }) + ); + assert_eq!( + search_body["input"] + .as_array() + .context("search input should be an array")? + .last(), + Some(&json!({ + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "Search the web"}], + })) + ); + + assert_eq!( + requests[1].function_call_output(call_id), + json!({ + "type": "function_call_output", + "call_id": call_id, + "output": [{ + "type": "encrypted_content", + "encrypted_content": "ciphertext", + }], + }) + ); + + Ok(()) +} + +async fn mount_search_response(server: &MockServer) { + Mock::given(method("POST")) + .and(path("/api/codex/alpha/search")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "encrypted_output": "ciphertext", + }))) + .expect(1) + .mount(server) + .await; +} + +fn has_hosted_web_search(body: &Value) -> bool { + body.get("tools") + .and_then(Value::as_array) + .is_some_and(|tools| { + tools + .iter() + .any(|tool| tool.get("type").and_then(Value::as_str) == Some("web_search")) + }) +} + +async fn search_request_body(server: &MockServer) -> Result { + server + .received_requests() + .await + .context("failed to fetch received requests")? + .into_iter() + .find(|request| request.url.path() == "/api/codex/alpha/search") + .context("expected standalone search request")? + .body_json() + .context("search request body should be JSON") +} + +fn create_config_toml(codex_home: &Path, server_uri: &str) -> std::io::Result<()> { + std::fs::write( + codex_home.join("config.toml"), + format!( + r#" +model = "mock-model" +approval_policy = "never" +sandbox_mode = "read-only" +model_provider = "openai-custom" +chatgpt_base_url = "{server_uri}" + +[features] +standalone_web_search = true + +[model_providers.openai-custom] +name = "OpenAI" +base_url = "{server_uri}/api/codex" +wire_api = "responses" +request_max_retries = 0 +stream_max_retries = 0 +supports_websockets = false +requires_openai_auth = true +"# + ), + ) +} diff --git a/codex-rs/codex-api/Cargo.toml b/codex-rs/codex-api/Cargo.toml index 08f70cf33c..07d855725a 100644 --- a/codex-rs/codex-api/Cargo.toml +++ b/codex-rs/codex-api/Cargo.toml @@ -16,6 +16,7 @@ codex-utils-rustls-provider = { workspace = true } futures = { workspace = true } http = { workspace = true } reqwest = { workspace = true, features = ["json", "stream"] } +schemars = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } thiserror = { workspace = true } diff --git a/codex-rs/codex-api/src/search.rs b/codex-rs/codex-api/src/search.rs index b841d06a30..8328415f83 100644 --- a/codex-rs/codex-api/src/search.rs +++ b/codex-rs/codex-api/src/search.rs @@ -1,5 +1,6 @@ use crate::common::Reasoning; use codex_protocol::models::ResponseItem; +use schemars::JsonSchema; use serde::Deserialize; use serde::Serialize; @@ -27,75 +28,100 @@ pub enum SearchInput { Items(Vec), } -#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq)] +#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, JsonSchema)] pub struct SearchCommands { + /// Query the internet search engine for a given list of queries. #[serde(skip_serializing_if = "Option::is_none")] pub search_query: Option>, + /// Query the image search engine for a given list of queries. #[serde(skip_serializing_if = "Option::is_none")] pub image_query: Option>, + /// Open pages by reference id or URL. #[serde(skip_serializing_if = "Option::is_none")] pub open: Option>, + /// Open links from previously opened pages. #[serde(skip_serializing_if = "Option::is_none")] pub click: Option>, + /// Find text patterns in pages. #[serde(skip_serializing_if = "Option::is_none")] pub find: Option>, + /// Take screenshots of PDF pages. #[serde(skip_serializing_if = "Option::is_none")] pub screenshot: Option>, + /// Look up prices for the given stock symbols. #[serde(skip_serializing_if = "Option::is_none")] pub finance: Option>, + /// Look up weather forecasts. #[serde(skip_serializing_if = "Option::is_none")] pub weather: Option>, + /// Look up sports schedules and standings. #[serde(skip_serializing_if = "Option::is_none")] pub sports: Option>, + /// Get time for the given UTC offsets. #[serde(skip_serializing_if = "Option::is_none")] pub time: Option>, + /// Set the length of the response to be returned. #[serde(skip_serializing_if = "Option::is_none")] pub response_length: Option, } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, JsonSchema)] pub struct SearchQuery { + /// Search query. pub q: String, + /// Whether to filter by recency, as a number of recent days. #[serde(skip_serializing_if = "Option::is_none")] pub recency: Option, + /// Whether to filter by a specific list of domains. #[serde(skip_serializing_if = "Option::is_none")] pub domains: Option>, } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, JsonSchema)] pub struct OpenOperation { + /// Reference id or URL to open. pub ref_id: String, + /// Line number to position the page at. #[serde(skip_serializing_if = "Option::is_none")] pub lineno: Option, } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] pub struct ClickOperation { + /// Reference id containing the numbered link. pub ref_id: String, + /// Numbered link id to open. pub id: u64, } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] pub struct FindOperation { + /// Reference id or URL to search within. pub ref_id: String, + /// Text pattern to find. pub pattern: String, } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] pub struct ScreenshotOperation { + /// Reference id or URL to screenshot. pub ref_id: String, + /// Zero-indexed PDF page number. pub pageno: u64, } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, JsonSchema)] pub struct FinanceOperation { + /// Ticker symbol to look up. pub ticker: String, + /// Asset type to look up. pub r#type: FinanceAssetType, + /// ISO 3166-1 alpha-3 country code, "OTC", or "" for cryptocurrency. #[serde(skip_serializing_if = "Option::is_none")] pub market: Option, } -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] #[serde(rename_all = "lowercase")] pub enum FinanceAssetType { Equity, @@ -104,49 +130,61 @@ pub enum FinanceAssetType { Index, } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, JsonSchema)] pub struct WeatherOperation { + /// Location in "Country, Area, City" format. pub location: String, + /// Start date in YYYY-MM-DD format. Defaults to today. #[serde(skip_serializing_if = "Option::is_none")] pub start: Option, + /// Number of days to return. Defaults to 7. #[serde(skip_serializing_if = "Option::is_none")] pub duration: Option, } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, JsonSchema)] pub struct SportsOperation { + /// Tool name for sports requests. #[serde(skip_serializing_if = "Option::is_none")] pub tool: Option, + /// Sports function to call. pub r#fn: SportsFunction, + /// League to look up. pub league: SportsLeague, + /// Team to look up, using the common 3 or 4 letter alias used in broadcasts. #[serde(skip_serializing_if = "Option::is_none")] pub team: Option, + /// Opponent to use with `team` when narrowing the lookup. #[serde(skip_serializing_if = "Option::is_none")] pub opponent: Option, + /// Start date in YYYY-MM-DD format. #[serde(skip_serializing_if = "Option::is_none")] pub date_from: Option, + /// End date in YYYY-MM-DD format. #[serde(skip_serializing_if = "Option::is_none")] pub date_to: Option, + /// Number of games to return. #[serde(skip_serializing_if = "Option::is_none")] pub num_games: Option, + /// Locale for the lookup. #[serde(skip_serializing_if = "Option::is_none")] pub locale: Option, } -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] #[serde(rename_all = "lowercase")] pub enum SportsToolName { Sports, } -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] #[serde(rename_all = "lowercase")] pub enum SportsFunction { Schedule, Standings, } -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] #[serde(rename_all = "lowercase")] pub enum SportsLeague { Nba, @@ -160,12 +198,13 @@ pub enum SportsLeague { Ipl, } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] pub struct TimeOperation { + /// UTC offset formatted like "+03:00". pub utc_offset: String, } -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] #[serde(rename_all = "lowercase")] pub enum SearchResponseLength { Short, diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.json index f6e15c8063..6af30b4071 100644 --- a/codex-rs/core/config.schema.json +++ b/codex-rs/core/config.schema.json @@ -575,6 +575,9 @@ "sqlite": { "type": "boolean" }, + "standalone_web_search": { + "type": "boolean" + }, "steer": { "type": "boolean" }, @@ -4505,6 +4508,9 @@ "sqlite": { "type": "boolean" }, + "standalone_web_search": { + "type": "boolean" + }, "steer": { "type": "boolean" }, diff --git a/codex-rs/core/src/tools/spec_plan.rs b/codex-rs/core/src/tools/spec_plan.rs index c3656f2434..2610bee95c 100644 --- a/codex-rs/core/src/tools/spec_plan.rs +++ b/codex-rs/core/src/tools/spec_plan.rs @@ -238,11 +238,12 @@ fn spec_for_model_request( } } -pub(crate) fn hosted_model_tool_specs(turn_context: &TurnContext) -> Vec { +fn hosted_model_tool_specs(context: &CoreToolPlanContext<'_>) -> Vec { + let turn_context = context.turn_context; let mut specs = Vec::new(); let provider_capabilities = turn_context.provider.capabilities(); - let web_search_mode = provider_capabilities - .web_search + let web_search_mode = (!standalone_web_run_available(context.extension_tool_executors) + && provider_capabilities.web_search) .then_some(turn_context.config.web_search_mode.value()); let web_search_config = if provider_capabilities.web_search { turn_context.config.web_search_config.as_ref() @@ -504,11 +505,20 @@ fn add_tool_sources(context: &CoreToolPlanContext<'_>, planned_tools: &mut Plann add_mcp_runtime_tools(context, planned_tools); add_dynamic_tools(context, planned_tools); add_extension_tools(context, planned_tools); - for spec in hosted_model_tool_specs(context.turn_context) { + for spec in hosted_model_tool_specs(context) { planned_tools.add_hosted_spec(spec); } } +fn standalone_web_run_available( + extension_tools: &[Arc>], +) -> bool { + let web_run = ToolName::namespaced("web", "run"); + extension_tools + .iter() + .any(|executor| executor.tool_name() == web_run) +} + fn add_shell_tools(context: &CoreToolPlanContext<'_>, planned_tools: &mut PlannedTools) { let turn_context = context.turn_context; let features = turn_context.features.get(); diff --git a/codex-rs/core/src/tools/spec_plan_tests.rs b/codex-rs/core/src/tools/spec_plan_tests.rs index 59d6954890..2eb6eea5d2 100644 --- a/codex-rs/core/src/tools/spec_plan_tests.rs +++ b/codex-rs/core/src/tools/spec_plan_tests.rs @@ -20,8 +20,11 @@ use codex_tools::DiscoverablePluginInfo; use codex_tools::DiscoverableTool; use codex_tools::ResponsesApiNamespaceTool; use codex_tools::ResponsesApiTool; +use codex_tools::ToolCall as ExtensionToolCall; +use codex_tools::ToolExecutor; use codex_tools::ToolExposure; use codex_tools::ToolName; +use codex_tools::ToolOutput; use codex_tools::ToolSpec; use pretty_assertions::assert_eq; use serde_json::json; @@ -37,6 +40,7 @@ struct ToolPlanInputs { mcp_tools: Option>, deferred_mcp_tools: Option>, discoverable_tools: Option>, + extension_tool_executors: Vec>>, dynamic_tools: Vec, } @@ -176,7 +180,7 @@ async fn probe_with( mcp_tools: inputs.mcp_tools, deferred_mcp_tools: inputs.deferred_mcp_tools, discoverable_tools: inputs.discoverable_tools, - extension_tool_executors: Vec::new(), + extension_tool_executors: inputs.extension_tool_executors, dynamic_tools: inputs.dynamic_tools.as_slice(), }, ); @@ -253,6 +257,37 @@ fn use_bedrock_provider(turn: &mut TurnContext) { turn.provider = create_model_provider(provider_info, turn.auth_manager.clone()); } +struct WebRunExtensionTool; + +#[async_trait::async_trait] +impl ToolExecutor for WebRunExtensionTool { + fn tool_name(&self) -> ToolName { + ToolName::namespaced("web", "run") + } + + fn spec(&self) -> ToolSpec { + ToolSpec::Namespace(codex_tools::ResponsesApiNamespace { + name: "web".to_string(), + description: "Test web namespace.".to_string(), + tools: vec![ResponsesApiNamespaceTool::Function(ResponsesApiTool { + name: "run".to_string(), + description: "Test standalone web search tool.".to_string(), + strict: false, + defer_loading: None, + parameters: codex_tools::JsonSchema::default(), + output_schema: None, + })], + }) + } + + async fn handle( + &self, + _call: ExtensionToolCall, + ) -> Result, codex_tools::FunctionCallError> { + Ok(Box::new(codex_tools::JsonToolOutput::new(json!({})))) + } +} + fn duplicate_primary_environment(turn: &mut TurnContext) { let mut second_environment = turn.environments.turn_environments[0].clone(); second_environment.environment_id = "secondary".to_string(); @@ -947,6 +982,26 @@ async fn hosted_tools_follow_provider_auth_model_and_config_gates() { } ); + let standalone_web_search_without_web_run = probe(|turn| { + set_feature(turn, Feature::StandaloneWebSearch, /*enabled*/ true); + set_web_search_mode(turn, WebSearchMode::Live); + }) + .await; + standalone_web_search_without_web_run.assert_visible_contains(&["web_search"]); + + let standalone_web_search = probe_with( + |turn| { + set_feature(turn, Feature::StandaloneWebSearch, /*enabled*/ true); + set_web_search_mode(turn, WebSearchMode::Live); + }, + ToolPlanInputs { + extension_tool_executors: vec![Arc::new(WebRunExtensionTool)], + ..Default::default() + }, + ) + .await; + standalone_web_search.assert_visible_lacks(&["web_search"]); + let unsupported_provider = probe(|turn| { set_web_search_mode(turn, WebSearchMode::Live); use_bedrock_provider(turn); diff --git a/codex-rs/ext/web-search/BUILD.bazel b/codex-rs/ext/web-search/BUILD.bazel new file mode 100644 index 0000000000..e8c26644f6 --- /dev/null +++ b/codex-rs/ext/web-search/BUILD.bazel @@ -0,0 +1,9 @@ +load("//:defs.bzl", "codex_rust_crate") + +codex_rust_crate( + name = "web-search", + crate_name = "codex_web_search_extension", + compile_data = [ + "web_run_description.md", + ], +) diff --git a/codex-rs/ext/web-search/Cargo.toml b/codex-rs/ext/web-search/Cargo.toml new file mode 100644 index 0000000000..978aa4c45d --- /dev/null +++ b/codex-rs/ext/web-search/Cargo.toml @@ -0,0 +1,31 @@ +[package] +edition.workspace = true +license.workspace = true +name = "codex-web-search-extension" +version.workspace = true + +[lib] +name = "codex_web_search_extension" +path = "src/lib.rs" +doctest = false + +[lints] +workspace = true + +[dependencies] +async-trait = { workspace = true } +codex-api = { workspace = true } +codex-core = { workspace = true } +codex-extension-api = { workspace = true } +codex-features = { workspace = true } +codex-login = { workspace = true } +codex-model-provider = { workspace = true } +codex-model-provider-info = { workspace = true } +codex-protocol = { workspace = true } +codex-tools = { workspace = true } +http = { workspace = true } +schemars = { workspace = true } +serde_json = { workspace = true } + +[dev-dependencies] +pretty_assertions = { workspace = true } diff --git a/codex-rs/ext/web-search/src/extension.rs b/codex-rs/ext/web-search/src/extension.rs new file mode 100644 index 0000000000..228d77897b --- /dev/null +++ b/codex-rs/ext/web-search/src/extension.rs @@ -0,0 +1,173 @@ +use std::sync::Arc; + +use codex_api::ApproximateLocation; +use codex_api::LocationType; +use codex_api::SearchContextSize; +use codex_api::SearchFilters; +use codex_api::SearchSettings; +use codex_core::config::Config; +use codex_extension_api::ConfigContributor; +use codex_extension_api::ExtensionData; +use codex_extension_api::ExtensionRegistryBuilder; +use codex_extension_api::ThreadLifecycleContributor; +use codex_extension_api::ThreadStartInput; +use codex_extension_api::ToolContributor; +use codex_features::Feature; +use codex_login::AuthManager; +use codex_model_provider::create_model_provider; +use codex_model_provider_info::ModelProviderInfo; +use codex_protocol::config_types::WebSearchContextSize; +use codex_protocol::config_types::WebSearchMode; + +use crate::tool::WebSearchTool; + +#[derive(Clone)] +struct WebSearchExtension { + auth_manager: Arc, +} + +#[derive(Clone)] +struct WebSearchExtensionConfig { + enabled: bool, + provider: ModelProviderInfo, + settings: SearchSettings, +} + +impl From<&Config> for WebSearchExtensionConfig { + fn from(config: &Config) -> Self { + let web_search_mode = config.web_search_mode.value(); + Self { + enabled: config.features.enabled(Feature::StandaloneWebSearch) + && config.model_provider.is_openai() + && web_search_mode != WebSearchMode::Disabled, + provider: config.model_provider.clone(), + settings: search_settings(config, web_search_mode), + } + } +} + +fn search_settings(config: &Config, web_search_mode: WebSearchMode) -> SearchSettings { + let web_search_config = config.web_search_config.as_ref(); + SearchSettings { + user_location: web_search_config + .and_then(|config| config.user_location.as_ref()) + .map(|location| ApproximateLocation { + r#type: LocationType::Approximate, + country: location.country.clone(), + region: location.region.clone(), + city: location.city.clone(), + timezone: location.timezone.clone(), + }), + search_context_size: web_search_config + .and_then(|config| config.search_context_size) + .map(|size| match size { + WebSearchContextSize::Low => SearchContextSize::Low, + WebSearchContextSize::Medium => SearchContextSize::Medium, + WebSearchContextSize::High => SearchContextSize::High, + }), + filters: web_search_config + .and_then(|config| config.filters.as_ref()) + .map(|filters| SearchFilters { + allowed_domains: filters.allowed_domains.clone(), + blocked_domains: None, + }), + external_web_access: Some(match web_search_mode { + WebSearchMode::Live => true, + WebSearchMode::Cached | WebSearchMode::Disabled => false, + }), + ..Default::default() + } +} + +#[async_trait::async_trait] +impl ThreadLifecycleContributor for WebSearchExtension { + async fn on_thread_start(&self, input: ThreadStartInput<'_, Config>) { + input + .thread_store + .insert(WebSearchExtensionConfig::from(input.config)); + } +} + +impl ConfigContributor for WebSearchExtension { + fn on_config_changed( + &self, + _session_store: &ExtensionData, + thread_store: &ExtensionData, + _previous_config: &Config, + new_config: &Config, + ) { + thread_store.insert(WebSearchExtensionConfig::from(new_config)); + } +} + +impl ToolContributor for WebSearchExtension { + fn tools( + &self, + session_store: &ExtensionData, + thread_store: &ExtensionData, + ) -> Vec>> { + let Some(config) = thread_store.get::() else { + return Vec::new(); + }; + if !config.enabled { + return Vec::new(); + } + + vec![Arc::new(WebSearchTool { + session_id: session_store.level_id().to_string(), + provider: create_model_provider( + config.provider.clone(), + Some(self.auth_manager.clone()), + ), + settings: config.settings.clone(), + })] + } +} + +pub fn install(registry: &mut ExtensionRegistryBuilder, auth_manager: Arc) { + let extension = Arc::new(WebSearchExtension { auth_manager }); + registry.thread_lifecycle_contributor(extension.clone()); + registry.config_contributor(extension.clone()); + registry.tool_contributor(extension); +} + +#[cfg(test)] +mod tests { + use codex_extension_api::ExtensionData; + use codex_extension_api::ExtensionRegistryBuilder; + use codex_extension_api::ToolName; + use codex_login::CodexAuth; + use codex_model_provider_info::ModelProviderInfo; + use pretty_assertions::assert_eq; + + use super::AuthManager; + use super::Config; + use super::WebSearchExtensionConfig; + use super::install; + + #[test] + fn installed_extension_contributes_web_run_when_enabled() { + let mut builder = ExtensionRegistryBuilder::::new(); + install( + &mut builder, + AuthManager::from_auth_for_testing(CodexAuth::from_api_key("dummy")), + ); + let registry = builder.build(); + let session_store = ExtensionData::new("session"); + let thread_store = ExtensionData::new("11111111-1111-4111-8111-111111111111"); + thread_store.insert(WebSearchExtensionConfig { + enabled: true, + provider: ModelProviderInfo::create_openai_provider(/*base_url*/ None), + settings: Default::default(), + }); + + let tool_names = registry + .tool_contributors() + .iter() + .flat_map(|contributor| contributor.tools(&session_store, &thread_store)) + .map(|tool| tool.tool_name()) + .collect::>(); + + assert_eq!(tool_names, vec![ToolName::namespaced("web", "run")]); + } +} diff --git a/codex-rs/ext/web-search/src/history.rs b/codex-rs/ext/web-search/src/history.rs new file mode 100644 index 0000000000..b7a26d32cf --- /dev/null +++ b/codex-rs/ext/web-search/src/history.rs @@ -0,0 +1,170 @@ +use codex_api::SearchInput; +use codex_core::parse_turn_item; +use codex_protocol::items::TurnItem; +use codex_protocol::models::ContentItem; +use codex_protocol::models::ResponseItem; +use codex_tools::retain_tail_from_last_n_user_messages; +use codex_tools::truncate_assistant_output_text_to_token_budget; + +const ASSISTANT_CONTEXT_TOKEN_LIMIT: usize = 1_000; +const ASSISTANT_ROLE: &str = "assistant"; +const USER_ROLE: &str = "user"; + +/// Builds the conversation tail for standalone web search. +/// +/// The tail keeps the previous user text message, up to 1k tokens of assistant +/// text that followed it, and the current user text message. +pub(crate) fn recent_input(items: &[ResponseItem]) -> Option { + let mut messages = Vec::new(); + for item in items { + push_visible_message(&mut messages, item); + } + + retain_tail_from_last_n_user_messages(&mut messages, /*user_message_count*/ 2); + truncate_assistant_output_text_to_token_budget(&mut messages, ASSISTANT_CONTEXT_TOKEN_LIMIT); + (!messages.is_empty()).then_some(SearchInput::Items(messages)) +} + +fn push_visible_message(messages: &mut Vec, item: &ResponseItem) { + match item { + ResponseItem::Message { role, .. } if role == ASSISTANT_ROLE => { + messages.push(item.clone()); + } + ResponseItem::Message { + id, + role, + content, + phase, + } if role == USER_ROLE + && matches!(parse_turn_item(item), Some(TurnItem::UserMessage(_))) => + { + let content = content + .iter() + .filter(|item| matches!(item, ContentItem::InputText { .. })) + .cloned() + .collect::>(); + if !content.is_empty() { + messages.push(ResponseItem::Message { + id: id.clone(), + role: role.clone(), + content, + phase: phase.clone(), + }); + } + } + _ => {} + } +} + +#[cfg(test)] +mod tests { + use codex_api::SearchInput; + use codex_protocol::models::ContentItem; + use codex_protocol::models::ResponseItem; + use pretty_assertions::assert_eq; + + use super::ASSISTANT_ROLE; + use super::USER_ROLE; + use super::recent_input; + + fn message(role: &str, text: &str) -> ResponseItem { + ResponseItem::Message { + id: None, + role: role.to_string(), + content: vec![if role == ASSISTANT_ROLE { + ContentItem::OutputText { + text: text.to_string(), + } + } else { + ContentItem::InputText { + text: text.to_string(), + } + }], + phase: None, + } + } + + #[test] + fn keeps_current_user_and_previous_visible_turn() { + let items = vec![ + message("system", "system"), + message(USER_ROLE, "old user"), + message(ASSISTANT_ROLE, "old assistant"), + message(USER_ROLE, "previous user"), + ResponseItem::FunctionCall { + id: None, + name: "tool".to_string(), + namespace: None, + arguments: "{}".to_string(), + call_id: "call-1".to_string(), + }, + message(ASSISTANT_ROLE, "previous assistant"), + message("developer", "developer"), + message(USER_ROLE, "current user"), + message(ASSISTANT_ROLE, "current commentary"), + ]; + + assert_eq!( + recent_input(&items), + Some(SearchInput::Items(vec![ + message(USER_ROLE, "previous user"), + message(ASSISTANT_ROLE, "previous assistant"), + message(USER_ROLE, "current user"), + ])) + ); + } + + #[test] + fn keeps_only_text_from_recent_user_messages() { + let previous_user = ResponseItem::Message { + id: None, + role: USER_ROLE.to_string(), + content: vec![ + ContentItem::InputText { + text: "previous user".to_string(), + }, + ContentItem::InputImage { + image_url: "data:image/png;base64,image".to_string(), + detail: None, + }, + ], + phase: None, + }; + let items = vec![ + previous_user, + message(ASSISTANT_ROLE, "previous assistant"), + message(USER_ROLE, "current user"), + ]; + + assert_eq!( + recent_input(&items), + Some(SearchInput::Items(vec![ + message(USER_ROLE, "previous user"), + message(ASSISTANT_ROLE, "previous assistant"), + message(USER_ROLE, "current user"), + ])) + ); + } + + #[test] + fn ignores_contextual_user_messages_when_selecting_recent_turns() { + let items = vec![ + message(USER_ROLE, "previous user"), + message(ASSISTANT_ROLE, "previous assistant"), + message( + USER_ROLE, + "\n/tmp\n", + ), + message(USER_ROLE, "current user"), + ]; + + assert_eq!( + recent_input(&items), + Some(SearchInput::Items(vec![ + message(USER_ROLE, "previous user"), + message(ASSISTANT_ROLE, "previous assistant"), + message(USER_ROLE, "current user"), + ])) + ); + } +} diff --git a/codex-rs/ext/web-search/src/lib.rs b/codex-rs/ext/web-search/src/lib.rs new file mode 100644 index 0000000000..1b18039e49 --- /dev/null +++ b/codex-rs/ext/web-search/src/lib.rs @@ -0,0 +1,7 @@ +mod extension; +mod history; +mod output; +mod schema; +mod tool; + +pub use extension::install; diff --git a/codex-rs/ext/web-search/src/output.rs b/codex-rs/ext/web-search/src/output.rs new file mode 100644 index 0000000000..124271c216 --- /dev/null +++ b/codex-rs/ext/web-search/src/output.rs @@ -0,0 +1,72 @@ +use codex_extension_api::ToolOutput; +use codex_extension_api::ToolPayload; +use codex_protocol::models::FunctionCallOutputContentItem; +use codex_protocol::models::FunctionCallOutputPayload; +use codex_protocol::models::ResponseInputItem; + +pub(crate) struct EncryptedSearchOutput { + encrypted_output: String, +} + +impl EncryptedSearchOutput { + pub(crate) fn new(encrypted_output: String) -> Self { + Self { encrypted_output } + } +} + +impl ToolOutput for EncryptedSearchOutput { + fn log_preview(&self) -> String { + "[encrypted standalone web search output]".to_string() + } + + fn success_for_logging(&self) -> bool { + true + } + + fn to_response_item(&self, call_id: &str, _payload: &ToolPayload) -> ResponseInputItem { + // TODO: Make standalone search honor memories.disable_on_external_context, + // as hosted web search does. + ResponseInputItem::FunctionCallOutput { + call_id: call_id.to_string(), + output: FunctionCallOutputPayload::from_content_items(vec![ + FunctionCallOutputContentItem::EncryptedContent { + encrypted_content: self.encrypted_output.clone(), + }, + ]), + } + } +} + +#[cfg(test)] +mod tests { + use codex_extension_api::ToolPayload; + use codex_protocol::models::FunctionCallOutputContentItem; + use codex_protocol::models::FunctionCallOutputPayload; + use codex_protocol::models::ResponseInputItem; + use pretty_assertions::assert_eq; + + use super::EncryptedSearchOutput; + use super::ToolOutput; + + #[test] + fn emits_encrypted_function_call_output() { + let output = EncryptedSearchOutput::new("encrypted-search-output".to_string()); + + assert_eq!( + output.to_response_item( + "call-1", + &ToolPayload::Function { + arguments: "{}".to_string(), + }, + ), + ResponseInputItem::FunctionCallOutput { + call_id: "call-1".to_string(), + output: FunctionCallOutputPayload::from_content_items(vec![ + FunctionCallOutputContentItem::EncryptedContent { + encrypted_content: "encrypted-search-output".to_string(), + }, + ]), + } + ); + } +} diff --git a/codex-rs/ext/web-search/src/schema.rs b/codex-rs/ext/web-search/src/schema.rs new file mode 100644 index 0000000000..2f71f1595c --- /dev/null +++ b/codex-rs/ext/web-search/src/schema.rs @@ -0,0 +1,36 @@ +use codex_api::SearchCommands; +use schemars::r#gen::SchemaSettings; +use serde_json::Map; +use serde_json::Value; + +pub(crate) fn commands_schema() -> Value { + let schema = SchemaSettings::draft2019_09() + .with(|settings| { + settings.inline_subschemas = true; + settings.option_add_null_type = false; + }) + .into_generator() + .into_root_schema_for::(); + let schema = match serde_json::to_value(schema) { + Ok(schema) => schema, + Err(err) => panic!("search commands schema should serialize: {err}"), + }; + let Value::Object(mut schema) = schema else { + unreachable!("search commands schema must be an object"); + }; + + let mut tool_schema = Map::new(); + for key in [ + "properties", + "required", + "type", + "additionalProperties", + "$defs", + "definitions", + ] { + if let Some(value) = schema.remove(key) { + tool_schema.insert(key.to_string(), value); + } + } + Value::Object(tool_schema) +} diff --git a/codex-rs/ext/web-search/src/tool.rs b/codex-rs/ext/web-search/src/tool.rs new file mode 100644 index 0000000000..1e1db2deca --- /dev/null +++ b/codex-rs/ext/web-search/src/tool.rs @@ -0,0 +1,113 @@ +use codex_api::ReqwestTransport; +use codex_api::SearchClient; +use codex_api::SearchCommands; +use codex_api::SearchRequest; +use codex_api::SearchSettings; +use codex_extension_api::FunctionCallError; +use codex_extension_api::ResponsesApiTool; +use codex_extension_api::ToolCall; +use codex_extension_api::ToolExecutor; +use codex_extension_api::ToolName; +use codex_extension_api::ToolOutput; +use codex_extension_api::ToolSpec; +use codex_extension_api::parse_tool_input_schema; +use codex_login::default_client::build_reqwest_client; +use codex_model_provider::SharedModelProvider; +use codex_tools::ResponsesApiNamespace; +use codex_tools::ResponsesApiNamespaceTool; +use codex_tools::ToolExposure; +use codex_tools::default_namespace_description; +use http::HeaderMap; + +use crate::history::recent_input; +use crate::output::EncryptedSearchOutput; +use crate::schema::commands_schema; + +const WEB_NAMESPACE: &str = "web"; +const RUN_TOOL_NAME: &str = "run"; +const WEB_RUN_DESCRIPTION: &str = include_str!("../web_run_description.md"); + +pub(crate) struct WebSearchTool { + pub(crate) session_id: String, + pub(crate) provider: SharedModelProvider, + pub(crate) settings: SearchSettings, +} + +#[async_trait::async_trait] +impl ToolExecutor for WebSearchTool { + fn tool_name(&self) -> ToolName { + ToolName::namespaced(WEB_NAMESPACE, RUN_TOOL_NAME) + } + + fn spec(&self) -> ToolSpec { + let parameters = match parse_tool_input_schema(&commands_schema()) { + Ok(parameters) => parameters, + Err(err) => panic!("search command schema should parse: {err}"), + }; + + ToolSpec::Namespace(ResponsesApiNamespace { + name: WEB_NAMESPACE.to_string(), + description: default_namespace_description(WEB_NAMESPACE), + tools: vec![ResponsesApiNamespaceTool::Function(ResponsesApiTool { + name: RUN_TOOL_NAME.to_string(), + description: WEB_RUN_DESCRIPTION.to_string(), + strict: false, + parameters, + output_schema: None, + defer_loading: None, + })], + }) + } + + fn exposure(&self) -> ToolExposure { + ToolExposure::DirectModelOnly + } + + async fn handle(&self, call: ToolCall) -> Result, FunctionCallError> { + let commands = parse_commands(&call)?; + let provider = self + .provider + .api_provider() + .await + .map_err(|err| FunctionCallError::Fatal(err.to_string()))?; + let auth = self + .provider + .api_auth() + .await + .map_err(|err| FunctionCallError::Fatal(err.to_string()))?; + let client = SearchClient::new( + ReqwestTransport::new(build_reqwest_client()), + provider, + auth, + ); + let request = SearchRequest { + id: self.session_id.clone(), + model: None, + reasoning: None, + input: recent_input(call.conversation_history.items()), + commands: Some(commands), + settings: Some(self.settings.clone()), + max_output_tokens: Some( + u64::try_from(call.truncation_policy.token_budget()).unwrap_or(u64::MAX), + ), + }; + let response = client + .search(&request, HeaderMap::new()) + .await + .map_err(|err| FunctionCallError::Fatal(err.to_string()))?; + + Ok(Box::new(EncryptedSearchOutput::new( + response.encrypted_output, + ))) + } +} + +fn parse_commands(call: &ToolCall) -> Result { + let arguments = call.function_arguments()?; + if arguments.trim().is_empty() { + return Ok(SearchCommands::default()); + } + + serde_json::from_str(arguments) + .map_err(|err| FunctionCallError::RespondToModel(err.to_string())) +} diff --git a/codex-rs/ext/web-search/web_run_description.md b/codex-rs/ext/web-search/web_run_description.md new file mode 100644 index 0000000000..bccc3d81f6 --- /dev/null +++ b/codex-rs/ext/web-search/web_run_description.md @@ -0,0 +1,80 @@ +Tool for accessing the internet. + + +--- + +## Examples of different commands available in this tool + +Examples of different commands available in this tool: +* `search_query`: {"search_query": [{"q": "What is the capital of France?"}, {"q": "What is the capital of belgium?"}]}. Searches the internet for a given query (and optionally with a domain or recency filter) +* `image_query`: {"image_query":[{"q": "waterfalls"}]}. +* `open`: {"open": [{"ref_id": "turn0search0"}, {"ref_id": "https://www.openai.com", "lineno": 120}]} +* `click`: {"click": [{"ref_id": "turn0fetch3", "id": 17}]} +* `find`: {"find": [{"ref_id": "turn0fetch3", "pattern": "Annie Case"}]} +* `screenshot`: {"screenshot": [{"ref_id": "turn1view0", "pageno": 0}, {"ref_id": "turn1view0", "pageno": 3}]} +* `finance`: {"finance":[{"ticker":"AMD","type":"equity","market":"USA"}]}, {"finance":[{"ticker":"BTC","type":"crypto","market":""}]} +* `weather`: {"weather":[{"location":"San Francisco, CA"}]} +* `sports`: {"sports":[{"fn":"standings","league":"nfl"}, {"fn":"schedule","league":"nba","team":"GSW","date_from":"2025-02-24"}]} +* `time`: {"time":[{"utc_offset":"+03:00"}]} + +--- + +## Usage hints +To use this tool efficiently: +* Use multiple commands and queries in one call to get more results faster; e.g. {"search_query": [{"q": "bitcoin news"}], "finance":[{"ticker":"BTC","type":"crypto","market":""}], "find": [{"ref_id": "turn0search0", "pattern": "Annie Case"}, {"ref_id": "turn0search1", "pattern": "John Smith"}]} +* Use "response_length" to control the number of results returned by this tool, omit it if you intend to pass "short" in +* Only write required parameters; do not write empty lists or nulls where they could be omitted. +* `search_query` must have length at most 4 in each call. If it has length > 3, response_length must be medium or long +* If you find yourself in a situation where you accidentally call the `web.run` tool, it's best just to send an empty query: {"search_query": [{"q": ""}]}. + +--- + +## Decision boundary + +If the user makes an explicit request to search the internet, find latest information, look up, etc (or to not do so), you must obey their request. +When you make an assumption, always consider whether it is temporally stable; i.e. whether there's even a small (>10%) chance it has changed. If it is unstable, you must verify with browsing the internet for verification. + + +Below is a list of scenarios where browsing the internet MUST be used. PAY CLOSE ATTENTION: you MUST browse the internet in these cases. If you're unsure or on the fence, you MUST bias towards browsing the internet. +- The information could have changed recently: for example news; prices; laws; schedules; product specs; sports scores; economic indicators; political/public/company figures (e.g. the question relates to 'the president of country A' or 'the CEO of company B', which might change over time); rules; regulations; standards; software libraries that could be updated; exchange rates; recommendations (i.e., recommendations about various topics or things might be informed by what currently exists / is popular / is safe / is unsafe / is in the zeitgeist / etc.); and many many many more categories -- again, if you're on the fence, you MUST browse the internet! + - For news queries, prioritize more recent events, ensuring you compare publish dates and the date that the event happened. +- The user is seeking recommendations that could lead them to spend substantial time or money -- researching products, restaurants, travel plans, etc. +- The user wants (or would benefit from) direct quotes, links, or precise source attribution. +- A specific page, paper, dataset, PDF, or site is referenced and you haven't been given its contents. +- You're unsure about a fact, the topic is niche or emerging, or you suspect there's at least a 10% chance you will incorrectly recall it +- High-stakes accuracy matters (medical, legal, financial guidance). For these you generally should search by default because this information is highly temporally unstable +- The user explicitly says to search, browse, verify, or look it up. + + +--- + +## Special cases +If these conflict with any other instructions, these should take precedence. + + +- When the user asks for information about how to use OpenAI products, (ChatGPT, the OpenAI API, etc.), you should check the code in local env and only browse as fallback, when you browse restrict your sources to official OpenAI websites using the domains filter, unless otherwise requested. +- When using search to answer technical questions, you must only rely on primary sources (research papers, official documentation, etc.) +- Clearly indicate when you are making an inference from sources. + + +--- + +## Word limits +Responses may not excessively quote or draw on a specific source. There are several limits here: +- **Limit on verbatim quotes:** + - You may not quote more than 25 words verbatim from any single non-lyrical source, unless the source is reddit. + - For song lyrics, verbatim quotes must be limited to at most 10 words. + - Long quotes from reddit are allowed, as long as you indicate that those are direct quotes via a markdown blockquote starting with ">", copy verbatim, and link the source. +- **Word limits:** + - Each webpage source in the sources has a word limit label formatted like "[wordlim N]", in which N is the maximum number of words in the whole response that are attributed to that source. If omitted, the word limit is 200 words. + - Non-contiguous words derived from a given source must be counted to the word limit. + - The summarization limit N is a maximum for each source. + - When using multiple sources, their summarization limits add together. However, each article used must be relevant to the response. +- **Copyright compliance:** + - You must avoid providing full articles, long verbatim passages, or extensive direct quotes due to copyright concerns. + - If the user asked for a verbatim quote, the response should provide a short compliant excerpt and then answer with paraphrases and summaries. + - Again, this limit does not apply to reddit content, as long as it's appropriately indicated that those are direct quotes and you link to the source. + +--- + +Make sure to provide links to the sources you used in your response. diff --git a/codex-rs/features/src/lib.rs b/codex-rs/features/src/lib.rs index cd50de4330..391e33e77d 100644 --- a/codex-rs/features/src/lib.rs +++ b/codex-rs/features/src/lib.rs @@ -103,6 +103,8 @@ pub enum Feature { /// Allow the model to request web searches that fetch cached content. /// Takes precedence over `WebSearchRequest`. WebSearchCached, + /// Expose the extension-backed standalone web search tool. + StandaloneWebSearch, /// Use the legacy Landlock Linux sandbox fallback instead of the default /// bubblewrap pipeline. UseLegacyLandlock, @@ -787,6 +789,12 @@ pub const FEATURES: &[FeatureSpec] = &[ stage: Stage::Deprecated, default_enabled: false, }, + FeatureSpec { + id: Feature::StandaloneWebSearch, + key: "standalone_web_search", + stage: Stage::UnderDevelopment, + default_enabled: false, + }, FeatureSpec { id: Feature::SearchTool, key: "search_tool", diff --git a/codex-rs/protocol/src/models.rs b/codex-rs/protocol/src/models.rs index 28a7d6522c..6112ea8a6c 100644 --- a/codex-rs/protocol/src/models.rs +++ b/codex-rs/protocol/src/models.rs @@ -902,6 +902,13 @@ pub enum ResponseItem { Other, } +impl ResponseItem { + /// Returns whether this item is an ordinary user-role message. + pub fn is_user_message(&self) -> bool { + matches!(self, Self::Message { role, .. } if role == "user") + } +} + pub const BASE_INSTRUCTIONS_DEFAULT: &str = include_str!("prompts/base_instructions/default.md"); /// Base instructions for the model in a thread. Corresponds to the `instructions` field in the ResponsesAPI. diff --git a/codex-rs/tools/src/lib.rs b/codex-rs/tools/src/lib.rs index c141bfb37a..5e8c6fc1bc 100644 --- a/codex-rs/tools/src/lib.rs +++ b/codex-rs/tools/src/lib.rs @@ -8,6 +8,7 @@ mod image_detail; mod json_schema; mod mcp_tool; mod request_plugin_install; +mod response_history; mod responses_api; mod tool_call; mod tool_config; @@ -45,6 +46,8 @@ pub use request_plugin_install::RequestPluginInstallResult; pub use request_plugin_install::all_requested_connectors_picked_up; pub use request_plugin_install::build_request_plugin_install_elicitation_request; pub use request_plugin_install::verified_connector_install_completed; +pub use response_history::retain_tail_from_last_n_user_messages; +pub use response_history::truncate_assistant_output_text_to_token_budget; pub use responses_api::FreeformTool; pub use responses_api::FreeformToolFormat; pub use responses_api::LoadableToolSpec; diff --git a/codex-rs/tools/src/response_history.rs b/codex-rs/tools/src/response_history.rs new file mode 100644 index 0000000000..cee6f438cb --- /dev/null +++ b/codex-rs/tools/src/response_history.rs @@ -0,0 +1,149 @@ +use codex_protocol::models::ContentItem; +use codex_protocol::models::ResponseItem; +use codex_utils_output_truncation::TruncationPolicy; +use codex_utils_output_truncation::approx_token_count; +use codex_utils_output_truncation::truncate_text; + +/// Retains items from the earliest of the last `user_message_count` user +/// messages through the latest user message. +pub fn retain_tail_from_last_n_user_messages( + items: &mut Vec, + user_message_count: usize, +) { + if user_message_count == 0 { + items.clear(); + return; + } + + let Some(latest_user_idx) = items.iter().rposition(ResponseItem::is_user_message) else { + items.clear(); + return; + }; + items.truncate(latest_user_idx + 1); + + let earliest_retained_user_idx = items + .iter() + .enumerate() + .rev() + .filter(|(_, item)| item.is_user_message()) + .take(user_message_count) + .last() + .map(|(idx, _)| idx) + .unwrap_or(latest_user_idx); + items.drain(..earliest_retained_user_idx); +} + +/// Truncates assistant output text to a shared token budget across items. +pub fn truncate_assistant_output_text_to_token_budget( + items: &mut Vec, + max_tokens: usize, +) { + let mut remaining_budget = max_tokens; + + items.retain_mut(|item| { + let ResponseItem::Message { role, content, .. } = item else { + return true; + }; + if role != "assistant" { + return true; + } + + content.retain_mut(|content_item| { + let ContentItem::OutputText { text } = content_item else { + return true; + }; + if remaining_budget == 0 { + return false; + } + + let token_count = approx_token_count(text); + if token_count <= remaining_budget { + remaining_budget = remaining_budget.saturating_sub(token_count); + return true; + } + + *text = truncate_text(text, TruncationPolicy::Tokens(remaining_budget)); + remaining_budget = 0; + true + }); + !content.is_empty() + }); +} + +#[cfg(test)] +mod tests { + use codex_protocol::models::ContentItem; + use codex_protocol::models::ResponseItem; + use codex_utils_output_truncation::TruncationPolicy; + use codex_utils_output_truncation::truncate_text; + use pretty_assertions::assert_eq; + + use super::retain_tail_from_last_n_user_messages; + use super::truncate_assistant_output_text_to_token_budget; + + fn message(role: &str, text: &str) -> ResponseItem { + ResponseItem::Message { + id: None, + role: role.to_string(), + content: vec![if role == "assistant" { + ContentItem::OutputText { + text: text.to_string(), + } + } else { + ContentItem::InputText { + text: text.to_string(), + } + }], + phase: None, + } + } + + #[test] + fn retains_tail_through_latest_user_message() { + let mut items = vec![ + message("system", "system"), + message("user", "old user"), + message("assistant", "old assistant"), + message("user", "previous user"), + message("assistant", "previous assistant"), + message("user", "current user"), + message("assistant", "later assistant"), + ]; + + retain_tail_from_last_n_user_messages(&mut items, /*user_message_count*/ 2); + + assert_eq!( + items, + vec![ + message("user", "previous user"), + message("assistant", "previous assistant"), + message("user", "current user"), + ] + ); + } + + #[test] + fn truncates_assistant_output_text_across_items() { + let long_assistant = "a".repeat(16); + let mut items = vec![ + message("user", "previous user"), + message("assistant", &long_assistant), + message("assistant", "after budget"), + message("user", "current user"), + ]; + + truncate_assistant_output_text_to_token_budget(&mut items, /*max_tokens*/ 2); + + assert_eq!( + items, + vec![ + message("user", "previous user"), + message( + "assistant", + &truncate_text(&long_assistant, TruncationPolicy::Tokens(2)), + ), + message("user", "current user"), + ] + ); + } +} From 833c19ed53a8db3c0d1ceb6b174958e847e7c68c Mon Sep 17 00:00:00 2001 From: Felipe Coury Date: Tue, 26 May 2026 16:08:45 -0300 Subject: [PATCH 07/16] fix(tui): keep raw output above composer in zellij (#24593) ## Why Raw output mode intentionally sends logical source lines to the terminal without Codex-inserted wrapping so copied content retains its original line structure. In Zellij, soft-wrapped continuation rows from those raw lines are not confined by the inline history scroll region. When raw mode replays a long transcript, continuation rows can occupy the composer viewport and are overwritten on the following draw, leaving the transcript visibly truncated underneath the composer. This is specific to the combination of Zellij and raw terminal-wrapped history. Rich output and non-Zellij terminals should continue using the existing insertion behavior. Related context: #20819 introduced raw output mode, and #22214 removed the broad Zellij insertion workaround after the standard rich-output path no longer required it. | Before | After | |---|---| | image | image | ## What Changed - Cache Zellij detection in `Tui` and select a dedicated insertion mode only for `HistoryLineWrapPolicy::Terminal` batches in Zellij. - For that guarded path, clear the existing viewport, append raw source lines through the terminal so its soft wrapping remains selection-friendly, and reserve empty viewport rows before redrawing the composer. - Add snapshot regressions for both an incremental soft-wrapped raw insert and an overflowing raw transcript replay that starts at the top of the cleared terminal. ## How to Test 1. Start Codex inside Zellij with raw output enabled or toggle raw output after a multiline response is in history. 2. Produce or replay output containing long logical lines, such as a fenced shell command with several wrapped lines. 3. Confirm the wrapped history remains visible above the composer and the composer no longer overwrites the end of the response. 4. Toggle back to rich output or run outside Zellij and confirm standard history rendering still behaves normally. Targeted tests run: - `just test -p codex-tui vt100_zellij_raw -- --nocapture` Additional validation notes: - `just test -p codex-tui` was attempted; the two new Zellij raw insertion tests passed, while two existing `app::tests::update_feature_flags_disabling_guardian_*` tests failed outside this history insertion path. - `just argument-comment-lint` was attempted but local Bazel analysis fails before reaching the changed source because the LLVM `compiler-rt` package is missing `include/sanitizer/*.h`. Modified literal callsites were inspected manually. --- codex-rs/tui/src/insert_history.rs | 232 ++++++++++++++---- ...llij_raw_terminal_wrap_above_viewport.snap | 10 + ...terminal_wrap_overflow_above_viewport.snap | 10 + codex-rs/tui/src/tui.rs | 26 +- 4 files changed, 228 insertions(+), 50 deletions(-) create mode 100644 codex-rs/tui/src/snapshots/codex_tui__insert_history__tests__zellij_raw_terminal_wrap_above_viewport.snap create mode 100644 codex-rs/tui/src/snapshots/codex_tui__insert_history__tests__zellij_raw_terminal_wrap_overflow_above_viewport.snap diff --git a/codex-rs/tui/src/insert_history.rs b/codex-rs/tui/src/insert_history.rs index ede7d03c78..40da6ccbcc 100644 --- a/codex-rs/tui/src/insert_history.rs +++ b/codex-rs/tui/src/insert_history.rs @@ -40,6 +40,17 @@ pub enum HistoryLineWrapPolicy { Terminal, } +/// Selects the terminal escape strategy used when writing history above the viewport. +/// +/// Raw lines intentionally remain unbroken so terminal selection copies their source faithfully. +/// Zellij does not constrain soft-wrapped continuation rows to Codex's scroll region, so its raw +/// path appends history through the terminal and reserves blank rows for the next viewport draw. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum InsertHistoryMode { + Standard, + ZellijRaw, +} + /// Insert `lines` above the viewport using the terminal's backend writer /// (avoids direct stdout references). pub fn insert_history_lines( @@ -57,6 +68,23 @@ pub fn insert_history_lines_with_wrap_policy( lines: Vec, wrap_policy: HistoryLineWrapPolicy, ) -> io::Result<()> +where + B: Backend + Write, +{ + insert_history_lines_with_mode_and_wrap_policy( + terminal, + lines, + InsertHistoryMode::Standard, + wrap_policy, + ) +} + +pub(crate) fn insert_history_lines_with_mode_and_wrap_policy( + terminal: &mut crate::custom_terminal::Terminal, + lines: Vec, + mode: InsertHistoryMode, + wrap_policy: HistoryLineWrapPolicy, +) -> io::Result<()> where B: Backend + Write, { @@ -65,7 +93,6 @@ where let mut area = terminal.viewport_area; let mut should_update_area = false; let last_cursor_pos = terminal.last_known_cursor_pos; - let writer = terminal.backend_mut(); // Pre-wrap lines for terminal scrollback. Three paths: // @@ -102,60 +129,92 @@ where wrapped.extend(line_wrapped); } let wrapped_lines = wrapped_rows as u16; - let cursor_top = if area.bottom() < screen_size.height { - // If the viewport is not at the bottom of the screen, scroll it down to make room. - // Don't scroll it past the bottom of the screen. - let scroll_amount = wrapped_lines.min(screen_size.height - area.bottom()); + match mode { + InsertHistoryMode::ZellijRaw => { + // The existing viewport is immediately replaced in the same draw pass. Clear it + // before terminal scrolling can move composer contents into scrollback. + terminal.clear_after_position(area.as_position())?; + let writer = terminal.backend_mut(); + queue!(writer, MoveTo(/*x*/ 0, area.top()))?; + for (index, line) in wrapped.iter().enumerate() { + if index > 0 { + queue!(writer, Print("\r\n"))?; + } + write_history_line(writer, line, wrap_width)?; + } - let top_1based = area.top() + 1; - queue!(writer, SetScrollRegion(top_1based..screen_size.height))?; - queue!(writer, MoveTo(/*x*/ 0, area.top()))?; - for _ in 0..scroll_amount { - queue!(writer, Print("\x1bM"))?; + // Writing raw source text through the terminal preserves its soft-wrap metadata. + // Advance through empty rows for the viewport so history ends immediately above the + // composer even when a replay batch is taller than the visible history region. + for _ in 0..area.height { + queue!(writer, Print("\r\n"), Clear(ClearType::UntilNewLine))?; + } + queue!(writer, MoveTo(last_cursor_pos.x, last_cursor_pos.y))?; + + let viewport_top = area + .top() + .saturating_add(wrapped_lines) + .min(screen_size.height.saturating_sub(area.height)); + if area.y != viewport_top { + area.y = viewport_top; + should_update_area = true; + } } - queue!(writer, ResetScrollRegion)?; + InsertHistoryMode::Standard => { + let writer = terminal.backend_mut(); + let cursor_top = if area.bottom() < screen_size.height { + // If the viewport is not at the bottom of the screen, scroll it down to make room. + // Don't scroll it past the bottom of the screen. + let scroll_amount = wrapped_lines.min(screen_size.height - area.bottom()); - let cursor_top = area.top().saturating_sub(1); - area.y += scroll_amount; - should_update_area = true; - cursor_top - } else { - area.top().saturating_sub(1) - }; + let top_1based = area.top() + 1; + queue!(writer, SetScrollRegion(top_1based..screen_size.height))?; + queue!(writer, MoveTo(/*x*/ 0, area.top()))?; + for _ in 0..scroll_amount { + queue!(writer, Print("\x1bM"))?; + } + queue!(writer, ResetScrollRegion)?; - // Limit the scroll region to the lines from the top of the screen to the - // top of the viewport. With this in place, when we add lines inside this - // area, only the lines in this area will be scrolled. We place the cursor - // at the end of the scroll region, and add lines starting there. - // - // ┌─Screen───────────────────────┐ - // │┌╌Scroll region╌╌╌╌╌╌╌╌╌╌╌╌╌╌┐│ - // │┆ ┆│ - // │┆ ┆│ - // │┆ ┆│ - // │█╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┘│ - // │╭─Viewport───────────────────╮│ - // ││ ││ - // │╰────────────────────────────╯│ - // └──────────────────────────────┘ - queue!(writer, SetScrollRegion(1..area.top()))?; + let cursor_top = area.top().saturating_sub(1); + area.y += scroll_amount; + should_update_area = true; + cursor_top + } else { + area.top().saturating_sub(1) + }; - // NB: we are using MoveTo instead of set_cursor_position here to avoid messing with the - // terminal's last_known_cursor_position, which hopefully will still be accurate after we - // fetch/restore the cursor position. insert_history_lines should be cursor-position-neutral :) - queue!(writer, MoveTo(/*x*/ 0, cursor_top))?; + // Limit the scroll region to the lines from the top of the screen to the + // top of the viewport. With this in place, when we add lines inside this + // area, only the lines in this area will be scrolled. We place the cursor + // at the end of the scroll region, and add lines starting there. + // + // ┌─Screen───────────────────────┐ + // │┌╌Scroll region╌╌╌╌╌╌╌╌╌╌╌╌╌╌┐│ + // │┆ ┆│ + // │┆ ┆│ + // │┆ ┆│ + // │█╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┘│ + // │╭─Viewport───────────────────╮│ + // ││ ││ + // │╰────────────────────────────╯│ + // └──────────────────────────────┘ + queue!(writer, SetScrollRegion(1..area.top()))?; - for line in &wrapped { - queue!(writer, Print("\r\n"))?; - write_history_line(writer, line, wrap_width)?; + // NB: we are using MoveTo instead of set_cursor_position here to avoid messing with the + // terminal's last_known_cursor_position, which hopefully will still be accurate after we + // fetch/restore the cursor position. insert_history_lines should be cursor-position-neutral :) + queue!(writer, MoveTo(/*x*/ 0, cursor_top))?; + + for line in &wrapped { + queue!(writer, Print("\r\n"))?; + write_history_line(writer, line, wrap_width)?; + } + + queue!(writer, ResetScrollRegion)?; + queue!(writer, MoveTo(last_cursor_pos.x, last_cursor_pos.y))?; + } } - queue!(writer, ResetScrollRegion)?; - - // Restore the cursor position to where it was before we started. - queue!(writer, MoveTo(last_cursor_pos.x, last_cursor_pos.y))?; - - let _ = writer; if should_update_area { terminal.set_viewport_area(area); } @@ -796,6 +855,85 @@ mod tests { ); } + #[test] + fn vt100_zellij_raw_insert_keeps_soft_wrapped_tail_above_viewport() { + let width: u16 = 20; + let height: u16 = 8; + let backend = VT100Backend::new(width, height); + let mut term = crate::custom_terminal::Terminal::with_options(backend).expect("terminal"); + let viewport = Rect::new( + /*x*/ 0, + /*y*/ height - 2, + /*width*/ width, + /*height*/ 2, + ); + term.set_viewport_area(viewport); + + let line = Line::from("raw-start-aaaaaaaaaaaaaaaaaaaaaaaa-tail-must-remain"); + insert_history_lines_with_mode_and_wrap_policy( + &mut term, + vec![line], + InsertHistoryMode::ZellijRaw, + HistoryLineWrapPolicy::Terminal, + ) + .expect("insert Zellij raw history"); + + let rows: Vec = term.backend().vt100().screen().rows(0, width).collect(); + insta::assert_snapshot!("zellij_raw_terminal_wrap_above_viewport", rows.join("\n")); + let history_rows = rows[..usize::from(term.viewport_area.y)] + .iter() + .map(|row| row.trim_end()) + .collect::(); + let viewport_rows = rows[usize::from(term.viewport_area.y)..].join("\n"); + assert!( + history_rows.contains("tail-must-remain"), + "expected wrapped raw tail above the viewport, rows: {rows:?}" + ); + assert!( + !viewport_rows.contains("tail-must-remain"), + "raw tail must not be written through the viewport, rows: {rows:?}" + ); + } + + #[test] + fn vt100_zellij_raw_replay_keeps_overflowing_soft_wrapped_tail_above_viewport() { + let width: u16 = 20; + let height: u16 = 8; + let backend = VT100Backend::new(width, height); + let mut term = crate::custom_terminal::Terminal::with_options(backend).expect("terminal"); + term.set_viewport_area(Rect::new( + /*x*/ 0, /*y*/ 0, /*width*/ width, /*height*/ 2, + )); + + let line = Line::from(format!("raw-start-{}tail-must-remain", "a".repeat(130))); + insert_history_lines_with_mode_and_wrap_policy( + &mut term, + vec![line], + InsertHistoryMode::ZellijRaw, + HistoryLineWrapPolicy::Terminal, + ) + .expect("replay Zellij raw history"); + + let rows: Vec = term.backend().vt100().screen().rows(0, width).collect(); + insta::assert_snapshot!( + "zellij_raw_terminal_wrap_overflow_above_viewport", + rows.join("\n") + ); + let history_rows = rows[..usize::from(term.viewport_area.y)] + .iter() + .map(|row| row.trim_end()) + .collect::(); + let viewport_rows = rows[usize::from(term.viewport_area.y)..].join("\n"); + assert!( + history_rows.contains("tail-must-remain"), + "expected overflowing raw tail above the viewport, rows: {rows:?}" + ); + assert!( + !viewport_rows.contains("tail-must-remain"), + "overflowing raw tail must not be written through the viewport, rows: {rows:?}" + ); + } + #[test] fn vt100_unwrapped_url_like_clears_continuation_rows() { let width: u16 = 20; diff --git a/codex-rs/tui/src/snapshots/codex_tui__insert_history__tests__zellij_raw_terminal_wrap_above_viewport.snap b/codex-rs/tui/src/snapshots/codex_tui__insert_history__tests__zellij_raw_terminal_wrap_above_viewport.snap new file mode 100644 index 0000000000..812030a9d1 --- /dev/null +++ b/codex-rs/tui/src/snapshots/codex_tui__insert_history__tests__zellij_raw_terminal_wrap_above_viewport.snap @@ -0,0 +1,10 @@ +--- +source: tui/src/insert_history.rs +expression: "rows.join(\"\\n\")" +--- + + + +raw-start-aaaaaaaaaa +aaaaaaaaaaaaaa-tail- +must-remain diff --git a/codex-rs/tui/src/snapshots/codex_tui__insert_history__tests__zellij_raw_terminal_wrap_overflow_above_viewport.snap b/codex-rs/tui/src/snapshots/codex_tui__insert_history__tests__zellij_raw_terminal_wrap_overflow_above_viewport.snap new file mode 100644 index 0000000000..091250848a --- /dev/null +++ b/codex-rs/tui/src/snapshots/codex_tui__insert_history__tests__zellij_raw_terminal_wrap_overflow_above_viewport.snap @@ -0,0 +1,10 @@ +--- +source: tui/src/insert_history.rs +expression: "rows.join(\"\\n\")" +--- +aaaaaaaaaaaaaaaaaaaa +aaaaaaaaaaaaaaaaaaaa +aaaaaaaaaaaaaaaaaaaa +aaaaaaaaaaaaaaaaaaaa +aaaaaaaaaaaaaaaaaaaa +tail-must-remain diff --git a/codex-rs/tui/src/tui.rs b/codex-rs/tui/src/tui.rs index 49d838e866..ef2f1b07dc 100644 --- a/codex-rs/tui/src/tui.rs +++ b/codex-rs/tui/src/tui.rs @@ -41,6 +41,7 @@ pub use self::frame_requester::FrameRequester; use crate::custom_terminal; use crate::custom_terminal::Terminal as CustomTerminal; use crate::insert_history::HistoryLineWrapPolicy; +use crate::insert_history::InsertHistoryMode; use crate::notifications::DesktopNotificationBackend; use crate::notifications::detect_backend; use crate::tui::event_stream::EventBroker; @@ -500,6 +501,8 @@ pub struct Tui { enhanced_keys_supported: bool, notification_backend: Option, notification_condition: NotificationCondition, + // Raw terminal-wrapped history needs a non-scroll-region insertion path in Zellij. + is_zellij: bool, // When false, enter_alt_screen() becomes a no-op. alt_screen_enabled: bool, // Keeps unmanaged process stderr writes out of the inline viewport. @@ -535,6 +538,7 @@ impl Tui { // Cache this to avoid contention with the event reader. supports_color::on_cached(supports_color::Stream::Stdout); let _ = crate::terminal_palette::default_colors(); + let is_zellij = codex_terminal_detection::terminal_info().is_zellij(); Self { frame_requester, @@ -552,6 +556,7 @@ impl Tui { enhanced_keys_supported, notification_backend: Some(detect_backend(NotificationMethod::default())), notification_condition: NotificationCondition::default(), + is_zellij, alt_screen_enabled: true, _stderr_guard: stderr_guard, } @@ -797,15 +802,22 @@ impl Tui { fn flush_pending_history_lines( terminal: &mut Terminal, pending_history_lines: &mut Vec, + is_zellij: bool, ) -> Result<()> { if pending_history_lines.is_empty() { return Ok(()); } for batch in pending_history_lines.iter() { - crate::insert_history::insert_history_lines_with_wrap_policy( + let mode = if is_zellij && batch.wrap_policy == HistoryLineWrapPolicy::Terminal { + InsertHistoryMode::ZellijRaw + } else { + InsertHistoryMode::Standard + }; + crate::insert_history::insert_history_lines_with_mode_and_wrap_policy( terminal, batch.lines.clone(), + mode, batch.wrap_policy, )?; } @@ -862,7 +874,11 @@ impl Tui { terminal.set_viewport_area(area); } - Self::flush_pending_history_lines(terminal, &mut self.pending_history_lines)?; + Self::flush_pending_history_lines( + terminal, + &mut self.pending_history_lines, + self.is_zellij, + )?; // Update the y position for suspending so Ctrl-Z can place the cursor correctly. #[cfg(unix)] @@ -968,7 +984,11 @@ impl Tui { let terminal = &mut self.terminal; let needs_full_repaint = Self::update_inline_viewport_for_resize_reflow(terminal, height)?; - Self::flush_pending_history_lines(terminal, &mut self.pending_history_lines)?; + Self::flush_pending_history_lines( + terminal, + &mut self.pending_history_lines, + self.is_zellij, + )?; if needs_full_repaint { terminal.invalidate_viewport(); From cd934c8bcb1ba50791a980db1d0b38cef7cf84f9 Mon Sep 17 00:00:00 2001 From: canvrno-oai Date: Tue, 26 May 2026 12:09:07 -0700 Subject: [PATCH 08/16] tui: keep inaccessible apps out of mentions (#24625) ## Summary Fix the TUI `$` app mention paths so App Directory rows that are not accessible are not treated as usable apps. This includes the core preservation fix from #24104, but expands it to the other app mention paths: - preserve app-server `is_accessible` flags when partial `app/list/updated` snapshots reach the TUI - require apps to be both accessible and enabled when resolving exact `$slug` mentions - require restored/stale `app://...` bindings to point at accessible, enabled apps before emitting structured app mentions - remove the now-unused `codex-chatgpt` dependency from `codex-tui`, which addresses the `cargo shear` failure seen on #24104 ## Root Cause The app server already sends merged app snapshots with accessibility computed. The TUI handled app-server app list updates as partial app loads and re-ran the old accessible-app merge path. That path treated every notification row as accessible, so App Directory entries with `isAccessible=false` could appear in `$` suggestions. Regression source: #22914 routed app-list updates through the app server while reusing the old TUI partial-load handling. Related precursor: #14717 introduced the partial-load path, but #22914 made it user-visible for app-server updates. ## Issues Fixes #24145 Fixes #24205 Fixes #24319 ## Validation - `just fmt` - `git diff --check` - `just bazel-lock-update` - `just bazel-lock-check` - `just argument-comment-lint -p codex-tui` - `just test -p codex-tui chatwidget::tests::popups_and_settings::apps_notification_update_excludes_inaccessible_apps_from_mentions chatwidget::tests::composer_submission::submit_user_message_ignores_inaccessible_app_mentions_from_bindings chatwidget::skills::tests::find_app_mentions_requires_accessible_enabled_apps_for_bound_paths chatwidget::skills::tests::find_app_mentions_requires_accessible_enabled_apps_for_slugs` --- codex-rs/Cargo.lock | 1 - codex-rs/tui/Cargo.toml | 1 - codex-rs/tui/src/chatwidget.rs | 2 +- codex-rs/tui/src/chatwidget/connectors.rs | 7 -- .../tui/src/chatwidget/input_submission.rs | 8 +- codex-rs/tui/src/chatwidget/skills.rs | 81 ++++++++++++++++++- .../chatwidget/tests/composer_submission.rs | 56 +++++++++++++ .../chatwidget/tests/popups_and_settings.rs | 71 ++++++++++++++++ 8 files changed, 212 insertions(+), 15 deletions(-) diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index a1884ba9e2..044146d9de 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -3790,7 +3790,6 @@ dependencies = [ "codex-app-server-client", "codex-app-server-protocol", "codex-arg0", - "codex-chatgpt", "codex-cli", "codex-cloud-requirements", "codex-config", diff --git a/codex-rs/tui/Cargo.toml b/codex-rs/tui/Cargo.toml index c213e92bae..ffc5c5a942 100644 --- a/codex-rs/tui/Cargo.toml +++ b/codex-rs/tui/Cargo.toml @@ -31,7 +31,6 @@ codex-app-server-client = { workspace = true } codex-app-server-protocol = { workspace = true } codex-arg0 = { workspace = true } codex-install-context = { workspace = true } -codex-chatgpt = { workspace = true } codex-cloud-requirements = { workspace = true } codex-config = { workspace = true } codex-connectors = { workspace = true } diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 2b86ab3d71..b9c5101187 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -124,7 +124,6 @@ use codex_app_server_protocol::TurnCompletedNotification; use codex_app_server_protocol::TurnPlanStepStatus; use codex_app_server_protocol::TurnStatus; use codex_app_server_protocol::UserInput; -use codex_chatgpt::connectors as chatgpt_connectors; use codex_config::ConfigLayerStackOrdering; use codex_config::types::ApprovalsReviewer; use codex_config::types::Notifications; @@ -362,6 +361,7 @@ mod slash_dispatch; use self::skills::collect_tool_mentions; use self::skills::find_app_mentions; use self::skills::find_skill_mentions_with_tool_mentions; +use self::skills::is_app_mentionable; mod plugins; use self::plugins::PluginInstallAuthFlowState; use self::plugins::PluginListFetchState; diff --git a/codex-rs/tui/src/chatwidget/connectors.rs b/codex-rs/tui/src/chatwidget/connectors.rs index 2df64df7d7..683bff135b 100644 --- a/codex-rs/tui/src/chatwidget/connectors.rs +++ b/codex-rs/tui/src/chatwidget/connectors.rs @@ -303,13 +303,6 @@ impl ChatWidget { match result { Ok(mut snapshot) => { - if !is_final { - snapshot.connectors = chatgpt_connectors::merge_connectors_with_accessible( - Vec::new(), - snapshot.connectors, - /*all_connectors_loaded*/ false, - ); - } if let ConnectorsCacheState::Ready(existing_snapshot) = &self.connectors.cache { let enabled_by_id: HashMap<&str, bool> = existing_snapshot .connectors diff --git a/codex-rs/tui/src/chatwidget/input_submission.rs b/codex-rs/tui/src/chatwidget/input_submission.rs index 2c960ee3d5..4700ffc0cc 100644 --- a/codex-rs/tui/src/chatwidget/input_submission.rs +++ b/codex-rs/tui/src/chatwidget/input_submission.rs @@ -263,10 +263,14 @@ impl ChatWidget { else { continue; }; - if !selected_app_ids.insert(app_id.to_string()) { + if selected_app_ids.contains(app_id) { continue; } - if let Some(app) = apps.iter().find(|app| app.id == app_id && app.is_enabled) { + if let Some(app) = apps + .iter() + .find(|app| app.id == app_id && is_app_mentionable(app)) + { + selected_app_ids.insert(app_id.to_string()); items.push(UserInput::Mention { name: app.name.clone(), path: binding.path.clone(), diff --git a/codex-rs/tui/src/chatwidget/skills.rs b/codex-rs/tui/src/chatwidget/skills.rs index 356e191b8e..687ce56416 100644 --- a/codex-rs/tui/src/chatwidget/skills.rs +++ b/codex-rs/tui/src/chatwidget/skills.rs @@ -315,12 +315,12 @@ pub(crate) fn find_app_mentions( } let mut slug_counts: HashMap = HashMap::new(); - for app in apps.iter().filter(|app| app.is_enabled) { + for app in apps.iter().filter(|app| is_app_mentionable(app)) { let slug = codex_connectors::metadata::connector_mention_slug(app); *slug_counts.entry(slug).or_insert(0) += 1; } - for app in apps.iter().filter(|app| app.is_enabled) { + for app in apps.iter().filter(|app| is_app_mentionable(app)) { let slug = codex_connectors::metadata::connector_mention_slug(app); let slug_count = slug_counts.get(&slug).copied().unwrap_or(0); if mentions.names.contains(&slug) @@ -333,11 +333,15 @@ pub(crate) fn find_app_mentions( } apps.iter() - .filter(|app| app.is_enabled && selected_ids.contains(&app.id)) + .filter(|app| is_app_mentionable(app) && selected_ids.contains(&app.id)) .cloned() .collect() } +pub(crate) fn is_app_mentionable(app: &AppInfo) -> bool { + app.is_accessible && app.is_enabled +} + pub(crate) struct ToolMentions { names: HashSet, linked_paths: HashMap, @@ -497,3 +501,74 @@ fn app_id_from_path(path: &str) -> Option<&str> { path.strip_prefix("app://") .filter(|value| !value.is_empty()) } + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + + fn app(id: &str, name: &str) -> AppInfo { + AppInfo { + id: id.to_string(), + name: name.to_string(), + description: None, + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + branding: None, + app_metadata: None, + labels: None, + install_url: None, + is_accessible: true, + is_enabled: true, + plugin_display_names: Vec::new(), + } + } + + #[test] + fn find_app_mentions_requires_accessible_enabled_apps_for_slugs() { + let apps = vec![ + app("google_drive", "Google Drive"), + AppInfo { + is_accessible: false, + ..app("arabica_uae", "% Arabica UAE") + }, + AppInfo { + is_enabled: false, + ..app("linear", "Linear") + }, + ]; + let mentions = collect_tool_mentions("$google-drive $arabica-uae $linear", &HashMap::new()); + + assert_eq!( + find_app_mentions(&mentions, &apps, &HashSet::new()), + vec![apps[0].clone()] + ); + } + + #[test] + fn find_app_mentions_requires_accessible_enabled_apps_for_bound_paths() { + let apps = vec![ + app("google_drive", "Google Drive"), + AppInfo { + is_accessible: false, + ..app("arabica_uae", "% Arabica UAE") + }, + AppInfo { + is_enabled: false, + ..app("linear", "Linear") + }, + ]; + let mention_paths = HashMap::from([ + ("google-drive".to_string(), "app://google_drive".to_string()), + ("arabica-uae".to_string(), "app://arabica_uae".to_string()), + ("linear".to_string(), "app://linear".to_string()), + ]); + let mentions = collect_tool_mentions("$google-drive $arabica-uae $linear", &mention_paths); + + assert_eq!( + find_app_mentions(&mentions, &apps, &HashSet::new()), + vec![apps[0].clone()] + ); + } +} diff --git a/codex-rs/tui/src/chatwidget/tests/composer_submission.rs b/codex-rs/tui/src/chatwidget/tests/composer_submission.rs index 70b5c49b0e..bfe563b94c 100644 --- a/codex-rs/tui/src/chatwidget/tests/composer_submission.rs +++ b/codex-rs/tui/src/chatwidget/tests/composer_submission.rs @@ -1,4 +1,5 @@ use super::*; +use crate::app_event::ConnectorsSnapshot; use codex_protocol::models::ManagedFileSystemPermissions; use codex_protocol::permissions::FileSystemAccessMode; use codex_protocol::permissions::FileSystemPath; @@ -1172,6 +1173,61 @@ async fn enqueueing_history_prompt_multiple_times_is_stable() { } } +#[tokio::test] +async fn submit_user_message_ignores_inaccessible_app_mentions_from_bindings() { + let (mut chat, _rx, mut op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + chat.thread_id = Some(ThreadId::new()); + set_chatgpt_auth(&mut chat); + chat.config + .features + .enable(Feature::Apps) + .expect("test config should allow feature update"); + + chat.on_connectors_loaded( + Ok(ConnectorsSnapshot { + connectors: vec![AppInfo { + id: "arabica_uae".to_string(), + name: "% Arabica UAE".to_string(), + description: Some("Directory-only app".to_string()), + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + branding: None, + app_metadata: None, + labels: None, + install_url: Some("https://example.test/arabica".to_string()), + is_accessible: false, + is_enabled: true, + plugin_display_names: Vec::new(), + }], + }), + /*is_final*/ false, + ); + + chat.submit_user_message(UserMessage { + text: "$arabica-uae".to_string(), + local_images: Vec::new(), + remote_image_urls: Vec::new(), + text_elements: Vec::new(), + mention_bindings: vec![MentionBinding { + mention: "arabica-uae".to_string(), + path: "app://arabica_uae".to_string(), + }], + }); + + let items = match next_submit_op(&mut op_rx) { + Op::UserTurn { items, .. } => items, + other => panic!("expected Op::UserTurn, got {other:?}"), + }; + assert_eq!( + items, + vec![UserInput::Text { + text: "$arabica-uae".to_string(), + text_elements: Vec::new(), + }] + ); +} + #[test] fn user_message_display_from_inputs_matches_flattened_user_message_shape() { let local_image = PathBuf::from("/tmp/local.png"); diff --git a/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs b/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs index 09d911ad16..3a4bdd1329 100644 --- a/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs +++ b/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs @@ -1399,6 +1399,77 @@ async fn apps_popup_stays_loading_until_final_snapshot_updates() { ); } +#[tokio::test] +async fn apps_notification_update_excludes_inaccessible_apps_from_mentions() { + let (mut chat, _rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + set_chatgpt_auth(&mut chat); + chat.config + .features + .enable(Feature::Apps) + .expect("test config should allow feature update"); + chat.bottom_pane.set_connectors_enabled(/*enabled*/ true); + chat.bottom_pane + .set_composer_text("$".to_string(), Vec::new(), Vec::new()); + + chat.on_connectors_loaded( + Ok(ConnectorsSnapshot { + connectors: vec![ + AppInfo { + id: "google_drive".to_string(), + name: "Google Drive".to_string(), + description: Some("Connected files".to_string()), + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + branding: None, + app_metadata: None, + labels: None, + install_url: Some("https://example.test/google-drive".to_string()), + is_accessible: true, + is_enabled: true, + plugin_display_names: Vec::new(), + }, + AppInfo { + id: "arabica_uae".to_string(), + name: "% Arabica UAE".to_string(), + description: Some("Directory-only app".to_string()), + logo_url: None, + logo_url_dark: None, + distribution_channel: None, + branding: None, + app_metadata: None, + labels: None, + install_url: Some("https://example.test/arabica".to_string()), + is_accessible: false, + is_enabled: true, + plugin_display_names: Vec::new(), + }, + ], + }), + /*is_final*/ false, + ); + + assert_matches!( + &chat.connectors.partial_snapshot, + Some(snapshot) + if snapshot + .connectors + .iter() + .find(|connector| connector.id == "arabica_uae") + .is_some_and(|connector| !connector.is_accessible) + ); + + let popup = render_bottom_popup(&chat, /*width*/ 80); + assert!( + popup.contains("Google Drive"), + "expected accessible apps to appear in the mention popup, got:\n{popup}" + ); + assert!( + !popup.contains("% Arabica UAE"), + "did not expect an inaccessible directory app in the mention popup, got:\n{popup}" + ); +} + #[tokio::test] async fn apps_refresh_failure_keeps_existing_full_snapshot() { let (mut chat, _rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; From 768848ab6fc6f05f19c447786fd76fd233fa1d69 Mon Sep 17 00:00:00 2001 From: pakrym-oai Date: Tue, 26 May 2026 13:02:34 -0700 Subject: [PATCH 09/16] Add experimental turn additional context (#24154) ## Summary Adds experimental `additionalContext` support to `turn/start` and `turn/steer` so clients can provide ephemeral external context, such as browser or automation state, without turning that plumbing into a visible user prompt or triggering user-prompt lifecycle behavior. ## API Shape The parameter shape is: ```ts additionalContext?: Record | null ``` Example: ```json { "additionalContext": { "browser_info": { "value": "Active tab is CI failures.", "kind": "untrusted" }, "automation_info": { "value": "CI rerun is in progress.", "kind": "application" } } } ``` The keys are opaque and caller-defined. ## Context Injection When provided, accepted entries are inserted into model context as hidden contextual message items, not as visible thread user-message items. `kind: "untrusted"` entries are inserted with role `user`: ```text ${value} ``` `kind: "application"` entries are inserted with role `developer`: ```text <${key}>${value} ``` Values are not escaped. Each value is truncated to 1k approximate tokens before wrapping. For `turn/start`, accepted additional context is inserted before normal user input. For `turn/steer`, additional context is merged only when the steer includes non-empty user input; context-only steers still reject as empty input. ## Dedupe Strategy `AdditionalContextStore` lives on session state and stores the latest complete additional-context map. Each `turn/start` or non-empty `turn/steer` treats its `additionalContext` as the current complete set of values. Entries are injected only when the key is new or the exact entry for that key changed, including `value` or `kind`. After merging, the store is replaced with the provided map, so omitted keys are removed from the retained set and can be injected again later if reintroduced. Omitting `additionalContext`, passing `null`, or passing an empty object resets the store to empty and injects nothing. ## What Changed - Threads experimental v2 `additionalContext` through app-server into core turn start and steer handling. - Adds separate contextual fragment types for untrusted user-role context and application developer-role context. - Uses pending response input items so additional context can be combined with normal user input without treating it as prompt text. - Adds integration coverage for start/steer flow, role routing, dedupe/reset behavior, deletion/re-add behavior, hook-blocked input behavior, empty context-only steer rejection, external-fragment marker matching, and truncation. --- .../analytics/src/analytics_client_tests.rs | 1 + codex-rs/analytics/src/client_tests.rs | 1 + .../schema/json/ClientRequest.json | 22 + .../codex_app_server_protocol.schemas.json | 22 + .../codex_app_server_protocol.v2.schemas.json | 22 + .../schema/json/v2/TurnStartParams.json | 22 + .../schema/json/v2/TurnSteerParams.json | 22 + .../typescript/v2/AdditionalContextEntry.ts | 6 + .../typescript/v2/AdditionalContextKind.ts | 5 + .../schema/typescript/v2/index.ts | 2 + .../src/protocol/v2/tests.rs | 1 + .../src/protocol/v2/turn.rs | 25 + .../src/message_processor_tracing_tests.rs | 1 + codex-rs/app-server/src/request_processors.rs | 3 + .../src/request_processors/turn_processor.rs | 29 + .../tests/suite/v2/client_metadata.rs | 1 + .../app-server/tests/suite/v2/turn_start.rs | 80 +++ .../app-server/tests/suite/v2/turn_steer.rs | 123 ++++ codex-rs/core/src/agent/control_tests.rs | 3 + codex-rs/core/src/codex_delegate.rs | 1 + codex-rs/core/src/codex_thread.rs | 10 +- .../src/context/contextual_user_message.rs | 4 + codex-rs/core/src/context/fragment.rs | 14 + codex-rs/core/src/context/fragments.rs | 92 +++ codex-rs/core/src/context/goal_context.rs | 13 - codex-rs/core/src/context/mod.rs | 3 + codex-rs/core/src/goals.rs | 1 + codex-rs/core/src/guardian/review_session.rs | 1 + codex-rs/core/src/session/handlers.rs | 16 +- codex-rs/core/src/session/input_queue.rs | 6 +- codex-rs/core/src/session/mod.rs | 26 +- codex-rs/core/src/session/review.rs | 4 +- codex-rs/core/src/session/tests.rs | 62 +- codex-rs/core/src/session/turn.rs | 8 +- codex-rs/core/src/state/additional_context.rs | 37 ++ codex-rs/core/src/state/mod.rs | 2 + codex-rs/core/src/state/session.rs | 3 + codex-rs/core/src/tasks/mod.rs | 11 +- .../src/tools/handlers/multi_agents_tests.rs | 1 + codex-rs/core/tests/common/test_codex.rs | 1 + codex-rs/core/tests/suite/abort_tasks.rs | 5 + .../core/tests/suite/additional_context.rs | 581 ++++++++++++++++++ codex-rs/core/tests/suite/apply_patch_cli.rs | 1 + codex-rs/core/tests/suite/approvals.rs | 2 + codex-rs/core/tests/suite/client.rs | 34 + .../core/tests/suite/client_websockets.rs | 2 + codex-rs/core/tests/suite/code_mode.rs | 1 + .../tests/suite/collaboration_instructions.rs | 17 + codex-rs/core/tests/suite/compact.rs | 32 + codex-rs/core/tests/suite/compact_remote.rs | 55 ++ .../core/tests/suite/compact_remote_parity.rs | 1 + .../core/tests/suite/compact_resume_fork.rs | 1 + codex-rs/core/tests/suite/exec_policy.rs | 2 + codex-rs/core/tests/suite/fork_thread.rs | 2 + codex-rs/core/tests/suite/hooks.rs | 2 + codex-rs/core/tests/suite/image_rollout.rs | 2 + codex-rs/core/tests/suite/items.rs | 10 + codex-rs/core/tests/suite/json_result.rs | 1 + .../core/tests/suite/mcp_turn_metadata.rs | 1 + codex-rs/core/tests/suite/mod.rs | 1 + codex-rs/core/tests/suite/model_switching.rs | 1 + .../core/tests/suite/model_visible_layout.rs | 8 + codex-rs/core/tests/suite/models_cache_ttl.rs | 1 + .../core/tests/suite/models_etag_responses.rs | 1 + codex-rs/core/tests/suite/otel.rs | 22 + codex-rs/core/tests/suite/pending_input.rs | 5 + .../core/tests/suite/permissions_messages.rs | 15 + codex-rs/core/tests/suite/personality.rs | 1 + codex-rs/core/tests/suite/plugins.rs | 3 + codex-rs/core/tests/suite/prompt_caching.rs | 15 + codex-rs/core/tests/suite/quota_exceeded.rs | 1 + .../core/tests/suite/realtime_conversation.rs | 3 + codex-rs/core/tests/suite/remote_env.rs | 1 + codex-rs/core/tests/suite/remote_models.rs | 7 + .../core/tests/suite/request_compression.rs | 2 + .../core/tests/suite/request_permissions.rs | 1 + .../tests/suite/request_permissions_tool.rs | 1 + .../core/tests/suite/request_user_input.rs | 3 + .../suite/responses_api_proxy_headers.rs | 1 + codex-rs/core/tests/suite/resume.rs | 7 + codex-rs/core/tests/suite/review.rs | 1 + codex-rs/core/tests/suite/rmcp_client.rs | 1 + .../tests/suite/safety_check_downgrade.rs | 1 + codex-rs/core/tests/suite/search_tool.rs | 4 + codex-rs/core/tests/suite/shell_snapshot.rs | 4 + codex-rs/core/tests/suite/skill_approval.rs | 1 + codex-rs/core/tests/suite/skills.rs | 1 + ...text__additional_context_simple_input.snap | 11 + codex-rs/core/tests/suite/sqlite_state.rs | 1 + .../suite/stream_error_allows_next_turn.rs | 2 + .../core/tests/suite/stream_no_completed.rs | 1 + .../tests/suite/subagent_notifications.rs | 1 + codex-rs/core/tests/suite/tool_harness.rs | 5 + codex-rs/core/tests/suite/tool_parallelism.rs | 2 + codex-rs/core/tests/suite/truncation.rs | 1 + codex-rs/core/tests/suite/unified_exec.rs | 7 + .../core/tests/suite/user_notification.rs | 1 + codex-rs/core/tests/suite/user_shell_cmd.rs | 1 + codex-rs/core/tests/suite/view_image.rs | 1 + .../core/tests/suite/websocket_fallback.rs | 1 + codex-rs/core/tests/suite/window_headers.rs | 1 + codex-rs/exec/src/lib.rs | 1 + codex-rs/ext/goal/src/steering.rs | 1 + codex-rs/mcp-server/src/codex_tool_runner.rs | 2 + codex-rs/memories/write/src/runtime.rs | 1 + codex-rs/protocol/src/protocol.rs | 24 + codex-rs/thread-manager-sample/src/main.rs | 1 + codex-rs/tui/src/app_server_session.rs | 2 + 108 files changed, 1583 insertions(+), 57 deletions(-) create mode 100644 codex-rs/app-server-protocol/schema/typescript/v2/AdditionalContextEntry.ts create mode 100644 codex-rs/app-server-protocol/schema/typescript/v2/AdditionalContextKind.ts create mode 100644 codex-rs/core/src/context/fragments.rs create mode 100644 codex-rs/core/src/state/additional_context.rs create mode 100644 codex-rs/core/tests/suite/additional_context.rs create mode 100644 codex-rs/core/tests/suite/snapshots/all__suite__additional_context__additional_context_simple_input.snap diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs index cabea80886..26daadf5f5 100644 --- a/codex-rs/analytics/src/analytics_client_tests.rs +++ b/codex-rs/analytics/src/analytics_client_tests.rs @@ -401,6 +401,7 @@ fn sample_turn_steer_request( }, ], responsesapi_client_metadata: None, + additional_context: None, }, } } diff --git a/codex-rs/analytics/src/client_tests.rs b/codex-rs/analytics/src/client_tests.rs index bfb224d899..c6a928afa3 100644 --- a/codex-rs/analytics/src/client_tests.rs +++ b/codex-rs/analytics/src/client_tests.rs @@ -103,6 +103,7 @@ fn sample_turn_steer_request() -> ClientRequest { expected_turn_id: "turn-1".to_string(), input: Vec::new(), responsesapi_client_metadata: None, + additional_context: None, }, } } diff --git a/codex-rs/app-server-protocol/schema/json/ClientRequest.json b/codex-rs/app-server-protocol/schema/json/ClientRequest.json index d2bec3d2ed..552defad1b 100644 --- a/codex-rs/app-server-protocol/schema/json/ClientRequest.json +++ b/codex-rs/app-server-protocol/schema/json/ClientRequest.json @@ -12,6 +12,28 @@ ], "type": "string" }, + "AdditionalContextEntry": { + "properties": { + "kind": { + "$ref": "#/definitions/AdditionalContextKind" + }, + "value": { + "type": "string" + } + }, + "required": [ + "kind", + "value" + ], + "type": "object" + }, + "AdditionalContextKind": { + "enum": [ + "untrusted", + "application" + ], + "type": "string" + }, "ApprovalsReviewer": { "description": "Configures who approval requests are routed to for review. Examples include sandbox escapes, blocked network access, MCP approval prompts, and ARC escalations. Defaults to `user`. `auto_review` uses a carefully prompted subagent to gather relevant context and apply a risk-based decision framework before approving or denying the request. The legacy value `guardian_subagent` is accepted for compatibility.", "enum": [ diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index 4c6d5a6eee..ab19f3e770 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -5768,6 +5768,28 @@ ], "type": "string" }, + "AdditionalContextEntry": { + "properties": { + "kind": { + "$ref": "#/definitions/v2/AdditionalContextKind" + }, + "value": { + "type": "string" + } + }, + "required": [ + "kind", + "value" + ], + "type": "object" + }, + "AdditionalContextKind": { + "enum": [ + "untrusted", + "application" + ], + "type": "string" + }, "AdditionalFileSystemPermissions": { "properties": { "entries": { diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index e74323745e..b12b0eedd8 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -164,6 +164,28 @@ ], "type": "string" }, + "AdditionalContextEntry": { + "properties": { + "kind": { + "$ref": "#/definitions/AdditionalContextKind" + }, + "value": { + "type": "string" + } + }, + "required": [ + "kind", + "value" + ], + "type": "object" + }, + "AdditionalContextKind": { + "enum": [ + "untrusted", + "application" + ], + "type": "string" + }, "AdditionalFileSystemPermissions": { "properties": { "entries": { diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json index ecea8f1997..ffc130dcc9 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json @@ -5,6 +5,28 @@ "description": "A path that is guaranteed to be absolute and normalized (though it is not guaranteed to be canonicalized or exist on the filesystem).\n\nIMPORTANT: When deserializing an `AbsolutePathBuf`, a base path must be set using [AbsolutePathBufGuard::new]. If no base path is set, the deserialization will fail unless the path being deserialized is already absolute.", "type": "string" }, + "AdditionalContextEntry": { + "properties": { + "kind": { + "$ref": "#/definitions/AdditionalContextKind" + }, + "value": { + "type": "string" + } + }, + "required": [ + "kind", + "value" + ], + "type": "object" + }, + "AdditionalContextKind": { + "enum": [ + "untrusted", + "application" + ], + "type": "string" + }, "ApprovalsReviewer": { "description": "Configures who approval requests are routed to for review. Examples include sandbox escapes, blocked network access, MCP approval prompts, and ARC escalations. Defaults to `user`. `auto_review` uses a carefully prompted subagent to gather relevant context and apply a risk-based decision framework before approving or denying the request. The legacy value `guardian_subagent` is accepted for compatibility.", "enum": [ diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnSteerParams.json b/codex-rs/app-server-protocol/schema/json/v2/TurnSteerParams.json index 1b7cfbf400..ef05b27673 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnSteerParams.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnSteerParams.json @@ -1,6 +1,28 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "definitions": { + "AdditionalContextEntry": { + "properties": { + "kind": { + "$ref": "#/definitions/AdditionalContextKind" + }, + "value": { + "type": "string" + } + }, + "required": [ + "kind", + "value" + ], + "type": "object" + }, + "AdditionalContextKind": { + "enum": [ + "untrusted", + "application" + ], + "type": "string" + }, "ByteRange": { "properties": { "end": { diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/AdditionalContextEntry.ts b/codex-rs/app-server-protocol/schema/typescript/v2/AdditionalContextEntry.ts new file mode 100644 index 0000000000..8d959269d4 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/AdditionalContextEntry.ts @@ -0,0 +1,6 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. +import type { AdditionalContextKind } from "./AdditionalContextKind"; + +export type AdditionalContextEntry = { value: string, kind: AdditionalContextKind, }; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/AdditionalContextKind.ts b/codex-rs/app-server-protocol/schema/typescript/v2/AdditionalContextKind.ts new file mode 100644 index 0000000000..cd60bd7a8d --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/AdditionalContextKind.ts @@ -0,0 +1,5 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type AdditionalContextKind = "untrusted" | "application"; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/index.ts b/codex-rs/app-server-protocol/schema/typescript/v2/index.ts index 5b4f2ed283..2a4220cb60 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/index.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/index.ts @@ -7,6 +7,8 @@ export type { AccountUpdatedNotification } from "./AccountUpdatedNotification"; export type { ActivePermissionProfile } from "./ActivePermissionProfile"; export type { AddCreditsNudgeCreditType } from "./AddCreditsNudgeCreditType"; export type { AddCreditsNudgeEmailStatus } from "./AddCreditsNudgeEmailStatus"; +export type { AdditionalContextEntry } from "./AdditionalContextEntry"; +export type { AdditionalContextKind } from "./AdditionalContextKind"; export type { AdditionalFileSystemPermissions } from "./AdditionalFileSystemPermissions"; export type { AdditionalNetworkPermissions } from "./AdditionalNetworkPermissions"; export type { AdditionalPermissionProfile } from "./AdditionalPermissionProfile"; diff --git a/codex-rs/app-server-protocol/src/protocol/v2/tests.rs b/codex-rs/app-server-protocol/src/protocol/v2/tests.rs index 8f6824f397..c2b4f24b51 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/tests.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/tests.rs @@ -3428,6 +3428,7 @@ fn turn_start_params_preserve_explicit_null_service_tier() { thread_id: "thread_123".to_string(), input: vec![], responsesapi_client_metadata: None, + additional_context: None, environments: None, cwd: None, runtime_workspace_roots: None, diff --git a/codex-rs/app-server-protocol/src/protocol/v2/turn.rs b/codex-rs/app-server-protocol/src/protocol/v2/turn.rs index 50660af1d7..ab5e59a464 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/turn.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/turn.rs @@ -41,6 +41,23 @@ pub struct TurnEnvironmentParams { pub cwd: AbsolutePathBuf, } +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "lowercase")] +#[ts(rename_all = "lowercase")] +#[ts(export_to = "v2/")] +pub enum AdditionalContextKind { + Untrusted, + Application, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(export_to = "v2/")] +pub struct AdditionalContextEntry { + pub value: String, + pub kind: AdditionalContextKind, +} + #[derive( Serialize, Deserialize, Debug, Default, Clone, PartialEq, JsonSchema, TS, ExperimentalApi, )] @@ -53,6 +70,10 @@ pub struct TurnStartParams { #[experimental("turn/start.responsesapiClientMetadata")] #[ts(optional = nullable)] pub responsesapi_client_metadata: Option>, + /// Optional client-provided context fragments keyed by an opaque source identifier. + #[experimental("turn/start.additionalContext")] + #[ts(optional = nullable)] + pub additional_context: Option>, /// Optional turn-scoped environments. /// /// Omitted uses the thread sticky environments. Empty disables @@ -141,6 +162,10 @@ pub struct TurnSteerParams { #[experimental("turn/steer.responsesapiClientMetadata")] #[ts(optional = nullable)] pub responsesapi_client_metadata: Option>, + /// Optional client-provided context fragments keyed by an opaque source identifier. + #[experimental("turn/steer.additionalContext")] + #[ts(optional = nullable)] + pub additional_context: Option>, /// Required active turn id precondition. The request fails when it does not /// match the currently active turn. pub expected_turn_id: String, diff --git a/codex-rs/app-server/src/message_processor_tracing_tests.rs b/codex-rs/app-server/src/message_processor_tracing_tests.rs index a9625d3086..97c497b877 100644 --- a/codex-rs/app-server/src/message_processor_tracing_tests.rs +++ b/codex-rs/app-server/src/message_processor_tracing_tests.rs @@ -658,6 +658,7 @@ async fn turn_start_jsonrpc_span_parents_core_turn_spans() -> Result<()> { text_elements: Vec::new(), }], responsesapi_client_metadata: None, + additional_context: None, cwd: None, runtime_workspace_roots: None, approval_policy: None, diff --git a/codex-rs/app-server/src/request_processors.rs b/codex-rs/app-server/src/request_processors.rs index d0958e80ab..66ab8e3a1d 100644 --- a/codex-rs/app-server/src/request_processors.rs +++ b/codex-rs/app-server/src/request_processors.rs @@ -25,6 +25,8 @@ use codex_app_server_protocol::AccountLoginCompletedNotification; use codex_app_server_protocol::AccountUpdatedNotification; use codex_app_server_protocol::AddCreditsNudgeCreditType; use codex_app_server_protocol::AddCreditsNudgeEmailStatus; +use codex_app_server_protocol::AdditionalContextEntry; +use codex_app_server_protocol::AdditionalContextKind; use codex_app_server_protocol::AppInfo; use codex_app_server_protocol::AppListUpdatedNotification; use codex_app_server_protocol::AppSummary; @@ -421,6 +423,7 @@ use codex_thread_store::ThreadStore; use codex_thread_store::ThreadStoreError; use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_pty::DEFAULT_OUTPUT_BYTES_CAP; +use std::collections::BTreeMap; use std::collections::HashMap; use std::collections::HashSet; use std::io::Error as IoError; diff --git a/codex-rs/app-server/src/request_processors/turn_processor.rs b/codex-rs/app-server/src/request_processors/turn_processor.rs index ff2825be67..975ff22d07 100644 --- a/codex-rs/app-server/src/request_processors/turn_processor.rs +++ b/codex-rs/app-server/src/request_processors/turn_processor.rs @@ -1,4 +1,6 @@ use super::*; +use codex_protocol::protocol::AdditionalContextEntry as CoreAdditionalContextEntry; +use codex_protocol::protocol::AdditionalContextKind as CoreAdditionalContextKind; #[derive(Clone)] pub(crate) struct TurnRequestProcessor { @@ -30,6 +32,29 @@ fn resolve_runtime_workspace_roots( resolved_roots } +fn map_additional_context( + additional_context: Option>, +) -> BTreeMap { + additional_context + .unwrap_or_default() + .into_iter() + .map(|(key, entry)| { + ( + key, + CoreAdditionalContextEntry { + value: entry.value, + kind: match entry.kind { + AdditionalContextKind::Untrusted => CoreAdditionalContextKind::Untrusted, + AdditionalContextKind::Application => { + CoreAdditionalContextKind::Application + } + }, + }, + ) + }) + .collect() +} + struct ThreadSettingsBuildParams { method: &'static str, cwd: Option, @@ -391,6 +416,7 @@ impl TurnRequestProcessor { .into_iter() .map(V2UserInput::into_core) .collect(); + let additional_context = map_additional_context(params.additional_context); let turn_has_input = !mapped_items.is_empty(); let thread_settings = self .build_thread_settings_overrides( @@ -419,6 +445,7 @@ impl TurnRequestProcessor { environments: environment_selections, final_output_json_schema: params.output_schema, responsesapi_client_metadata: params.responsesapi_client_metadata, + additional_context, thread_settings, }; let turn_id = self @@ -746,10 +773,12 @@ impl TurnRequestProcessor { .into_iter() .map(V2UserInput::into_core) .collect(); + let additional_context = map_additional_context(params.additional_context); let turn_id = thread .steer_input( mapped_items, + additional_context, Some(¶ms.expected_turn_id), params.responsesapi_client_metadata, ) diff --git a/codex-rs/app-server/tests/suite/v2/client_metadata.rs b/codex-rs/app-server/tests/suite/v2/client_metadata.rs index 8d68888e7e..80f41d7b11 100644 --- a/codex-rs/app-server/tests/suite/v2/client_metadata.rs +++ b/codex-rs/app-server/tests/suite/v2/client_metadata.rs @@ -179,6 +179,7 @@ async fn turn_steer_updates_client_metadata_on_follow_up_responses_request_v2() text_elements: Vec::new(), }], responsesapi_client_metadata: Some(steer_metadata.clone()), + additional_context: None, expected_turn_id: turn_id.clone(), }) .await?; diff --git a/codex-rs/app-server/tests/suite/v2/turn_start.rs b/codex-rs/app-server/tests/suite/v2/turn_start.rs index 236d57a0ee..88596b4639 100644 --- a/codex-rs/app-server/tests/suite/v2/turn_start.rs +++ b/codex-rs/app-server/tests/suite/v2/turn_start.rs @@ -16,6 +16,8 @@ use app_test_support::write_mock_responses_config_toml_with_chatgpt_base_url; use app_test_support::write_models_cache; use codex_app_server::INPUT_TOO_LARGE_ERROR_CODE; use codex_app_server::INVALID_PARAMS_ERROR_CODE; +use codex_app_server_protocol::AdditionalContextEntry; +use codex_app_server_protocol::AdditionalContextKind; use codex_app_server_protocol::ByteRange; use codex_app_server_protocol::ClientInfo; use codex_app_server_protocol::CollabAgentStatus; @@ -301,6 +303,82 @@ async fn turn_start_with_empty_input_runs_model_request() -> Result<()> { Ok(()) } +#[tokio::test] +async fn turn_start_additional_context_flows_to_model_input() -> Result<()> { + let responses = vec![create_final_assistant_message_sse_response("Done")?]; + let server = create_mock_responses_server_sequence_unchecked(responses).await; + + let codex_home = TempDir::new()?; + create_config_toml( + codex_home.path(), + &server.uri(), + "never", + &BTreeMap::default(), + )?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let thread_req = mcp + .send_thread_start_request(ThreadStartParams { + model: Some("mock-model".to_string()), + ..Default::default() + }) + .await?; + let thread_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(thread_req)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response::(thread_resp)?; + + let turn_req = mcp + .send_turn_start_request(TurnStartParams { + thread_id: thread.id, + input: vec![V2UserInput::Text { + text: "inspect tab".to_string(), + text_elements: Vec::new(), + }], + additional_context: Some(HashMap::from([( + "custom_source".to_string(), + AdditionalContextEntry { + value: "source value".to_string(), + kind: AdditionalContextKind::Untrusted, + }, + )])), + ..Default::default() + }) + .await?; + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(turn_req)), + ) + .await??; + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("turn/completed"), + ) + .await??; + + let requests = server + .received_requests() + .await + .context("failed to fetch received requests")?; + let request = requests + .iter() + .find(|request| request.url.path().ends_with("/responses")) + .context("expected model request")?; + let body = request + .body_json::() + .context("request body should be JSON")?; + assert!( + body.to_string() + .contains("source value") + ); + + Ok(()) +} + #[tokio::test] async fn turn_start_sends_originator_header() -> Result<()> { let responses = vec![create_final_assistant_message_sse_response("Done")?]; @@ -2069,6 +2147,7 @@ async fn turn_start_updates_sandbox_and_cwd_between_turns_v2() -> Result<()> { text_elements: Vec::new(), }], responsesapi_client_metadata: None, + additional_context: None, cwd: Some(first_cwd.clone()), runtime_workspace_roots: None, approval_policy: Some(codex_app_server_protocol::AskForApproval::Never), @@ -2111,6 +2190,7 @@ async fn turn_start_updates_sandbox_and_cwd_between_turns_v2() -> Result<()> { text_elements: Vec::new(), }], responsesapi_client_metadata: None, + additional_context: None, cwd: Some(second_cwd.clone()), runtime_workspace_roots: None, approval_policy: Some(codex_app_server_protocol::AskForApproval::Never), diff --git a/codex-rs/app-server/tests/suite/v2/turn_steer.rs b/codex-rs/app-server/tests/suite/v2/turn_steer.rs index a92b2db528..66ac33972b 100644 --- a/codex-rs/app-server/tests/suite/v2/turn_steer.rs +++ b/codex-rs/app-server/tests/suite/v2/turn_steer.rs @@ -1,5 +1,6 @@ #![cfg(unix)] +use anyhow::Context; use anyhow::Result; use app_test_support::McpProcess; use app_test_support::create_mock_responses_server_sequence; @@ -9,6 +10,8 @@ use app_test_support::to_response; use app_test_support::write_mock_responses_config_toml_with_chatgpt_base_url; use codex_app_server::INPUT_TOO_LARGE_ERROR_CODE; use codex_app_server::INVALID_PARAMS_ERROR_CODE; +use codex_app_server_protocol::AdditionalContextEntry; +use codex_app_server_protocol::AdditionalContextKind; use codex_app_server_protocol::JSONRPCError; use codex_app_server_protocol::JSONRPCNotification; use codex_app_server_protocol::JSONRPCResponse; @@ -21,6 +24,8 @@ use codex_app_server_protocol::TurnSteerParams; use codex_app_server_protocol::TurnSteerResponse; use codex_app_server_protocol::UserInput as V2UserInput; use codex_protocol::user_input::MAX_USER_INPUT_TEXT_CHARS; +use serde_json::Value; +use std::collections::HashMap; use tempfile::TempDir; use tokio::time::timeout; @@ -67,6 +72,7 @@ async fn turn_steer_requires_active_turn() -> Result<()> { text_elements: Vec::new(), }], responsesapi_client_metadata: None, + additional_context: None, expected_turn_id: "turn-does-not-exist".to_string(), }) .await?; @@ -176,6 +182,7 @@ async fn turn_steer_rejects_oversized_text_input() -> Result<()> { text_elements: Vec::new(), }], responsesapi_client_metadata: None, + additional_context: None, expected_turn_id: turn.id.clone(), }) .await?; @@ -284,6 +291,7 @@ async fn turn_steer_returns_active_turn_id() -> Result<()> { text_elements: Vec::new(), }], responsesapi_client_metadata: None, + additional_context: None, expected_turn_id: turn.id.clone(), }) .await?; @@ -312,3 +320,118 @@ async fn turn_steer_returns_active_turn_id() -> Result<()> { Ok(()) } + +#[tokio::test] +async fn turn_steer_rejects_context_only_input_without_merging_context() -> Result<()> { + let tmp = TempDir::new()?; + let codex_home = tmp.path().join("codex_home"); + std::fs::create_dir(&codex_home)?; + let working_directory = tmp.path().join("workdir"); + std::fs::create_dir(&working_directory)?; + + let server = create_mock_responses_server_sequence_unchecked(vec![ + create_shell_command_sse_response( + vec!["sleep".to_string(), "1".to_string()], + Some(&working_directory), + Some(10_000), + "call_sleep", + )?, + app_test_support::create_final_assistant_message_sse_response("Done")?, + ]) + .await; + write_mock_responses_config_toml_with_chatgpt_base_url( + &codex_home, + &server.uri(), + &server.uri(), + )?; + mount_analytics_capture(&server, &codex_home).await?; + + let mut mcp = McpProcess::new_without_managed_config(&codex_home).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let thread_req = mcp + .send_thread_start_request(ThreadStartParams { + model: Some("mock-model".to_string()), + ..Default::default() + }) + .await?; + let thread_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(thread_req)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response::(thread_resp)?; + + let turn_req = mcp + .send_turn_start_request(TurnStartParams { + thread_id: thread.id.clone(), + input: vec![V2UserInput::Text { + text: "run sleep".to_string(), + text_elements: Vec::new(), + }], + cwd: Some(working_directory), + ..Default::default() + }) + .await?; + let turn_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(turn_req)), + ) + .await??; + let TurnStartResponse { turn } = to_response::(turn_resp)?; + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("turn/started"), + ) + .await??; + + let additional_context = Some(HashMap::from([( + "browser_info".to_string(), + AdditionalContextEntry { + value: "tab one".to_string(), + kind: AdditionalContextKind::Untrusted, + }, + )])); + let steer_req = mcp + .send_turn_steer_request(TurnSteerParams { + thread_id: thread.id.clone(), + input: Vec::new(), + responsesapi_client_metadata: None, + additional_context, + expected_turn_id: turn.id, + }) + .await?; + let steer_error: JSONRPCError = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_error_message(RequestId::Integer(steer_req)), + ) + .await??; + assert_eq!(steer_error.error.code, -32600); + assert_eq!(steer_error.error.message, "input must not be empty"); + + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("turn/completed"), + ) + .await??; + + let requests = server + .received_requests() + .await + .context("failed to fetch received requests")?; + let response_requests = requests + .iter() + .filter(|request| request.url.path().ends_with("/responses")) + .collect::>(); + assert_eq!(response_requests.len(), 2); + let body = response_requests[1] + .body_json::() + .context("request body should be JSON")?; + assert!( + !body + .to_string() + .contains("tab one") + ); + + Ok(()) +} diff --git a/codex-rs/core/src/agent/control_tests.rs b/codex-rs/core/src/agent/control_tests.rs index ed39df1548..8209272f10 100644 --- a/codex-rs/core/src/agent/control_tests.rs +++ b/codex-rs/core/src/agent/control_tests.rs @@ -444,6 +444,7 @@ async fn send_input_submits_user_message() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }, ); @@ -598,6 +599,7 @@ async fn spawn_agent_creates_thread_and_sends_prompt() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }, ); @@ -770,6 +772,7 @@ async fn spawn_agent_can_fork_parent_thread_history_with_sanitized_items() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }, ); diff --git a/codex-rs/core/src/codex_delegate.rs b/codex-rs/core/src/codex_delegate.rs index 4ed6b2368c..b27cf745c9 100644 --- a/codex-rs/core/src/codex_delegate.rs +++ b/codex-rs/core/src/codex_delegate.rs @@ -192,6 +192,7 @@ pub(crate) async fn run_codex_thread_one_shot( items: input, final_output_json_schema, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; diff --git a/codex-rs/core/src/codex_thread.rs b/codex-rs/core/src/codex_thread.rs index 4b77efdac9..1113a12c8a 100644 --- a/codex-rs/core/src/codex_thread.rs +++ b/codex-rs/core/src/codex_thread.rs @@ -21,6 +21,7 @@ use codex_protocol::models::PermissionProfile; use codex_protocol::models::ResponseInputItem; use codex_protocol::models::ResponseItem; use codex_protocol::openai_models::ReasoningEffort; +use codex_protocol::protocol::AdditionalContextEntry; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::Event; use codex_protocol::protocol::Op; @@ -41,6 +42,7 @@ use codex_thread_store::ThreadStoreError; use codex_thread_store::ThreadStoreResult; use codex_utils_absolute_path::AbsolutePathBuf; use rmcp::model::ReadResourceRequestParams; +use std::collections::BTreeMap; use std::collections::HashMap; use std::path::PathBuf; use std::sync::Arc; @@ -236,11 +238,17 @@ impl CodexThread { pub async fn steer_input( &self, input: Vec, + additional_context: BTreeMap, expected_turn_id: Option<&str>, responsesapi_client_metadata: Option>, ) -> Result { self.codex - .steer_input(input, expected_turn_id, responsesapi_client_metadata) + .steer_input( + input, + additional_context, + expected_turn_id, + responsesapi_client_metadata, + ) .await } diff --git a/codex-rs/core/src/context/contextual_user_message.rs b/codex-rs/core/src/context/contextual_user_message.rs index c7ada2317f..940a938fce 100644 --- a/codex-rs/core/src/context/contextual_user_message.rs +++ b/codex-rs/core/src/context/contextual_user_message.rs @@ -2,6 +2,7 @@ use codex_protocol::items::HookPromptItem; use codex_protocol::items::parse_hook_prompt_fragment; use codex_protocol::models::ContentItem; +use super::AdditionalContextUserFragment; use super::EnvironmentContext; use super::FragmentRegistration; use super::FragmentRegistrationProxy; @@ -19,6 +20,8 @@ static USER_INSTRUCTIONS_REGISTRATION: FragmentRegistrationProxy = FragmentRegistrationProxy::new(); +static ADDITIONAL_CONTEXT_REGISTRATION: FragmentRegistrationProxy = + FragmentRegistrationProxy::new(); static SKILL_INSTRUCTIONS_REGISTRATION: FragmentRegistrationProxy = FragmentRegistrationProxy::new(); static USER_SHELL_COMMAND_REGISTRATION: FragmentRegistrationProxy = @@ -42,6 +45,7 @@ static LEGACY_MODEL_MISMATCH_WARNING_REGISTRATION: FragmentRegistrationProxy< static CONTEXTUAL_USER_FRAGMENTS: &[&dyn FragmentRegistration] = &[ &USER_INSTRUCTIONS_REGISTRATION, &ENVIRONMENT_CONTEXT_REGISTRATION, + &ADDITIONAL_CONTEXT_REGISTRATION, &SKILL_INSTRUCTIONS_REGISTRATION, &USER_SHELL_COMMAND_REGISTRATION, &TURN_ABORTED_REGISTRATION, diff --git a/codex-rs/core/src/context/fragment.rs b/codex-rs/core/src/context/fragment.rs index 4a9e284d12..5c500fe8ef 100644 --- a/codex-rs/core/src/context/fragment.rs +++ b/codex-rs/core/src/context/fragment.rs @@ -1,4 +1,5 @@ use codex_protocol::models::ContentItem; +use codex_protocol::models::ResponseInputItem; use codex_protocol::models::ResponseItem; use std::marker::PhantomData; @@ -81,6 +82,19 @@ pub trait ContextualUserFragment { phase: None, } } + + fn into_response_input_item(self) -> ResponseInputItem + where + Self: Sized, + { + ResponseInputItem::Message { + role: Self::role().to_string(), + content: vec![ContentItem::InputText { + text: self.render(), + }], + phase: None, + } + } } fn matches_marked_text(start_marker: &str, end_marker: &str, text: &str) -> bool { diff --git a/codex-rs/core/src/context/fragments.rs b/codex-rs/core/src/context/fragments.rs new file mode 100644 index 0000000000..e23f96f59f --- /dev/null +++ b/codex-rs/core/src/context/fragments.rs @@ -0,0 +1,92 @@ +use super::ContextualUserFragment; +use codex_utils_string::truncate_middle_with_token_budget; + +const MAX_ADDITIONAL_CONTEXT_VALUE_TOKENS: usize = 1_000; +const ADDITIONAL_CONTEXT_END_MARKER_SUFFIX: &str = ">"; +const ADDITIONAL_CONTEXT_START_MARKER_PREFIX: &str = " Self { + Self { key, value } + } +} + +impl ContextualUserFragment for AdditionalContextUserFragment { + fn role() -> &'static str { + "user" + } + + fn markers(&self) -> (&'static str, &'static str) { + Self::type_markers() + } + + fn type_markers() -> (&'static str, &'static str) { + ( + ADDITIONAL_CONTEXT_START_MARKER_PREFIX, + ADDITIONAL_CONTEXT_END_MARKER_SUFFIX, + ) + } + + fn matches_text(text: &str) -> bool { + let trimmed = text.trim(); + let Some(rest) = trimmed.strip_prefix(ADDITIONAL_CONTEXT_START_MARKER_PREFIX) else { + return false; + }; + let Some((key, value_and_close)) = rest.split_once(ADDITIONAL_CONTEXT_END_MARKER_SUFFIX) + else { + return false; + }; + + value_and_close.ends_with(&format!("")) + } + + fn body(&self) -> String { + additional_context_body(&self.key, &self.value) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct AdditionalContextDeveloperFragment { + key: String, + value: String, +} + +impl AdditionalContextDeveloperFragment { + pub(crate) fn new(key: String, value: String) -> Self { + Self { key, value } + } +} + +impl ContextualUserFragment for AdditionalContextDeveloperFragment { + fn role() -> &'static str { + "developer" + } + + fn markers(&self) -> (&'static str, &'static str) { + Self::type_markers() + } + + fn type_markers() -> (&'static str, &'static str) { + ("", "") + } + + fn body(&self) -> String { + additional_context_developer_body(&self.key, &self.value) + } +} + +fn additional_context_body(key: &str, value: &str) -> String { + let value = truncate_middle_with_token_budget(value, MAX_ADDITIONAL_CONTEXT_VALUE_TOKENS).0; + format!("{key}>{value} String { + let value = truncate_middle_with_token_budget(value, MAX_ADDITIONAL_CONTEXT_VALUE_TOKENS).0; + format!("<{key}>{value}") +} diff --git a/codex-rs/core/src/context/goal_context.rs b/codex-rs/core/src/context/goal_context.rs index 13892bf466..ad4c47ef06 100644 --- a/codex-rs/core/src/context/goal_context.rs +++ b/codex-rs/core/src/context/goal_context.rs @@ -1,8 +1,6 @@ //! Hidden user-context fragment for runtime-owned goal steering prompts. use super::ContextualUserFragment; -use codex_protocol::models::ContentItem; -use codex_protocol::models::ResponseInputItem; /// Hidden runtime-owned goal steering context injected into model input. #[derive(Debug, Clone, PartialEq)] @@ -17,17 +15,6 @@ impl GoalContext { prompt: prompt.into(), } } - - /// Converts the registered fragment into an active-turn injectable item. - pub fn into_response_input_item(self) -> ResponseInputItem { - ResponseInputItem::Message { - role: ::role().to_string(), - content: vec![ContentItem::InputText { - text: self.render(), - }], - phase: None, - } - } } impl ContextualUserFragment for GoalContext { diff --git a/codex-rs/core/src/context/mod.rs b/codex-rs/core/src/context/mod.rs index 99575f6309..6d88df87e2 100644 --- a/codex-rs/core/src/context/mod.rs +++ b/codex-rs/core/src/context/mod.rs @@ -8,6 +8,7 @@ mod collaboration_mode_instructions; mod contextual_user_message; mod environment_context; mod fragment; +mod fragments; mod goal_context; mod guardian_followup_review_reminder; mod hook_additional_context; @@ -40,6 +41,8 @@ pub(crate) use environment_context::EnvironmentContext; pub use fragment::ContextualUserFragment; pub(crate) use fragment::FragmentRegistration; pub(crate) use fragment::FragmentRegistrationProxy; +pub(crate) use fragments::AdditionalContextDeveloperFragment; +pub(crate) use fragments::AdditionalContextUserFragment; pub use goal_context::GoalContext; pub(crate) use guardian_followup_review_reminder::GuardianFollowupReviewReminder; pub(crate) use hook_additional_context::HookAdditionalContext; diff --git a/codex-rs/core/src/goals.rs b/codex-rs/core/src/goals.rs index 75b84093e7..6df681d4b0 100644 --- a/codex-rs/core/src/goals.rs +++ b/codex-rs/core/src/goals.rs @@ -5,6 +5,7 @@ //! events, and owns helper hooks used by goal lifecycle behavior. use crate::StateDbHandle; +use crate::context::ContextualUserFragment; use crate::context::GoalContext; use crate::session::TurnInput; use crate::session::session::Session; diff --git a/codex-rs/core/src/guardian/review_session.rs b/codex-rs/core/src/guardian/review_session.rs index 15d6c142ca..e56d5ce170 100644 --- a/codex-rs/core/src/guardian/review_session.rs +++ b/codex-rs/core/src/guardian/review_session.rs @@ -714,6 +714,7 @@ async fn run_review_on_session( environments: None, final_output_json_schema: Some(params.schema.clone()), responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { #[allow(deprecated)] cwd: Some(params.parent_turn.cwd.to_path_buf()), diff --git a/codex-rs/core/src/session/handlers.rs b/codex-rs/core/src/session/handlers.rs index b1e36b0347..cf46ecba9a 100644 --- a/codex-rs/core/src/session/handlers.rs +++ b/codex-rs/core/src/session/handlers.rs @@ -10,6 +10,7 @@ use tracing::debug_span; use tracing::info_span; use crate::session::SteerInputError; +use crate::session::TurnInput; use crate::session::session::Session; use crate::session::session::SessionSettingsUpdate; @@ -194,6 +195,7 @@ pub(super) async fn user_input_or_turn_inner( environments, final_output_json_schema, responsesapi_client_metadata, + additional_context, thread_settings, } = op else { @@ -224,6 +226,7 @@ pub(super) async fn user_input_or_turn_inner( let accepted_items = match sess .steer_input( items.clone(), + additional_context.clone(), /*expected_turn_id*/ None, responsesapi_client_metadata.clone(), ) @@ -246,9 +249,20 @@ pub(super) async fn user_input_or_turn_inner( ) .await; let accepted_items = items.clone(); + let additional_context_input = { + let mut state = sess.state.lock().await; + state.additional_context.merge(additional_context) + }; + let mut task_input = additional_context_input + .into_iter() + .map(TurnInput::ResponseInputItem) + .collect::>(); + if !items.is_empty() { + task_input.push(TurnInput::UserInput(items)); + } sess.spawn_task( Arc::clone(¤t_context), - items, + task_input, crate::tasks::RegularTask::new(), ) .await; diff --git a/codex-rs/core/src/session/input_queue.rs b/codex-rs/core/src/session/input_queue.rs index 98e912c7e5..620c410eb8 100644 --- a/codex-rs/core/src/session/input_queue.rs +++ b/codex-rs/core/src/session/input_queue.rs @@ -156,13 +156,13 @@ impl InputQueue { .accept_mailbox_delivery_for_current_turn(); } - pub(super) async fn push_pending_input_and_accept_mailbox_delivery_for_turn_state( + pub(super) async fn extend_pending_input_and_accept_mailbox_delivery_for_turn_state( &self, turn_state: &Mutex, - input: TurnInput, + input: Vec, ) { let mut turn_state = turn_state.lock().await; - turn_state.pending_input.items.push(input); + turn_state.pending_input.items.extend(input); turn_state.accept_mailbox_delivery_for_current_turn(); } diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs index 67cc5484fc..1876cbf980 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -1,3 +1,4 @@ +use std::collections::BTreeMap; use std::collections::HashMap; use std::collections::HashSet; use std::fmt::Debug; @@ -97,6 +98,7 @@ use codex_protocol::openai_models::ModelInfo; use codex_protocol::openai_models::ModelPreset; use codex_protocol::permissions::FileSystemSandboxPolicy; use codex_protocol::permissions::NetworkSandboxPolicy; +use codex_protocol::protocol::AdditionalContextEntry; use codex_protocol::protocol::FileChange; use codex_protocol::protocol::HasLegacyEvent; use codex_protocol::protocol::InterAgentCommunication; @@ -742,11 +744,17 @@ impl Codex { pub async fn steer_input( &self, input: Vec, + additional_context: BTreeMap, expected_turn_id: Option<&str>, responsesapi_client_metadata: Option>, ) -> Result { self.session - .steer_input(input, expected_turn_id, responsesapi_client_metadata) + .steer_input( + input, + additional_context, + expected_turn_id, + responsesapi_client_metadata, + ) .await } @@ -1086,6 +1094,7 @@ impl Session { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }, /*mirror_user_text_to_realtime*/ None, @@ -3152,6 +3161,7 @@ impl Session { pub async fn steer_input( &self, input: Vec, + additional_context: BTreeMap, expected_turn_id: Option<&str>, responsesapi_client_metadata: Option>, ) -> Result { @@ -3192,6 +3202,11 @@ impl Session { return Err(SteerInputError::EmptyInput); } + let additional_context_input = { + let mut state = self.state.lock().await; + state.additional_context.merge(additional_context) + }; + if let Some(responsesapi_client_metadata) = responsesapi_client_metadata { active_task .turn_context @@ -3199,10 +3214,15 @@ impl Session { .set_responsesapi_client_metadata(responsesapi_client_metadata); } + let mut pending_input = additional_context_input + .into_iter() + .map(TurnInput::ResponseInputItem) + .collect::>(); + pending_input.push(TurnInput::UserInput(input)); self.input_queue - .push_pending_input_and_accept_mailbox_delivery_for_turn_state( + .extend_pending_input_and_accept_mailbox_delivery_for_turn_state( active_turn.turn_state.as_ref(), - TurnInput::UserInput(input), + pending_input, ) .await; Ok(active_turn_id.clone()) diff --git a/codex-rs/core/src/session/review.rs b/codex-rs/core/src/session/review.rs index 059dc8abbd..32122873cb 100644 --- a/codex-rs/core/src/session/review.rs +++ b/codex-rs/core/src/session/review.rs @@ -132,11 +132,11 @@ pub(super) async fn spawn_review_thread( }; // Seed the child task with the review prompt as the initial user message. - let input: Vec = vec![UserInput::Text { + let input = vec![TurnInput::UserInput(vec![UserInput::Text { text: review_prompt, // Review prompt is synthesized; no UI element ranges to preserve. text_elements: Vec::new(), - }]; + }])]; let tc = Arc::new(review_turn_context); tc.turn_metadata_state.spawn_git_enrichment_task(); // TODO(ccunningham): Review turns currently rely on `spawn_task` for TurnComplete but do not diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs index 428e762244..0638168422 100644 --- a/codex-rs/core/src/session/tests.rs +++ b/codex-rs/core/src/session/tests.rs @@ -2287,6 +2287,7 @@ async fn fork_startup_context_then_first_turn_diff_snapshot() -> anyhow::Result< }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -2333,6 +2334,7 @@ async fn fork_startup_context_then_first_turn_diff_snapshot() -> anyhow::Result< }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: ThreadSettingsOverrides { approval_policy: Some(AskForApproval::Never), collaboration_mode: Some(collaboration_mode), @@ -5430,6 +5432,7 @@ fn op_kind_for_input_and_context_ops() { items: vec![], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), } .kind(), @@ -5460,6 +5463,7 @@ async fn user_turn_updates_approvals_reviewer() { environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(config.cwd.to_path_buf()), approval_policy: Some(config.permissions.approval_policy.value()), @@ -5781,10 +5785,10 @@ async fn spawn_task_turn_span_inherits_dispatch_trace_context() { async { sess.spawn_task( Arc::clone(&tc), - vec![UserInput::Text { + vec![TurnInput::UserInput(vec![UserInput::Text { text: "hello".to_string(), text_elements: Vec::new(), - }], + }])], TraceCaptureTask { captured_trace: Arc::clone(&captured_trace), }, @@ -6592,10 +6596,10 @@ async fn spawn_task_does_not_update_previous_turn_settings_for_non_run_turn_task let (sess, tc, _rx) = make_session_and_context_with_rx().await; sess.set_previous_turn_settings(/*previous_turn_settings*/ None) .await; - let input = vec![UserInput::Text { + let input = vec![TurnInput::UserInput(vec![UserInput::Text { text: "hello".to_string(), text_elements: Vec::new(), - }]; + }])]; sess.spawn_task( Arc::clone(&tc), @@ -7861,10 +7865,10 @@ impl SessionTask for GuardianDeniedApprovalTask { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn guardian_auto_review_interrupts_after_three_consecutive_denials() { let (sess, tc, rx) = make_session_and_context_with_rx().await; - let input = vec![UserInput::Text { + let input = vec![TurnInput::UserInput(vec![UserInput::Text { text: "trigger guardian denials".to_string(), text_elements: Vec::new(), - }]; + }])]; sess.spawn_task(Arc::clone(&tc), input, GuardianDeniedApprovalTask) .await; @@ -7892,10 +7896,10 @@ async fn guardian_auto_review_interrupts_after_three_consecutive_denials() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn guardian_helper_review_interrupts_after_three_consecutive_denials() { let (sess, tc, rx) = make_session_and_context_with_rx().await; - let input = vec![UserInput::Text { + let input = vec![TurnInput::UserInput(vec![UserInput::Text { text: "keep turn active for helper reviews".to_string(), text_elements: Vec::new(), - }]; + }])]; sess.spawn_task( Arc::clone(&tc), input, @@ -7952,10 +7956,10 @@ async fn guardian_helper_review_interrupts_after_three_consecutive_denials() { #[test_log::test] async fn abort_regular_task_emits_turn_aborted_only() { let (sess, tc, rx) = make_session_and_context_with_rx().await; - let input = vec![UserInput::Text { + let input = vec![TurnInput::UserInput(vec![UserInput::Text { text: "hello".to_string(), text_elements: Vec::new(), - }]; + }])]; sess.spawn_task( Arc::clone(&tc), input, @@ -7985,10 +7989,10 @@ async fn abort_regular_task_emits_turn_aborted_only() { #[tokio::test] async fn abort_gracefully_emits_turn_aborted_only() { let (sess, tc, rx) = make_session_and_context_with_rx().await; - let input = vec![UserInput::Text { + let input = vec![TurnInput::UserInput(vec![UserInput::Text { text: "hello".to_string(), text_elements: Vec::new(), - }]; + }])]; sess.spawn_task( Arc::clone(&tc), input, @@ -8018,10 +8022,10 @@ async fn abort_gracefully_emits_turn_aborted_only() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn task_finish_emits_turn_item_lifecycle_for_leftover_pending_user_input() { let (sess, tc, rx) = make_session_and_context_with_rx().await; - let input = vec![UserInput::Text { + let input = vec![TurnInput::UserInput(vec![UserInput::Text { text: "hello".to_string(), text_elements: Vec::new(), - }]; + }])]; sess.spawn_task( Arc::clone(&tc), input, @@ -8044,6 +8048,7 @@ async fn task_finish_emits_turn_item_lifecycle_for_leftover_pending_user_input() }]; sess.steer_input( pending_user_input.clone(), + /*additional_context*/ Default::default(), Some(&tc.sub_id), /*responsesapi_client_metadata*/ None, ) @@ -8140,7 +8145,10 @@ async fn steer_input_requires_active_turn() { let err = sess .steer_input( - input, /*expected_turn_id*/ None, /*responsesapi_client_metadata*/ None, + input, + /*additional_context*/ Default::default(), + /*expected_turn_id*/ None, + /*responsesapi_client_metadata*/ None, ) .await .expect_err("steering without active turn should fail"); @@ -8151,10 +8159,10 @@ async fn steer_input_requires_active_turn() { #[tokio::test] async fn steer_input_enforces_expected_turn_id() { let (sess, tc, _rx) = make_session_and_context_with_rx().await; - let input = vec![UserInput::Text { + let input = vec![TurnInput::UserInput(vec![UserInput::Text { text: "hello".to_string(), text_elements: Vec::new(), - }]; + }])]; sess.spawn_task( Arc::clone(&tc), input, @@ -8172,6 +8180,7 @@ async fn steer_input_enforces_expected_turn_id() { let err = sess .steer_input( steer_input, + /*additional_context*/ Default::default(), Some("different-turn-id"), /*responsesapi_client_metadata*/ None, ) @@ -8196,10 +8205,10 @@ async fn steer_input_rejects_non_regular_turns() { (TaskKind::Compact, NonSteerableTurnKind::Compact), ] { let (sess, _tc, _rx) = make_session_and_context_with_rx().await; - let input = vec![UserInput::Text { + let input = vec![TurnInput::UserInput(vec![UserInput::Text { text: "hello".to_string(), text_elements: Vec::new(), - }]; + }])]; let turn_context = sess.new_default_turn_with_sub_id("turn".to_string()).await; sess.spawn_task( turn_context, @@ -8218,6 +8227,7 @@ async fn steer_input_rejects_non_regular_turns() { let err = sess .steer_input( steer_input, + /*additional_context*/ Default::default(), /*expected_turn_id*/ None, /*responsesapi_client_metadata*/ None, ) @@ -8233,10 +8243,10 @@ async fn steer_input_rejects_non_regular_turns() { #[tokio::test] async fn steer_input_returns_active_turn_id() { let (sess, tc, _rx) = make_session_and_context_with_rx().await; - let input = vec![UserInput::Text { + let input = vec![TurnInput::UserInput(vec![UserInput::Text { text: "hello".to_string(), text_elements: Vec::new(), - }]; + }])]; sess.spawn_task( Arc::clone(&tc), input, @@ -8254,6 +8264,7 @@ async fn steer_input_returns_active_turn_id() { let turn_id = sess .steer_input( steer_input, + /*additional_context*/ Default::default(), Some(&tc.sub_id), /*responsesapi_client_metadata*/ None, ) @@ -8478,6 +8489,7 @@ async fn active_goal_continuation_runs_again_after_no_tool_turn() -> anyhow::Res }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -8583,6 +8595,7 @@ async fn pending_request_user_input_does_not_spawn_extra_goal_continuation() -> }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -9130,6 +9143,7 @@ async fn completed_goal_accounts_current_turn_tokens_before_tool_response() -> a }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -9296,6 +9310,7 @@ async fn steered_input_reopens_mailbox_delivery_for_current_turn() { text: "follow up".to_string(), text_elements: Vec::new(), }], + /*additional_context*/ Default::default(), Some(&tc.sub_id), /*responsesapi_client_metadata*/ None, ) @@ -9345,6 +9360,7 @@ async fn stale_defer_mailbox_delivery_does_not_override_steered_input() { text: "follow up".to_string(), text_elements: Vec::new(), }], + /*additional_context*/ Default::default(), Some(&tc.sub_id), /*responsesapi_client_metadata*/ None, ) @@ -9426,10 +9442,10 @@ async fn tool_calls_reopen_mailbox_delivery_for_current_turn() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn abort_review_task_emits_exited_then_aborted_and_records_history() { let (sess, tc, rx) = make_session_and_context_with_rx().await; - let input = vec![UserInput::Text { + let input = vec![TurnInput::UserInput(vec![UserInput::Text { text: "start review".to_string(), text_elements: Vec::new(), - }]; + }])]; sess.spawn_task(Arc::clone(&tc), input, ReviewTask::new()) .await; diff --git a/codex-rs/core/src/session/turn.rs b/codex-rs/core/src/session/turn.rs index a9e4c8463f..82fa35f108 100644 --- a/codex-rs/core/src/session/turn.rs +++ b/codex-rs/core/src/session/turn.rs @@ -407,14 +407,16 @@ async fn run_hooks_and_record_inputs( input: &[TurnInput], ) -> bool { let mut blocked_input = false; - let mut accepted_input = false; + let mut accepted_user_input = false; for input_item in input { let hook_outcome = inspect_pending_input(sess, turn_context, input_item).await; if hook_outcome.should_stop { blocked_input = true; record_additional_contexts(sess, turn_context, hook_outcome.additional_contexts).await; } else { - accepted_input = true; + if matches!(input_item, TurnInput::UserInput(items) if !items.is_empty()) { + accepted_user_input = true; + } record_pending_input( sess, turn_context, @@ -424,7 +426,7 @@ async fn run_hooks_and_record_inputs( .await; } } - blocked_input && !accepted_input + blocked_input && !accepted_user_input } #[expect( diff --git a/codex-rs/core/src/state/additional_context.rs b/codex-rs/core/src/state/additional_context.rs new file mode 100644 index 0000000000..eb5727ac23 --- /dev/null +++ b/codex-rs/core/src/state/additional_context.rs @@ -0,0 +1,37 @@ +use std::collections::BTreeMap; + +use crate::context::AdditionalContextDeveloperFragment; +use crate::context::AdditionalContextUserFragment; +use crate::context::ContextualUserFragment; +use codex_protocol::models::ResponseInputItem; +use codex_protocol::protocol::AdditionalContextEntry; +use codex_protocol::protocol::AdditionalContextKind; + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub(crate) struct AdditionalContextStore { + values: BTreeMap, +} + +impl AdditionalContextStore { + pub(crate) fn merge( + &mut self, + values: BTreeMap, + ) -> Vec { + let fragments = values + .iter() + .filter(|(key, value)| self.values.get(*key) != Some(*value)) + .map(|(key, entry)| match entry.kind { + AdditionalContextKind::Untrusted => { + AdditionalContextUserFragment::new(key.clone(), entry.value.clone()) + .into_response_input_item() + } + AdditionalContextKind::Application => { + AdditionalContextDeveloperFragment::new(key.clone(), entry.value.clone()) + .into_response_input_item() + } + }) + .collect(); + self.values = values; + fragments + } +} diff --git a/codex-rs/core/src/state/mod.rs b/codex-rs/core/src/state/mod.rs index 3122ec5f25..c693221bed 100644 --- a/codex-rs/core/src/state/mod.rs +++ b/codex-rs/core/src/state/mod.rs @@ -1,8 +1,10 @@ +mod additional_context; mod auto_compact_window; mod service; mod session; mod turn; +pub(crate) use additional_context::AdditionalContextStore; pub(crate) use auto_compact_window::AutoCompactWindowSnapshot; pub(crate) use service::SessionServices; pub(crate) use session::SessionState; diff --git a/codex-rs/core/src/state/session.rs b/codex-rs/core/src/state/session.rs index baeaddbbf9..ac59cb32d5 100644 --- a/codex-rs/core/src/state/session.rs +++ b/codex-rs/core/src/state/session.rs @@ -6,6 +6,7 @@ use codex_sandboxing::policy_transforms::merge_permission_profiles; use std::collections::HashSet; use std::collections::VecDeque; +use super::AdditionalContextStore; use super::auto_compact_window::AutoCompactWindow; use super::auto_compact_window::AutoCompactWindowSnapshot; use crate::context_manager::ContextManager; @@ -25,6 +26,7 @@ pub(crate) struct SessionState { pub(crate) latest_rate_limits: Option, pub(crate) server_reasoning_included: bool, pub(crate) mcp_dependency_prompted: HashSet, + pub(crate) additional_context: AdditionalContextStore, /// Settings used by the latest regular user turn, used for turn-to-turn /// model/realtime handling on subsequent regular turns (including full-context /// reinjection after resume or `/compact`). @@ -49,6 +51,7 @@ impl SessionState { latest_rate_limits: None, server_reasoning_included: false, mcp_dependency_prompted: HashSet::new(), + additional_context: AdditionalContextStore::default(), previous_turn_settings: None, auto_compact_window: AutoCompactWindow::new(), startup_prewarm: None, diff --git a/codex-rs/core/src/tasks/mod.rs b/codex-rs/core/src/tasks/mod.rs index e6e7663e50..aa161c1274 100644 --- a/codex-rs/core/src/tasks/mod.rs +++ b/codex-rs/core/src/tasks/mod.rs @@ -50,7 +50,6 @@ use codex_protocol::protocol::TurnAbortReason; use codex_protocol::protocol::TurnAbortedEvent; use codex_protocol::protocol::TurnCompleteEvent; use codex_protocol::protocol::WarningEvent; -use codex_protocol::user_input::UserInput; use codex_features::Feature; use codex_protocol::models::ContentItem; @@ -303,7 +302,7 @@ impl Session { pub async fn spawn_task( self: &Arc, turn_context: Arc, - input: Vec, + input: Vec, task: T, ) { self.abort_all_tasks(TurnAbortReason::Replaced).await; @@ -314,7 +313,7 @@ impl Session { pub(crate) async fn start_task( self: &Arc, turn_context: Arc, - input: Vec, + input: Vec, task: T, ) { let task: Arc = Arc::new(task); @@ -382,11 +381,7 @@ impl Session { )); let ctx = Arc::clone(&turn_context); let task_for_run = Arc::clone(&task); - let task_input = if input.is_empty() { - Vec::new() - } else { - vec![TurnInput::UserInput(input)] - }; + let task_input = input; let task_cancellation_token = cancellation_token.child_token(); // Task-owned turn spans keep a core-owned span open for the // full task lifecycle after the submission dispatch span ends. diff --git a/codex-rs/core/src/tools/handlers/multi_agents_tests.rs b/codex-rs/core/src/tools/handlers/multi_agents_tests.rs index a582805ed2..68b19adea9 100644 --- a/codex-rs/core/src/tools/handlers/multi_agents_tests.rs +++ b/codex-rs/core/src/tools/handlers/multi_agents_tests.rs @@ -2592,6 +2592,7 @@ async fn send_input_accepts_structured_items() { ], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }; let captured = manager diff --git a/codex-rs/core/tests/common/test_codex.rs b/codex-rs/core/tests/common/test_codex.rs index a398758291..73e9e0ece8 100644 --- a/codex-rs/core/tests/common/test_codex.rs +++ b/codex-rs/core/tests/common/test_codex.rs @@ -767,6 +767,7 @@ impl TestCodex { environments, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(self.config.cwd.to_path_buf()), approval_policy: Some(approval_policy), diff --git a/codex-rs/core/tests/suite/abort_tasks.rs b/codex-rs/core/tests/suite/abort_tasks.rs index c41502a9dc..82bb8c8798 100644 --- a/codex-rs/core/tests/suite/abort_tasks.rs +++ b/codex-rs/core/tests/suite/abort_tasks.rs @@ -53,6 +53,7 @@ async fn interrupt_long_running_tool_emits_turn_aborted() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -110,6 +111,7 @@ async fn interrupt_tool_records_history_entries() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -131,6 +133,7 @@ async fn interrupt_tool_records_history_entries() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -214,6 +217,7 @@ async fn interrupt_persists_turn_aborted_marker_in_next_request() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -235,6 +239,7 @@ async fn interrupt_persists_turn_aborted_marker_in_next_request() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await diff --git a/codex-rs/core/tests/suite/additional_context.rs b/codex-rs/core/tests/suite/additional_context.rs new file mode 100644 index 0000000000..f9bfdb966b --- /dev/null +++ b/codex-rs/core/tests/suite/additional_context.rs @@ -0,0 +1,581 @@ +use anyhow::Result; +use codex_protocol::items::TurnItem; +use codex_protocol::protocol::AdditionalContextEntry; +use codex_protocol::protocol::AdditionalContextKind; +use codex_protocol::protocol::EventMsg; +use codex_protocol::protocol::ItemCompletedEvent; +use codex_protocol::protocol::Op; +use codex_protocol::user_input::UserInput; +use core_test_support::context_snapshot; +use core_test_support::context_snapshot::ContextSnapshotOptions; +use core_test_support::context_snapshot::ContextSnapshotRenderMode; +use core_test_support::responses::ev_completed; +use core_test_support::responses::ev_response_created; +use core_test_support::responses::mount_sse_once; +use core_test_support::responses::sse; +use core_test_support::responses::start_mock_server; +use core_test_support::skip_if_no_network; +use core_test_support::test_codex::test_codex; +use core_test_support::wait_for_event_match; +use pretty_assertions::assert_eq; +use std::collections::BTreeMap; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn additional_context_is_model_visible_but_not_a_user_message_item() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let request = mount_sse_once( + &server, + sse(vec![ev_response_created("resp-1"), ev_completed("resp-1")]), + ) + .await; + let test = test_codex() + .with_config(|config| config.include_environment_context = false) + .build(&server) + .await?; + + test.codex + .submit(Op::UserInput { + environments: None, + items: vec![UserInput::Text { + text: "inspect the active tab".to_string(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + responsesapi_client_metadata: None, + additional_context: BTreeMap::from([ + ( + "browser_info".to_string(), + AdditionalContextEntry { + value: "tab one".to_string(), + kind: AdditionalContextKind::Untrusted, + }, + ), + ( + "automation_info".to_string(), + AdditionalContextEntry { + value: "run one".to_string(), + kind: AdditionalContextKind::Application, + }, + ), + ]), + thread_settings: Default::default(), + }) + .await?; + + let user_item = wait_for_event_match(&test.codex, |event| match event { + EventMsg::ItemCompleted(ItemCompletedEvent { + item: TurnItem::UserMessage(item), + .. + }) => Some(item.clone()), + _ => None, + }) + .await; + assert_eq!( + user_item.content, + vec![UserInput::Text { + text: "inspect the active tab".to_string(), + text_elements: Vec::new(), + }] + ); + wait_for_event_match(&test.codex, |event| { + matches!(event, EventMsg::TurnComplete(_)).then_some(()) + }) + .await; + + let request = request.single_request(); + insta::assert_snapshot!( + "additional_context_simple_input", + context_snapshot::format_labeled_requests_snapshot( + "additional context is inserted before the user turn input.", + &[("Request", &request)], + &ContextSnapshotOptions::default() + .strip_capability_instructions() + .render_mode(ContextSnapshotRenderMode::KindWithTextPrefix { max_chars: 160 }), + ) + ); + let developer_context_texts = request + .message_input_texts("developer") + .into_iter() + .filter(|text| text.starts_with("")) + .collect::>(); + assert_eq!( + developer_context_texts, + vec!["run one"] + ); + assert_eq!( + request.message_input_texts("user"), + vec![ + "tab one", + "inspect the active tab", + ] + ); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn external_context_like_user_text_remains_a_user_message_item() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let request = mount_sse_once( + &server, + sse(vec![ev_response_created("resp-1"), ev_completed("resp-1")]), + ) + .await; + let test = test_codex() + .with_config(|config| config.include_environment_context = false) + .build(&server) + .await?; + let user_input = UserInput::Text { + text: "".to_string(), + text_elements: Vec::new(), + }; + + test.codex + .submit(Op::UserInput { + environments: None, + items: vec![user_input.clone()], + final_output_json_schema: None, + responsesapi_client_metadata: None, + additional_context: BTreeMap::new(), + thread_settings: Default::default(), + }) + .await?; + + let user_item = wait_for_event_match(&test.codex, |event| match event { + EventMsg::ItemCompleted(ItemCompletedEvent { + item: TurnItem::UserMessage(item), + .. + }) => Some(item.clone()), + _ => None, + }) + .await; + assert_eq!(user_item.content, vec![user_input]); + wait_for_event_match(&test.codex, |event| { + matches!(event, EventMsg::TurnComplete(_)).then_some(()) + }) + .await; + + let request = request.single_request(); + assert_eq!(request.message_input_texts("user"), vec![""]); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn additional_context_trust_controls_message_role() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let request = mount_sse_once( + &server, + sse(vec![ev_response_created("resp-1"), ev_completed("resp-1")]), + ) + .await; + let test = test_codex() + .with_config(|config| config.include_environment_context = false) + .build(&server) + .await?; + + test.codex + .submit(Op::UserInput { + environments: None, + items: vec![UserInput::Text { + text: "inspect context".to_string(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + responsesapi_client_metadata: None, + additional_context: BTreeMap::from([ + ( + "browser_info".to_string(), + AdditionalContextEntry { + value: "tab one".to_string(), + kind: AdditionalContextKind::Untrusted, + }, + ), + ( + "automation_info".to_string(), + AdditionalContextEntry { + value: "run one".to_string(), + kind: AdditionalContextKind::Application, + }, + ), + ]), + thread_settings: Default::default(), + }) + .await?; + wait_for_event_match(&test.codex, |event| { + matches!(event, EventMsg::TurnComplete(_)).then_some(()) + }) + .await; + + let request = request.single_request(); + let developer_context_texts = request + .message_input_texts("developer") + .into_iter() + .filter(|text| text.starts_with("")) + .collect::>(); + assert_eq!( + developer_context_texts, + vec!["run one"] + ); + assert_eq!( + request.message_input_texts("user"), + vec![ + "tab one", + "inspect context", + ] + ); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn additional_context_is_deduplicated_between_turns_while_retained() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let first_request = mount_sse_once( + &server, + sse(vec![ev_response_created("resp-1"), ev_completed("resp-1")]), + ) + .await; + let second_request = mount_sse_once( + &server, + sse(vec![ev_response_created("resp-2"), ev_completed("resp-2")]), + ) + .await; + let test = test_codex() + .with_config(|config| config.include_environment_context = false) + .build(&server) + .await?; + let additional_context = BTreeMap::from([( + "browser_info".to_string(), + AdditionalContextEntry { + value: "same tab".to_string(), + kind: AdditionalContextKind::Untrusted, + }, + )]); + + test.codex + .submit(Op::UserInput { + environments: None, + items: vec![UserInput::Text { + text: "first turn".to_string(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + responsesapi_client_metadata: None, + additional_context: additional_context.clone(), + thread_settings: Default::default(), + }) + .await?; + wait_for_event_match(&test.codex, |event| { + matches!(event, EventMsg::TurnComplete(_)).then_some(()) + }) + .await; + + test.codex + .submit(Op::UserInput { + environments: None, + items: vec![UserInput::Text { + text: "second turn".to_string(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + responsesapi_client_metadata: None, + additional_context, + thread_settings: Default::default(), + }) + .await?; + wait_for_event_match(&test.codex, |event| { + matches!(event, EventMsg::TurnComplete(_)).then_some(()) + }) + .await; + + assert_eq!( + first_request.single_request().message_input_texts("user"), + vec![ + "same tab", + "first turn", + ] + ); + assert_eq!( + second_request.single_request().message_input_texts("user"), + vec![ + "same tab", + "first turn", + "second turn", + ] + ); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn additional_context_removes_one_value_while_adding_another() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + let first_request = mount_sse_once( + &server, + sse(vec![ev_response_created("resp-1"), ev_completed("resp-1")]), + ) + .await; + let second_request = mount_sse_once( + &server, + sse(vec![ev_response_created("resp-2"), ev_completed("resp-2")]), + ) + .await; + let third_request = mount_sse_once( + &server, + sse(vec![ev_response_created("resp-3"), ev_completed("resp-3")]), + ) + .await; + let test = test_codex() + .with_config(|config| config.include_environment_context = false) + .build(&server) + .await?; + + test.codex + .submit(Op::UserInput { + environments: None, + items: vec![UserInput::Text { + text: "first turn".to_string(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + responsesapi_client_metadata: None, + additional_context: BTreeMap::from([ + ( + "automation_info".to_string(), + AdditionalContextEntry { + value: "run one".to_string(), + kind: AdditionalContextKind::Untrusted, + }, + ), + ( + "browser_info".to_string(), + AdditionalContextEntry { + value: "tab one".to_string(), + kind: AdditionalContextKind::Untrusted, + }, + ), + ]), + thread_settings: Default::default(), + }) + .await?; + wait_for_event_match(&test.codex, |event| { + matches!(event, EventMsg::TurnComplete(_)).then_some(()) + }) + .await; + + test.codex + .submit(Op::UserInput { + environments: None, + items: vec![UserInput::Text { + text: "second turn".to_string(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + responsesapi_client_metadata: None, + additional_context: BTreeMap::from([ + ( + "automation_info".to_string(), + AdditionalContextEntry { + value: "run one".to_string(), + kind: AdditionalContextKind::Untrusted, + }, + ), + ( + "terminal_info".to_string(), + AdditionalContextEntry { + value: "pty one".to_string(), + kind: AdditionalContextKind::Untrusted, + }, + ), + ]), + thread_settings: Default::default(), + }) + .await?; + wait_for_event_match(&test.codex, |event| { + matches!(event, EventMsg::TurnComplete(_)).then_some(()) + }) + .await; + + test.codex + .submit(Op::UserInput { + environments: None, + items: vec![UserInput::Text { + text: "third turn".to_string(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + responsesapi_client_metadata: None, + additional_context: BTreeMap::from([ + ( + "automation_info".to_string(), + AdditionalContextEntry { + value: "run one".to_string(), + kind: AdditionalContextKind::Untrusted, + }, + ), + ( + "browser_info".to_string(), + AdditionalContextEntry { + value: "tab one".to_string(), + kind: AdditionalContextKind::Untrusted, + }, + ), + ( + "terminal_info".to_string(), + AdditionalContextEntry { + value: "pty one".to_string(), + kind: AdditionalContextKind::Untrusted, + }, + ), + ]), + thread_settings: Default::default(), + }) + .await?; + wait_for_event_match(&test.codex, |event| { + matches!(event, EventMsg::TurnComplete(_)).then_some(()) + }) + .await; + + assert_eq!( + first_request.single_request().message_input_texts("user"), + vec![ + "run one", + "tab one", + "first turn", + ] + ); + assert_eq!( + second_request.single_request().message_input_texts("user"), + vec![ + "run one", + "tab one", + "first turn", + "pty one", + "second turn", + ] + ); + assert_eq!( + third_request.single_request().message_input_texts("user"), + vec![ + "run one", + "tab one", + "first turn", + "pty one", + "second turn", + "tab one", + "third turn", + ] + ); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn additional_context_values_are_truncated_before_model_input() -> Result<()> { + skip_if_no_network!(Ok(())); + + const MAX_EXPECTED_EXTERNAL_CONTEXT_TEXT_BYTES: usize = 5 * 1024; + + let server = start_mock_server().await; + let request = mount_sse_once( + &server, + sse(vec![ev_response_created("resp-1"), ev_completed("resp-1")]), + ) + .await; + let test = test_codex() + .with_config(|config| config.include_environment_context = false) + .build(&server) + .await?; + let long_browser_value = format!("browser-head-{}browser-tail", "b".repeat(40_000)); + let long_automation_value = format!("automation-head-{}automation-tail", "a".repeat(40_000)); + let untruncated_browser_fragment = + format!("{long_browser_value}"); + let untruncated_automation_fragment = + format!("{long_automation_value}"); + + test.codex + .submit(Op::UserInput { + environments: None, + items: vec![UserInput::Text { + text: "summarize context".to_string(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + responsesapi_client_metadata: None, + additional_context: BTreeMap::from([ + ( + "automation_info".to_string(), + AdditionalContextEntry { + value: long_automation_value.clone(), + kind: AdditionalContextKind::Application, + }, + ), + ( + "browser_info".to_string(), + AdditionalContextEntry { + value: long_browser_value.clone(), + kind: AdditionalContextKind::Untrusted, + }, + ), + ]), + thread_settings: Default::default(), + }) + .await?; + wait_for_event_match(&test.codex, |event| { + matches!(event, EventMsg::TurnComplete(_)).then_some(()) + }) + .await; + + let request = request.single_request(); + let developer_texts = request + .message_input_texts("developer") + .into_iter() + .filter(|text| text.starts_with("")) + .collect::>(); + let [automation_text] = developer_texts.as_slice() else { + panic!("expected application additional context, got {developer_texts:?}"); + }; + assert!(automation_text.starts_with(&format!( + "automation-head-{}", + "a".repeat(1024) + ))); + assert!(automation_text.contains("tokens truncated")); + assert!(automation_text.ends_with("automation-tail")); + assert!(automation_text.len() < untruncated_automation_fragment.len()); + assert!( + automation_text.len() <= MAX_EXPECTED_EXTERNAL_CONTEXT_TEXT_BYTES, + "application additional context was not capped before model input: {} bytes", + automation_text.len() + ); + + let user_texts = request.message_input_texts("user"); + let [external_text, user_text] = user_texts.as_slice() else { + panic!("expected external context plus user input, got {user_texts:?}"); + }; + assert_eq!(user_text, "summarize context"); + assert!(external_text.starts_with(&format!( + "browser-head-{}", + "b".repeat(1024) + ))); + assert!(external_text.contains("tokens truncated")); + assert!(external_text.ends_with("browser-tail")); + assert!(external_text.len() < untruncated_browser_fragment.len()); + assert!( + external_text.len() <= MAX_EXPECTED_EXTERNAL_CONTEXT_TEXT_BYTES, + "untrusted additional context was not capped before model input: {} bytes", + external_text.len() + ); + + Ok(()) +} diff --git a/codex-rs/core/tests/suite/apply_patch_cli.rs b/codex-rs/core/tests/suite/apply_patch_cli.rs index f5a35003f8..da138851f5 100644 --- a/codex-rs/core/tests/suite/apply_patch_cli.rs +++ b/codex-rs/core/tests/suite/apply_patch_cli.rs @@ -91,6 +91,7 @@ async fn submit_without_wait_with_turn_permissions( environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(harness.cwd().to_path_buf()), approval_policy: Some(AskForApproval::Never), diff --git a/codex-rs/core/tests/suite/approvals.rs b/codex-rs/core/tests/suite/approvals.rs index fe3fd4ac7d..e16a4a14a2 100644 --- a/codex-rs/core/tests/suite/approvals.rs +++ b/codex-rs/core/tests/suite/approvals.rs @@ -653,6 +653,7 @@ async fn submit_turn( environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(test.cwd.path().to_path_buf()), approval_policy: Some(approval_policy), @@ -2598,6 +2599,7 @@ async fn matched_prefix_rule_runs_unsandboxed_under_zsh_fork() -> Result<()> { environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(test.cwd.path().to_path_buf()), approval_policy: Some(approval_policy), diff --git a/codex-rs/core/tests/suite/client.rs b/codex-rs/core/tests/suite/client.rs index f6731a0ea4..7234c59c0f 100644 --- a/codex-rs/core/tests/suite/client.rs +++ b/codex-rs/core/tests/suite/client.rs @@ -392,6 +392,7 @@ async fn resume_includes_initial_messages_and_sends_prior_items() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -758,6 +759,7 @@ async fn includes_session_id_thread_id_and_model_headers_in_request() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -969,6 +971,7 @@ async fn includes_base_instructions_override_in_request() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -1026,6 +1029,7 @@ async fn chatgpt_auth_sends_correct_request() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -1150,6 +1154,7 @@ async fn prefers_apikey_when_config_prefers_apikey_even_with_chatgpt_tokens() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -1189,6 +1194,7 @@ async fn includes_user_instructions_message_in_request() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -1277,6 +1283,7 @@ async fn includes_apps_guidance_as_developer_message_for_chatgpt_auth() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -1340,6 +1347,7 @@ async fn omits_apps_guidance_for_api_key_auth_even_when_feature_enabled() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -1399,6 +1407,7 @@ async fn omits_apps_guidance_when_configured_off() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -1441,6 +1450,7 @@ async fn omits_environment_context_when_configured_off() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -1498,6 +1508,7 @@ async fn skills_append_to_developer_message() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -1581,6 +1592,7 @@ async fn skills_use_aliases_in_developer_message_under_budget_pressure() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -1642,6 +1654,7 @@ async fn includes_configured_effort_in_request() -> anyhow::Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -1684,6 +1697,7 @@ async fn includes_no_effort_in_request() -> anyhow::Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -1727,6 +1741,7 @@ async fn includes_default_reasoning_effort_in_request_when_defined_by_model_info }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -1778,6 +1793,7 @@ async fn user_turn_collaboration_mode_overrides_model_and_effort() -> anyhow::Re environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(config.cwd.to_path_buf()), approval_policy: Some(config.permissions.approval_policy.value()), @@ -1834,6 +1850,7 @@ async fn configured_reasoning_summary_is_sent() -> anyhow::Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -1899,6 +1916,7 @@ async fn user_turn_explicit_reasoning_summary_overrides_model_catalog_default() environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(config.cwd.to_path_buf()), approval_policy: Some(config.permissions.approval_policy.value()), @@ -1959,6 +1977,7 @@ async fn reasoning_summary_is_omitted_when_disabled() -> anyhow::Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -2018,6 +2037,7 @@ async fn reasoning_summary_none_overrides_model_catalog_default() -> anyhow::Res }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -2057,6 +2077,7 @@ async fn includes_default_verbosity_in_request() -> anyhow::Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -2105,6 +2126,7 @@ async fn configured_verbosity_not_sent_for_models_without_support() -> anyhow::R }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -2152,6 +2174,7 @@ async fn configured_verbosity_is_sent() -> anyhow::Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -2204,6 +2227,7 @@ async fn includes_developer_instructions_message_in_request() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -2500,6 +2524,7 @@ async fn token_count_includes_rate_limits_snapshot() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -2639,6 +2664,7 @@ async fn usage_limit_error_emits_rate_limit_event() -> anyhow::Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -2716,6 +2742,7 @@ async fn context_window_error_sets_total_tokens_to_model_window() -> anyhow::Res }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -2731,6 +2758,7 @@ async fn context_window_error_sets_total_tokens_to_model_window() -> anyhow::Res }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -2816,6 +2844,7 @@ async fn incomplete_response_emits_content_filter_error_message() -> anyhow::Res }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -2927,6 +2956,7 @@ async fn azure_overrides_assign_properties_used_for_responses_url() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -3016,6 +3046,7 @@ async fn env_var_overrides_loaded_auth() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -3073,6 +3104,7 @@ async fn history_dedupes_streamed_and_final_messages_across_turns() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -3089,6 +3121,7 @@ async fn history_dedupes_streamed_and_final_messages_across_turns() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -3105,6 +3138,7 @@ async fn history_dedupes_streamed_and_final_messages_across_turns() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await diff --git a/codex-rs/core/tests/suite/client_websockets.rs b/codex-rs/core/tests/suite/client_websockets.rs index 74eb053325..09d0093f99 100755 --- a/codex-rs/core/tests/suite/client_websockets.rs +++ b/codex-rs/core/tests/suite/client_websockets.rs @@ -1319,6 +1319,7 @@ async fn responses_websocket_usage_limit_error_emits_rate_limit_event() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -1408,6 +1409,7 @@ async fn responses_websocket_invalid_request_error_with_status_is_forwarded() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await diff --git a/codex-rs/core/tests/suite/code_mode.rs b/codex-rs/core/tests/suite/code_mode.rs index 3cd5c67249..fb9e7fb420 100644 --- a/codex-rs/core/tests/suite/code_mode.rs +++ b/codex-rs/core/tests/suite/code_mode.rs @@ -2991,6 +2991,7 @@ text( environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(cwd), approval_policy: Some(AskForApproval::Never), diff --git a/codex-rs/core/tests/suite/collaboration_instructions.rs b/codex-rs/core/tests/suite/collaboration_instructions.rs index 0f9bee9431..bbbb0f1aa2 100644 --- a/codex-rs/core/tests/suite/collaboration_instructions.rs +++ b/codex-rs/core/tests/suite/collaboration_instructions.rs @@ -79,6 +79,7 @@ async fn no_collaboration_instructions_by_default() -> Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -133,6 +134,7 @@ async fn user_input_includes_collaboration_instructions_after_override() -> Resu }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -170,6 +172,7 @@ async fn collaboration_instructions_added_on_user_turn() -> Result<()> { environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(test.config.cwd.to_path_buf()), approval_policy: Some(test.config.permissions.approval_policy.value()), @@ -220,6 +223,7 @@ async fn collaboration_instructions_omitted_when_disabled() -> Result<()> { environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(test.config.cwd.to_path_buf()), approval_policy: Some(test.config.permissions.approval_policy.value()), @@ -279,6 +283,7 @@ async fn override_then_next_turn_uses_updated_collaboration_instructions() -> Re }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -327,6 +332,7 @@ async fn user_turn_overrides_collaboration_instructions_after_override() -> Resu environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(test.config.cwd.to_path_buf()), approval_policy: Some(test.config.permissions.approval_policy.value()), @@ -391,6 +397,7 @@ async fn collaboration_mode_update_emits_new_instruction_message() -> Result<()> }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -414,6 +421,7 @@ async fn collaboration_mode_update_emits_new_instruction_message() -> Result<()> }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -466,6 +474,7 @@ async fn collaboration_mode_update_noop_does_not_append() -> Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -489,6 +498,7 @@ async fn collaboration_mode_update_noop_does_not_append() -> Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -543,6 +553,7 @@ async fn collaboration_mode_update_emits_new_instruction_message_when_mode_chang }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -569,6 +580,7 @@ async fn collaboration_mode_update_emits_new_instruction_message_when_mode_chang }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -624,6 +636,7 @@ async fn collaboration_mode_update_noop_does_not_append_when_mode_is_unchanged() }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -650,6 +663,7 @@ async fn collaboration_mode_update_noop_does_not_append_when_mode_is_unchanged() }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -708,6 +722,7 @@ async fn resume_replays_collaboration_instructions() -> Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -724,6 +739,7 @@ async fn resume_replays_collaboration_instructions() -> Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -776,6 +792,7 @@ async fn empty_collaboration_instructions_are_ignored() -> Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; diff --git a/codex-rs/core/tests/suite/compact.rs b/codex-rs/core/tests/suite/compact.rs index 140b4ec2b1..be6d30c6fa 100644 --- a/codex-rs/core/tests/suite/compact.rs +++ b/codex-rs/core/tests/suite/compact.rs @@ -97,6 +97,7 @@ fn disabled_permission_user_turn(text: impl Into, cwd: PathBuf, model: S environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(cwd), approval_policy: Some(AskForApproval::Never), @@ -408,6 +409,7 @@ async fn summarize_context_three_requests_and_instructions() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -433,6 +435,7 @@ async fn summarize_context_three_requests_and_instructions() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -610,6 +613,7 @@ async fn manual_pre_compact_block_decision_does_not_block_compaction() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -683,6 +687,7 @@ async fn compact_hooks_respect_matchers_and_post_runs_after_compaction() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -753,6 +758,7 @@ async fn manual_compact_uses_custom_prompt() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -900,6 +906,7 @@ async fn manual_compact_emits_context_compaction_items() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -1066,6 +1073,7 @@ async fn multiple_auto_compact_per_task_runs_after_token_limit_hit() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -1538,6 +1546,7 @@ async fn auto_compact_runs_after_token_limit_hit() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -1554,6 +1563,7 @@ async fn auto_compact_runs_after_token_limit_hit() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -1570,6 +1580,7 @@ async fn auto_compact_runs_after_token_limit_hit() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -1741,6 +1752,7 @@ async fn auto_compact_emits_context_compaction_items() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -1822,6 +1834,7 @@ async fn auto_compact_starts_after_turn_started() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -1837,6 +1850,7 @@ async fn auto_compact_starts_after_turn_started() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -1852,6 +1866,7 @@ async fn auto_compact_starts_after_turn_started() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -2380,6 +2395,7 @@ async fn auto_compact_persists_rollout_entries() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -2395,6 +2411,7 @@ async fn auto_compact_persists_rollout_entries() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -2410,6 +2427,7 @@ async fn auto_compact_persists_rollout_entries() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -2499,6 +2517,7 @@ async fn manual_compact_retries_after_context_window_error() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -2603,6 +2622,7 @@ async fn manual_compact_non_context_failure_retries_then_emits_task_error() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -2698,6 +2718,7 @@ async fn manual_compact_twice_preserves_latest_user_messages() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -2716,6 +2737,7 @@ async fn manual_compact_twice_preserves_latest_user_messages() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -2734,6 +2756,7 @@ async fn manual_compact_twice_preserves_latest_user_messages() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -2898,6 +2921,7 @@ async fn auto_compact_allows_multiple_attempts_when_interleaved_with_other_turn_ }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -3003,6 +3027,7 @@ async fn snapshot_request_shape_mid_turn_continuation_compaction() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -3434,6 +3459,7 @@ async fn auto_compact_counts_encrypted_reasoning_before_last_user() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -3553,6 +3579,7 @@ async fn auto_compact_runs_when_reasoning_header_clears_between_turns() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -3615,6 +3642,7 @@ async fn snapshot_request_shape_pre_turn_compaction_including_incoming_user_mess }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -3647,6 +3675,7 @@ async fn snapshot_request_shape_pre_turn_compaction_including_incoming_user_mess ], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -3836,6 +3865,7 @@ async fn snapshot_request_shape_pre_turn_compaction_context_window_exceeded() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -3851,6 +3881,7 @@ async fn snapshot_request_shape_pre_turn_compaction_context_window_exceeded() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -3924,6 +3955,7 @@ async fn snapshot_request_shape_manual_compact_without_previous_user_messages() }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await diff --git a/codex-rs/core/tests/suite/compact_remote.rs b/codex-rs/core/tests/suite/compact_remote.rs index b614c09d3a..d8779bfb05 100644 --- a/codex-rs/core/tests/suite/compact_remote.rs +++ b/codex-rs/core/tests/suite/compact_remote.rs @@ -326,6 +326,7 @@ async fn remote_compact_replaces_history_for_followups() -> Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -343,6 +344,7 @@ async fn remote_compact_replaces_history_for_followups() -> Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -518,6 +520,7 @@ async fn assert_remote_manual_compact_request_parity( }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -538,6 +541,7 @@ async fn assert_remote_manual_compact_request_parity( ], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -552,6 +556,7 @@ async fn assert_remote_manual_compact_request_parity( }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -572,6 +577,7 @@ async fn assert_remote_manual_compact_request_parity( ], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -586,6 +592,7 @@ async fn assert_remote_manual_compact_request_parity( }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -754,6 +761,7 @@ async fn remote_compact_v2_reuses_compaction_trigger_for_followups() -> Result<( }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -771,6 +779,7 @@ async fn remote_compact_v2_reuses_compaction_trigger_for_followups() -> Result<( }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -874,6 +883,7 @@ async fn remote_compact_v2_retries_failures_with_stream_retry_budget() -> Result }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -891,6 +901,7 @@ async fn remote_compact_v2_retries_failures_with_stream_retry_budget() -> Result }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -977,6 +988,7 @@ async fn remote_compact_v2_accepts_additional_output_items_before_compaction() - }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -994,6 +1006,7 @@ async fn remote_compact_v2_accepts_additional_output_items_before_compaction() - }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -1085,6 +1098,7 @@ async fn remote_compact_filters_deferred_dynamic_tools() -> Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -1157,6 +1171,7 @@ async fn remote_compact_runs_automatically() -> Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -1238,6 +1253,7 @@ async fn remote_compact_trims_function_call_history_to_fit_context_window() -> R }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -1252,6 +1268,7 @@ async fn remote_compact_trims_function_call_history_to_fit_context_window() -> R }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -1370,6 +1387,7 @@ async fn auto_remote_compact_trims_function_call_history_to_fit_context_window() }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -1384,6 +1402,7 @@ async fn auto_remote_compact_trims_function_call_history_to_fit_context_window() }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -1404,6 +1423,7 @@ async fn auto_remote_compact_trims_function_call_history_to_fit_context_window() }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -1504,6 +1524,7 @@ async fn auto_remote_compact_failure_stops_agent_loop() -> Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -1518,6 +1539,7 @@ async fn auto_remote_compact_failure_stops_agent_loop() -> Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -1612,6 +1634,7 @@ async fn remote_compact_trim_estimate_uses_session_base_instructions() -> Result }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -1629,6 +1652,7 @@ async fn remote_compact_trim_estimate_uses_session_base_instructions() -> Result }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -1720,6 +1744,7 @@ async fn remote_compact_trim_estimate_uses_session_base_instructions() -> Result }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -1737,6 +1762,7 @@ async fn remote_compact_trim_estimate_uses_session_base_instructions() -> Result }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -1808,6 +1834,7 @@ async fn remote_manual_compact_emits_context_compaction_items() -> Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -1889,6 +1916,7 @@ async fn remote_manual_compact_failure_emits_task_error_event() -> Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -1973,6 +2001,7 @@ async fn remote_compact_persists_replacement_history_in_rollout() -> Result<()> }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -2116,6 +2145,7 @@ async fn remote_compact_and_resume_refresh_stale_developer_instructions() -> Res }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -2134,6 +2164,7 @@ async fn remote_compact_and_resume_refresh_stale_developer_instructions() -> Res }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -2159,6 +2190,7 @@ async fn remote_compact_and_resume_refresh_stale_developer_instructions() -> Res }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -2255,6 +2287,7 @@ async fn remote_compact_refreshes_stale_developer_instructions_without_resume() }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -2272,6 +2305,7 @@ async fn remote_compact_refreshes_stale_developer_instructions_without_resume() }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -2344,6 +2378,7 @@ async fn snapshot_request_shape_remote_pre_turn_compaction_restates_realtime_sta }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -2358,6 +2393,7 @@ async fn snapshot_request_shape_remote_pre_turn_compaction_restates_realtime_sta }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -2425,6 +2461,7 @@ async fn remote_request_uses_custom_experimental_realtime_start_instructions() - }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -2486,6 +2523,7 @@ async fn snapshot_request_shape_remote_pre_turn_compaction_restates_realtime_end }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -2502,6 +2540,7 @@ async fn snapshot_request_shape_remote_pre_turn_compaction_restates_realtime_end }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -2577,6 +2616,7 @@ async fn snapshot_request_shape_remote_manual_compact_restates_realtime_start() }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -2594,6 +2634,7 @@ async fn snapshot_request_shape_remote_manual_compact_restates_realtime_start() }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -2677,6 +2718,7 @@ async fn snapshot_request_shape_remote_mid_turn_compaction_does_not_restate_real }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -2693,6 +2735,7 @@ async fn snapshot_request_shape_remote_mid_turn_compaction_does_not_restate_real }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -2784,6 +2827,7 @@ async fn snapshot_request_shape_remote_compact_resume_restates_realtime_end() -> }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -2814,6 +2858,7 @@ async fn snapshot_request_shape_remote_compact_resume_restates_realtime_end() -> }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -2903,6 +2948,7 @@ async fn snapshot_request_shape_remote_pre_turn_compaction_including_incoming_us }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -2990,6 +3036,7 @@ async fn snapshot_request_shape_remote_pre_turn_compaction_strips_incoming_model }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -3012,6 +3059,7 @@ async fn snapshot_request_shape_remote_pre_turn_compaction_strips_incoming_model }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -3131,6 +3179,7 @@ async fn snapshot_request_shape_remote_pre_turn_compaction_context_window_exceed }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -3145,6 +3194,7 @@ async fn snapshot_request_shape_remote_pre_turn_compaction_context_window_exceed }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -3230,6 +3280,7 @@ async fn snapshot_request_shape_remote_mid_turn_continuation_compaction() -> Res }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -3308,6 +3359,7 @@ async fn snapshot_request_shape_remote_mid_turn_compaction_summary_only_reinject }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -3394,6 +3446,7 @@ async fn snapshot_request_shape_remote_mid_turn_compaction_multi_summary_reinjec }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -3411,6 +3464,7 @@ async fn snapshot_request_shape_remote_mid_turn_compaction_multi_summary_reinjec }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -3493,6 +3547,7 @@ async fn snapshot_request_shape_remote_manual_compact_without_previous_user_mess }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; diff --git a/codex-rs/core/tests/suite/compact_remote_parity.rs b/codex-rs/core/tests/suite/compact_remote_parity.rs index ccbd427b1f..8979b2cced 100644 --- a/codex-rs/core/tests/suite/compact_remote_parity.rs +++ b/codex-rs/core/tests/suite/compact_remote_parity.rs @@ -606,6 +606,7 @@ async fn submit_user_input(codex: &codex_core::CodexThread, items: Vec, text: &str) { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await diff --git a/codex-rs/core/tests/suite/exec_policy.rs b/codex-rs/core/tests/suite/exec_policy.rs index f1e01c22ce..9e350d2773 100644 --- a/codex-rs/core/tests/suite/exec_policy.rs +++ b/codex-rs/core/tests/suite/exec_policy.rs @@ -54,6 +54,7 @@ async fn submit_user_turn( environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(test.cwd_path().to_path_buf()), approval_policy: Some(approval_policy), @@ -141,6 +142,7 @@ async fn execpolicy_blocks_shell_invocation() -> Result<()> { environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(test.cwd_path().to_path_buf()), approval_policy: Some(AskForApproval::Never), diff --git a/codex-rs/core/tests/suite/fork_thread.rs b/codex-rs/core/tests/suite/fork_thread.rs index 68a167f790..544c530aa0 100644 --- a/codex-rs/core/tests/suite/fork_thread.rs +++ b/codex-rs/core/tests/suite/fork_thread.rs @@ -57,6 +57,7 @@ async fn fork_thread_twice_drops_to_first_message() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -181,6 +182,7 @@ async fn fork_thread_from_history_does_not_require_source_rollout_path() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await diff --git a/codex-rs/core/tests/suite/hooks.rs b/codex-rs/core/tests/suite/hooks.rs index 3791746551..a535eb4b8e 100644 --- a/codex-rs/core/tests/suite/hooks.rs +++ b/codex-rs/core/tests/suite/hooks.rs @@ -1850,6 +1850,7 @@ async fn blocked_queued_prompt_does_not_strand_earlier_accepted_prompt() -> Resu }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -1869,6 +1870,7 @@ async fn blocked_queued_prompt_does_not_strand_earlier_accepted_prompt() -> Resu }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; diff --git a/codex-rs/core/tests/suite/image_rollout.rs b/codex-rs/core/tests/suite/image_rollout.rs index 085ca96f37..67a79d603f 100644 --- a/codex-rs/core/tests/suite/image_rollout.rs +++ b/codex-rs/core/tests/suite/image_rollout.rs @@ -127,6 +127,7 @@ async fn copy_paste_local_image_persists_rollout_request_shape() -> anyhow::Resu environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(cwd.path().to_path_buf()), approval_policy: Some(AskForApproval::Never), @@ -223,6 +224,7 @@ async fn drag_drop_image_persists_rollout_request_shape() -> anyhow::Result<()> environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(cwd.path().to_path_buf()), approval_policy: Some(AskForApproval::Never), diff --git a/codex-rs/core/tests/suite/items.rs b/codex-rs/core/tests/suite/items.rs index 82e01e2451..a56b2ab95f 100644 --- a/codex-rs/core/tests/suite/items.rs +++ b/codex-rs/core/tests/suite/items.rs @@ -58,6 +58,7 @@ fn disabled_plan_turn( environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(cwd), approval_policy: Some(AskForApproval::Never), @@ -119,6 +120,7 @@ async fn user_message_item_is_emitted() -> anyhow::Result<()> { items: vec![expected_input.clone()], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -178,6 +180,7 @@ async fn assistant_message_item_is_emitted() -> anyhow::Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -239,6 +242,7 @@ async fn reasoning_item_is_emitted() -> anyhow::Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -301,6 +305,7 @@ async fn web_search_item_is_emitted() -> anyhow::Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -381,6 +386,7 @@ async fn image_generation_call_event_is_emitted() -> anyhow::Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -468,6 +474,7 @@ async fn image_generation_call_event_is_emitted_when_image_save_fails() -> anyho }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -524,6 +531,7 @@ async fn agent_message_content_delta_has_item_metadata() -> anyhow::Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -1108,6 +1116,7 @@ async fn reasoning_content_delta_has_item_metadata() -> anyhow::Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -1163,6 +1172,7 @@ async fn reasoning_raw_content_delta_respects_flag() -> anyhow::Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; diff --git a/codex-rs/core/tests/suite/json_result.rs b/codex-rs/core/tests/suite/json_result.rs index e81d7aa82a..67275d3146 100644 --- a/codex-rs/core/tests/suite/json_result.rs +++ b/codex-rs/core/tests/suite/json_result.rs @@ -83,6 +83,7 @@ async fn codex_returns_json_result(model: String) -> anyhow::Result<()> { environments: None, final_output_json_schema: Some(serde_json::from_str(SCHEMA)?), responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(cwd.path().to_path_buf()), approval_policy: Some(AskForApproval::Never), diff --git a/codex-rs/core/tests/suite/mcp_turn_metadata.rs b/codex-rs/core/tests/suite/mcp_turn_metadata.rs index 16e6cbe37d..897d862162 100644 --- a/codex-rs/core/tests/suite/mcp_turn_metadata.rs +++ b/codex-rs/core/tests/suite/mcp_turn_metadata.rs @@ -76,6 +76,7 @@ async fn submit_user_turn( environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(test.cwd.path().to_path_buf()), approval_policy: Some(approval_policy), diff --git a/codex-rs/core/tests/suite/mod.rs b/codex-rs/core/tests/suite/mod.rs index 87772aa279..f47de672bd 100644 --- a/codex-rs/core/tests/suite/mod.rs +++ b/codex-rs/core/tests/suite/mod.rs @@ -29,6 +29,7 @@ pub static CODEX_ALIASES_TEMP_DIR: Option = { #[cfg(not(target_os = "windows"))] mod abort_tasks; +mod additional_context; mod agent_jobs; mod agent_websocket; mod agents_md; diff --git a/codex-rs/core/tests/suite/model_switching.rs b/codex-rs/core/tests/suite/model_switching.rs index 7e77fb79d3..fd237a8d5c 100644 --- a/codex-rs/core/tests/suite/model_switching.rs +++ b/codex-rs/core/tests/suite/model_switching.rs @@ -48,6 +48,7 @@ fn read_only_user_turn(test: &TestCodex, items: Vec, model: String) - environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(test.cwd_path().to_path_buf()), approval_policy: Some(AskForApproval::Never), diff --git a/codex-rs/core/tests/suite/model_visible_layout.rs b/codex-rs/core/tests/suite/model_visible_layout.rs index 17bc0f6703..e02e1ffaef 100644 --- a/codex-rs/core/tests/suite/model_visible_layout.rs +++ b/codex-rs/core/tests/suite/model_visible_layout.rs @@ -125,6 +125,7 @@ async fn snapshot_model_visible_layout_turn_overrides() -> Result<()> { environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(first_turn_cwd), approval_policy: Some(AskForApproval::Never), @@ -160,6 +161,7 @@ async fn snapshot_model_visible_layout_turn_overrides() -> Result<()> { environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(preturn_context_diff_cwd), approval_policy: Some(AskForApproval::OnRequest), @@ -249,6 +251,7 @@ async fn snapshot_model_visible_layout_cwd_change_does_not_refresh_agents() -> R environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(cwd_one.clone()), approval_policy: Some(AskForApproval::Never), @@ -282,6 +285,7 @@ async fn snapshot_model_visible_layout_cwd_change_does_not_refresh_agents() -> R environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(cwd_two), approval_policy: Some(AskForApproval::Never), @@ -365,6 +369,7 @@ async fn snapshot_model_visible_layout_resume_with_personality_change() -> Resul }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -406,6 +411,7 @@ async fn snapshot_model_visible_layout_resume_with_personality_change() -> Resul environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(resume_override_cwd), approval_policy: Some(AskForApproval::Never), @@ -479,6 +485,7 @@ async fn snapshot_model_visible_layout_resume_override_matches_rollout_model() - }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -520,6 +527,7 @@ async fn snapshot_model_visible_layout_resume_override_matches_rollout_model() - }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; diff --git a/codex-rs/core/tests/suite/models_cache_ttl.rs b/codex-rs/core/tests/suite/models_cache_ttl.rs index 6d11b47cba..fcd90cbaea 100644 --- a/codex-rs/core/tests/suite/models_cache_ttl.rs +++ b/codex-rs/core/tests/suite/models_cache_ttl.rs @@ -100,6 +100,7 @@ async fn renews_cache_ttl_on_matching_models_etag() -> Result<()> { environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(test.cwd_path().to_path_buf()), approval_policy: Some(codex_protocol::protocol::AskForApproval::Never), diff --git a/codex-rs/core/tests/suite/models_etag_responses.rs b/codex-rs/core/tests/suite/models_etag_responses.rs index 49f34d8644..cbd73fffe9 100644 --- a/codex-rs/core/tests/suite/models_etag_responses.rs +++ b/codex-rs/core/tests/suite/models_etag_responses.rs @@ -112,6 +112,7 @@ async fn refresh_models_on_models_etag_mismatch_and_avoid_duplicate_models_fetch environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(cwd_path), approval_policy: Some(AskForApproval::Never), diff --git a/codex-rs/core/tests/suite/otel.rs b/codex-rs/core/tests/suite/otel.rs index 9d2df4e60e..7f5fa260cd 100644 --- a/codex-rs/core/tests/suite/otel.rs +++ b/codex-rs/core/tests/suite/otel.rs @@ -122,6 +122,7 @@ async fn responses_api_emits_api_request_event() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -168,6 +169,7 @@ async fn process_sse_emits_tracing_for_output_item() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -214,6 +216,7 @@ async fn process_sse_emits_failed_event_on_parse_error() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -261,6 +264,7 @@ async fn process_sse_records_failed_event_when_stream_closes_without_completed() }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -328,6 +332,7 @@ async fn process_sse_failed_event_records_response_error_message() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -393,6 +398,7 @@ async fn process_sse_failed_event_logs_parse_error() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -445,6 +451,7 @@ async fn process_sse_failed_event_logs_missing_error() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -506,6 +513,7 @@ async fn process_sse_failed_event_logs_response_completed_parse_error() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -561,6 +569,7 @@ async fn process_sse_emits_completed_telemetry() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -640,6 +649,7 @@ async fn turn_and_completed_response_spans_record_token_usage() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -728,6 +738,7 @@ async fn handle_responses_span_records_response_kind_and_tool_name() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -822,6 +833,7 @@ async fn record_responses_sets_span_fields_for_response_events() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -911,6 +923,7 @@ async fn handle_response_item_records_tool_result_for_custom_tool_call() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -987,6 +1000,7 @@ async fn handle_response_item_records_tool_result_for_function_call() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -1064,6 +1078,7 @@ async fn handle_response_item_records_tool_result_for_shell_command_call() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -1173,6 +1188,7 @@ async fn handle_shell_command_autoapprove_from_config_records_tool_decision() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -1228,6 +1244,7 @@ async fn handle_shell_command_user_approved_records_tool_decision() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -1298,6 +1315,7 @@ async fn handle_shell_command_user_approved_for_session_records_tool_decision() }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -1368,6 +1386,7 @@ async fn handle_sandbox_error_user_approves_retry_records_tool_decision() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -1438,6 +1457,7 @@ async fn handle_shell_command_user_denies_records_tool_decision() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -1508,6 +1528,7 @@ async fn handle_sandbox_error_user_approves_for_session_records_tool_decision() }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -1579,6 +1600,7 @@ async fn handle_sandbox_error_user_denies_records_tool_decision() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await diff --git a/codex-rs/core/tests/suite/pending_input.rs b/codex-rs/core/tests/suite/pending_input.rs index b057b86171..c8eec9bc8e 100644 --- a/codex-rs/core/tests/suite/pending_input.rs +++ b/codex-rs/core/tests/suite/pending_input.rs @@ -103,6 +103,7 @@ async fn submit_user_input(codex: &CodexThread, text: &str) { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -121,6 +122,7 @@ async fn submit_danger_full_access_user_turn(test: &TestCodex, text: &str) { environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(test.config.cwd.to_path_buf()), approval_policy: Some(AskForApproval::Never), @@ -148,6 +150,7 @@ async fn steer_user_input(codex: &CodexThread, text: &str) { text: text.to_string(), text_elements: Vec::new(), }], + /*additional_context*/ Default::default(), /*expected_turn_id*/ None, /*responsesapi_client_metadata*/ None, ) @@ -291,6 +294,7 @@ async fn injected_user_input_triggers_follow_up_request_with_deltas() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -310,6 +314,7 @@ async fn injected_user_input_triggers_follow_up_request_with_deltas() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await diff --git a/codex-rs/core/tests/suite/permissions_messages.rs b/codex-rs/core/tests/suite/permissions_messages.rs index b77e65c0f5..45d2e32549 100644 --- a/codex-rs/core/tests/suite/permissions_messages.rs +++ b/codex-rs/core/tests/suite/permissions_messages.rs @@ -58,6 +58,7 @@ async fn permissions_message_sent_once_on_start() -> Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -98,6 +99,7 @@ async fn permissions_message_added_on_override_change() -> Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -121,6 +123,7 @@ async fn permissions_message_added_on_override_change() -> Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -167,6 +170,7 @@ async fn permissions_message_not_added_when_no_change() -> Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -181,6 +185,7 @@ async fn permissions_message_not_added_when_no_change() -> Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -227,6 +232,7 @@ async fn permissions_message_omitted_when_disabled() -> Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -250,6 +256,7 @@ async fn permissions_message_omitted_when_disabled() -> Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -309,6 +316,7 @@ async fn resume_replays_permissions_messages() -> Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -333,6 +341,7 @@ async fn resume_replays_permissions_messages() -> Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -349,6 +358,7 @@ async fn resume_replays_permissions_messages() -> Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -409,6 +419,7 @@ async fn resume_and_fork_append_permissions_messages() -> Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -433,6 +444,7 @@ async fn resume_and_fork_append_permissions_messages() -> Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -455,6 +467,7 @@ async fn resume_and_fork_append_permissions_messages() -> Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -491,6 +504,7 @@ async fn resume_and_fork_append_permissions_messages() -> Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -552,6 +566,7 @@ async fn permissions_message_includes_writable_roots() -> Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; diff --git a/codex-rs/core/tests/suite/personality.rs b/codex-rs/core/tests/suite/personality.rs index 96695769c9..06781dc11d 100644 --- a/codex-rs/core/tests/suite/personality.rs +++ b/codex-rs/core/tests/suite/personality.rs @@ -68,6 +68,7 @@ fn read_only_text_turn_with_personality( environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(test.cwd_path().to_path_buf()), approval_policy: Some(approval_policy), diff --git a/codex-rs/core/tests/suite/plugins.rs b/codex-rs/core/tests/suite/plugins.rs index 917ab08554..b89b611a83 100644 --- a/codex-rs/core/tests/suite/plugins.rs +++ b/codex-rs/core/tests/suite/plugins.rs @@ -227,6 +227,7 @@ async fn capability_sections_render_in_developer_message_in_order() -> Result<() }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -306,6 +307,7 @@ async fn explicit_plugin_mentions_inject_plugin_guidance() -> Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -388,6 +390,7 @@ async fn explicit_plugin_mentions_track_plugin_used_analytics() -> Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; diff --git a/codex-rs/core/tests/suite/prompt_caching.rs b/codex-rs/core/tests/suite/prompt_caching.rs index 4ce137a557..836761961a 100644 --- a/codex-rs/core/tests/suite/prompt_caching.rs +++ b/codex-rs/core/tests/suite/prompt_caching.rs @@ -153,6 +153,7 @@ async fn prompt_tools_are_consistent_across_requests() -> anyhow::Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -167,6 +168,7 @@ async fn prompt_tools_are_consistent_across_requests() -> anyhow::Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -250,6 +252,7 @@ async fn gpt_5_tools_without_apply_patch_append_apply_patch_instructions() -> an }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -264,6 +267,7 @@ async fn gpt_5_tools_without_apply_patch_append_apply_patch_instructions() -> an }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -329,6 +333,7 @@ async fn prefixes_context_and_instructions_once_and_consistently_across_requests }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -343,6 +348,7 @@ async fn prefixes_context_and_instructions_once_and_consistently_across_requests }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -426,6 +432,7 @@ async fn overrides_turn_context_but_keeps_cached_prefix_and_key_constant() -> an }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -464,6 +471,7 @@ async fn overrides_turn_context_but_keeps_cached_prefix_and_key_constant() -> an }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -545,6 +553,7 @@ async fn override_before_first_turn_emits_environment_context() -> anyhow::Resul }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -699,6 +708,7 @@ async fn per_turn_overrides_keep_cached_prefix_and_key_constant() -> anyhow::Res }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -724,6 +734,7 @@ async fn per_turn_overrides_keep_cached_prefix_and_key_constant() -> anyhow::Res environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(new_cwd.path().to_path_buf()), approval_policy: Some(AskForApproval::Never), @@ -838,6 +849,7 @@ async fn send_user_turn_with_no_changes_does_not_send_environment_context() -> a environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(default_cwd.to_path_buf()), approval_policy: Some(default_approval_policy), @@ -866,6 +878,7 @@ async fn send_user_turn_with_no_changes_does_not_send_environment_context() -> a environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(default_cwd.to_path_buf()), approval_policy: Some(default_approval_policy), @@ -977,6 +990,7 @@ async fn send_user_turn_with_changes_sends_environment_context() -> anyhow::Resu environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(default_cwd.to_path_buf()), approval_policy: Some(default_approval_policy), @@ -1007,6 +1021,7 @@ async fn send_user_turn_with_changes_sends_environment_context() -> anyhow::Resu environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(default_cwd.to_path_buf()), approval_policy: Some(AskForApproval::Never), diff --git a/codex-rs/core/tests/suite/quota_exceeded.rs b/codex-rs/core/tests/suite/quota_exceeded.rs index 413855f641..904c116cfb 100644 --- a/codex-rs/core/tests/suite/quota_exceeded.rs +++ b/codex-rs/core/tests/suite/quota_exceeded.rs @@ -48,6 +48,7 @@ async fn quota_exceeded_emits_single_error_event() -> Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await diff --git a/codex-rs/core/tests/suite/realtime_conversation.rs b/codex-rs/core/tests/suite/realtime_conversation.rs index 232d2380b0..05428799ad 100644 --- a/codex-rs/core/tests/suite/realtime_conversation.rs +++ b/codex-rs/core/tests/suite/realtime_conversation.rs @@ -2168,6 +2168,7 @@ async fn conversation_user_text_turn_is_sent_to_realtime_when_active() -> Result }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -2303,6 +2304,7 @@ async fn conversation_user_text_turn_is_capped_when_mirrored_to_realtime() -> Re }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -3499,6 +3501,7 @@ async fn inbound_handoff_request_steers_active_turn() -> Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; diff --git a/codex-rs/core/tests/suite/remote_env.rs b/codex-rs/core/tests/suite/remote_env.rs index 873796ff6c..34cd119f66 100644 --- a/codex-rs/core/tests/suite/remote_env.rs +++ b/codex-rs/core/tests/suite/remote_env.rs @@ -75,6 +75,7 @@ async fn submit_turn_with_approval_and_environments( environments: Some(environments), final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(test.cwd.path().to_path_buf()), approval_policy: Some(AskForApproval::OnRequest), diff --git a/codex-rs/core/tests/suite/remote_models.rs b/codex-rs/core/tests/suite/remote_models.rs index 18ebd732b7..b7b16f9c29 100644 --- a/codex-rs/core/tests/suite/remote_models.rs +++ b/codex-rs/core/tests/suite/remote_models.rs @@ -160,6 +160,7 @@ async fn remote_models_config_context_window_override_clamps_to_max_context_wind environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -227,6 +228,7 @@ async fn remote_models_config_override_above_max_uses_max_context_window() -> Re environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -293,6 +295,7 @@ async fn remote_models_use_context_window_when_config_override_is_absent() -> Re environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -372,6 +375,7 @@ async fn remote_models_long_model_slug_is_sent_with_high_reasoning() -> Result<( environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -422,6 +426,7 @@ async fn namespaced_model_slug_uses_catalog_metadata_without_fallback_warning() environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -575,6 +580,7 @@ async fn remote_models_remote_model_uses_unified_exec() -> Result<()> { environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(cwd_path), approval_policy: Some(AskForApproval::Never), @@ -798,6 +804,7 @@ async fn remote_models_apply_remote_base_instructions() -> Result<()> { environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(cwd_path), approval_policy: Some(AskForApproval::Never), diff --git a/codex-rs/core/tests/suite/request_compression.rs b/codex-rs/core/tests/suite/request_compression.rs index 2d66389d53..fe9cb1e5f8 100644 --- a/codex-rs/core/tests/suite/request_compression.rs +++ b/codex-rs/core/tests/suite/request_compression.rs @@ -47,6 +47,7 @@ async fn request_body_is_zstd_compressed_for_codex_backend_when_enabled() -> any }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -97,6 +98,7 @@ async fn request_body_is_not_compressed_for_api_key_auth_even_when_enabled() -> }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; diff --git a/codex-rs/core/tests/suite/request_permissions.rs b/codex-rs/core/tests/suite/request_permissions.rs index 9800104685..a78e4a016e 100644 --- a/codex-rs/core/tests/suite/request_permissions.rs +++ b/codex-rs/core/tests/suite/request_permissions.rs @@ -198,6 +198,7 @@ async fn submit_turn( environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(test.cwd.path().to_path_buf()), approval_policy: Some(approval_policy), diff --git a/codex-rs/core/tests/suite/request_permissions_tool.rs b/codex-rs/core/tests/suite/request_permissions_tool.rs index 84e101e73a..a122cc399b 100644 --- a/codex-rs/core/tests/suite/request_permissions_tool.rs +++ b/codex-rs/core/tests/suite/request_permissions_tool.rs @@ -150,6 +150,7 @@ async fn submit_turn( environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(test.cwd.path().to_path_buf()), approval_policy: Some(approval_policy), diff --git a/codex-rs/core/tests/suite/request_user_input.rs b/codex-rs/core/tests/suite/request_user_input.rs index e0ec684797..d09071d72d 100644 --- a/codex-rs/core/tests/suite/request_user_input.rs +++ b/codex-rs/core/tests/suite/request_user_input.rs @@ -144,6 +144,7 @@ async fn request_user_input_round_trip_for_mode(mode: ModeKind) -> anyhow::Resul environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(cwd.path().to_path_buf()), approval_policy: Some(AskForApproval::Never), @@ -287,6 +288,7 @@ async fn request_user_input_interrupt_emits_deferred_token_count() -> anyhow::Re environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(cwd.path().to_path_buf()), approval_policy: Some(AskForApproval::Never), @@ -391,6 +393,7 @@ where environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(cwd.path().to_path_buf()), approval_policy: Some(AskForApproval::Never), diff --git a/codex-rs/core/tests/suite/responses_api_proxy_headers.rs b/codex-rs/core/tests/suite/responses_api_proxy_headers.rs index c18b431984..809a8bb701 100644 --- a/codex-rs/core/tests/suite/responses_api_proxy_headers.rs +++ b/codex-rs/core/tests/suite/responses_api_proxy_headers.rs @@ -143,6 +143,7 @@ async fn submit_turn_with_timeout(test: &TestCodex, prompt: &str) -> Result<()> environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(cwd), approval_policy: Some(AskForApproval::OnRequest), diff --git a/codex-rs/core/tests/suite/resume.rs b/codex-rs/core/tests/suite/resume.rs index fc7a23215f..0dc7eaddcb 100644 --- a/codex-rs/core/tests/suite/resume.rs +++ b/codex-rs/core/tests/suite/resume.rs @@ -93,6 +93,7 @@ async fn resume_includes_initial_messages_from_rollout_events() -> Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -179,6 +180,7 @@ async fn resume_includes_initial_messages_from_reasoning_events() -> Result<()> }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -269,6 +271,7 @@ async fn resume_switches_models_preserves_base_instructions() -> Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -312,6 +315,7 @@ async fn resume_switches_models_preserves_base_instructions() -> Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -330,6 +334,7 @@ async fn resume_switches_models_preserves_base_instructions() -> Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -403,6 +408,7 @@ async fn resume_model_switch_is_not_duplicated_after_pre_turn_override() -> Resu }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -441,6 +447,7 @@ async fn resume_model_switch_is_not_duplicated_after_pre_turn_override() -> Resu }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; diff --git a/codex-rs/core/tests/suite/review.rs b/codex-rs/core/tests/suite/review.rs index 05ec967d09..802035d7dc 100644 --- a/codex-rs/core/tests/suite/review.rs +++ b/codex-rs/core/tests/suite/review.rs @@ -684,6 +684,7 @@ async fn review_history_surfaces_in_parent_session() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await diff --git a/codex-rs/core/tests/suite/rmcp_client.rs b/codex-rs/core/tests/suite/rmcp_client.rs index 260edb3f64..2c86cb393c 100644 --- a/codex-rs/core/tests/suite/rmcp_client.rs +++ b/codex-rs/core/tests/suite/rmcp_client.rs @@ -131,6 +131,7 @@ fn user_turn_with_permission_profile( environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(cwd), approval_policy: Some(AskForApproval::Never), diff --git a/codex-rs/core/tests/suite/safety_check_downgrade.rs b/codex-rs/core/tests/suite/safety_check_downgrade.rs index ab985ce0f7..3d4cb93b9c 100644 --- a/codex-rs/core/tests/suite/safety_check_downgrade.rs +++ b/codex-rs/core/tests/suite/safety_check_downgrade.rs @@ -45,6 +45,7 @@ fn disabled_text_turn(test: &TestCodex, text: &str) -> Op { environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(test.cwd_path().to_path_buf()), approval_policy: Some(AskForApproval::Never), diff --git a/codex-rs/core/tests/suite/search_tool.rs b/codex-rs/core/tests/suite/search_tool.rs index bbf16b3657..e9ffeb62ef 100644 --- a/codex-rs/core/tests/suite/search_tool.rs +++ b/codex-rs/core/tests/suite/search_tool.rs @@ -452,6 +452,7 @@ async fn tool_search_returns_deferred_tools_without_follow_up_tool_injection() - }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -865,6 +866,7 @@ async fn tool_search_returns_deferred_dynamic_tool_and_routes_follow_up_call() - }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -1173,6 +1175,7 @@ async fn tool_search_surfaced_mcp_tool_errors_are_returned_to_model() -> Result< }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; @@ -1493,6 +1496,7 @@ async fn tool_search_matches_dynamic_tools_by_name_description_namespace_and_sch }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; diff --git a/codex-rs/core/tests/suite/shell_snapshot.rs b/codex-rs/core/tests/suite/shell_snapshot.rs index 14241ffafb..69db30b21b 100644 --- a/codex-rs/core/tests/suite/shell_snapshot.rs +++ b/codex-rs/core/tests/suite/shell_snapshot.rs @@ -167,6 +167,7 @@ async fn run_snapshot_command_with_options( environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(cwd), approval_policy: Some(AskForApproval::Never), @@ -267,6 +268,7 @@ async fn run_shell_command_snapshot_with_options( environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(cwd), approval_policy: Some(AskForApproval::Never), @@ -347,6 +349,7 @@ async fn run_tool_turn_on_harness( environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(cwd), approval_policy: Some(AskForApproval::Never), @@ -590,6 +593,7 @@ async fn shell_command_snapshot_still_intercepts_apply_patch() -> Result<()> { environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(cwd.clone()), approval_policy: Some(AskForApproval::Never), diff --git a/codex-rs/core/tests/suite/skill_approval.rs b/codex-rs/core/tests/suite/skill_approval.rs index ad8ad6a91a..2335ba6415 100644 --- a/codex-rs/core/tests/suite/skill_approval.rs +++ b/codex-rs/core/tests/suite/skill_approval.rs @@ -54,6 +54,7 @@ async fn submit_turn_with_policies( environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(test.cwd_path().to_path_buf()), approval_policy: Some(approval_policy), diff --git a/codex-rs/core/tests/suite/skills.rs b/codex-rs/core/tests/suite/skills.rs index 9173f60c4a..e21668fa84 100644 --- a/codex-rs/core/tests/suite/skills.rs +++ b/codex-rs/core/tests/suite/skills.rs @@ -88,6 +88,7 @@ async fn user_turn_includes_skill_instructions() -> Result<()> { environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(test.config.cwd.to_path_buf()), approval_policy: Some(AskForApproval::Never), diff --git a/codex-rs/core/tests/suite/snapshots/all__suite__additional_context__additional_context_simple_input.snap b/codex-rs/core/tests/suite/snapshots/all__suite__additional_context__additional_context_simple_input.snap new file mode 100644 index 0000000000..fefdfde60c --- /dev/null +++ b/codex-rs/core/tests/suite/snapshots/all__suite__additional_context__additional_context_simple_input.snap @@ -0,0 +1,11 @@ +--- +source: core/tests/suite/additional_context.rs +expression: "context_snapshot::format_labeled_requests_snapshot(\"additional context is inserted before the user turn input.\",\n&[(\"Request\", &request)], &additional_context_snapshot_options(),)" +--- +Scenario: additional context is inserted before the user turn input. + +## Request +00:message/developer: +01:message/developer:run one +02:message/user:tab one +03:message/user:inspect the active tab diff --git a/codex-rs/core/tests/suite/sqlite_state.rs b/codex-rs/core/tests/suite/sqlite_state.rs index 6f7a51069a..99b7fb0ec6 100644 --- a/codex-rs/core/tests/suite/sqlite_state.rs +++ b/codex-rs/core/tests/suite/sqlite_state.rs @@ -413,6 +413,7 @@ async fn mcp_call_marks_thread_memory_mode_polluted_when_configured() -> Result< environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(cwd), approval_policy: Some(AskForApproval::Never), diff --git a/codex-rs/core/tests/suite/stream_error_allows_next_turn.rs b/codex-rs/core/tests/suite/stream_error_allows_next_turn.rs index 5709cdd12b..d82692c26c 100644 --- a/codex-rs/core/tests/suite/stream_error_allows_next_turn.rs +++ b/codex-rs/core/tests/suite/stream_error_allows_next_turn.rs @@ -101,6 +101,7 @@ async fn continue_after_stream_error() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await @@ -123,6 +124,7 @@ async fn continue_after_stream_error() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await diff --git a/codex-rs/core/tests/suite/stream_no_completed.rs b/codex-rs/core/tests/suite/stream_no_completed.rs index 40763960f4..471c60db5a 100644 --- a/codex-rs/core/tests/suite/stream_no_completed.rs +++ b/codex-rs/core/tests/suite/stream_no_completed.rs @@ -83,6 +83,7 @@ async fn retries_on_early_close() { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await diff --git a/codex-rs/core/tests/suite/subagent_notifications.rs b/codex-rs/core/tests/suite/subagent_notifications.rs index d3c07a1153..8891f6ffec 100644 --- a/codex-rs/core/tests/suite/subagent_notifications.rs +++ b/codex-rs/core/tests/suite/subagent_notifications.rs @@ -771,6 +771,7 @@ async fn subagent_stop_replaces_stop_and_skips_internal_subagents() -> Result<() environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(test.config.cwd.to_path_buf()), approval_policy: Some(AskForApproval::Never), diff --git a/codex-rs/core/tests/suite/tool_harness.rs b/codex-rs/core/tests/suite/tool_harness.rs index 2227e5b34e..a6f808b9d7 100644 --- a/codex-rs/core/tests/suite/tool_harness.rs +++ b/codex-rs/core/tests/suite/tool_harness.rs @@ -110,6 +110,7 @@ async fn shell_command_tool_executes_command_and_streams_output() -> anyhow::Res environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(cwd_path), approval_policy: Some(AskForApproval::Never), @@ -191,6 +192,7 @@ async fn update_plan_tool_emits_plan_update_event() -> anyhow::Result<()> { environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(cwd_path), approval_policy: Some(AskForApproval::Never), @@ -282,6 +284,7 @@ async fn update_plan_tool_rejects_malformed_payload() -> anyhow::Result<()> { environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(cwd_path), approval_policy: Some(AskForApproval::Never), @@ -383,6 +386,7 @@ async fn apply_patch_tool_executes_and_emits_patch_events() -> anyhow::Result<() environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(cwd_path), approval_policy: Some(AskForApproval::Never), @@ -521,6 +525,7 @@ async fn apply_patch_reports_parse_diagnostics() -> anyhow::Result<()> { environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(cwd_path), approval_policy: Some(AskForApproval::Never), diff --git a/codex-rs/core/tests/suite/tool_parallelism.rs b/codex-rs/core/tests/suite/tool_parallelism.rs index 6663991e02..76bb27c423 100644 --- a/codex-rs/core/tests/suite/tool_parallelism.rs +++ b/codex-rs/core/tests/suite/tool_parallelism.rs @@ -45,6 +45,7 @@ async fn run_turn(test: &TestCodex, prompt: &str) -> anyhow::Result<()> { environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(test.cwd.path().to_path_buf()), approval_policy: Some(AskForApproval::Never), @@ -371,6 +372,7 @@ async fn shell_tools_start_before_response_completed_when_stream_delayed() -> an environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(test.cwd.path().to_path_buf()), approval_policy: Some(AskForApproval::Never), diff --git a/codex-rs/core/tests/suite/truncation.rs b/codex-rs/core/tests/suite/truncation.rs index 04b59a3331..fe078977ff 100644 --- a/codex-rs/core/tests/suite/truncation.rs +++ b/codex-rs/core/tests/suite/truncation.rs @@ -523,6 +523,7 @@ async fn mcp_image_output_preserves_image_and_no_text_summary() -> Result<()> { environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(fixture.cwd.path().to_path_buf()), approval_policy: Some(AskForApproval::Never), diff --git a/codex-rs/core/tests/suite/unified_exec.rs b/codex-rs/core/tests/suite/unified_exec.rs index 9eb831b084..ba754b025f 100644 --- a/codex-rs/core/tests/suite/unified_exec.rs +++ b/codex-rs/core/tests/suite/unified_exec.rs @@ -200,6 +200,7 @@ async fn submit_unified_exec_turn( environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(test.config.cwd.to_path_buf()), approval_policy: Some(AskForApproval::Never), @@ -291,6 +292,7 @@ async fn unified_exec_intercepts_apply_patch_exec_command() -> Result<()> { environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(cwd), approval_policy: Some(AskForApproval::Never), @@ -2146,6 +2148,7 @@ async fn unified_exec_keeps_long_running_session_after_turn_end() -> Result<()> environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(turn_cwd), approval_policy: Some(AskForApproval::Never), @@ -2249,6 +2252,7 @@ async fn unified_exec_interrupt_preserves_long_running_session() -> Result<()> { environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(turn_cwd), approval_policy: Some(AskForApproval::Never), @@ -2721,6 +2725,7 @@ async fn unified_exec_runs_under_sandbox() -> Result<()> { environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(turn_cwd), approval_policy: Some(AskForApproval::Never), @@ -2843,6 +2848,7 @@ async fn unified_exec_enforces_glob_deny_read_policy() -> Result<()> { environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(turn_cwd), approval_policy: Some(AskForApproval::Never), @@ -2981,6 +2987,7 @@ async fn unified_exec_python_prompt_under_seatbelt() -> Result<()> { environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(turn_cwd), approval_policy: Some(AskForApproval::Never), diff --git a/codex-rs/core/tests/suite/user_notification.rs b/codex-rs/core/tests/suite/user_notification.rs index 343afd8d53..054d926b2b 100644 --- a/codex-rs/core/tests/suite/user_notification.rs +++ b/codex-rs/core/tests/suite/user_notification.rs @@ -64,6 +64,7 @@ mv "${tmp_path}" "${payload_path}""#, }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; diff --git a/codex-rs/core/tests/suite/user_shell_cmd.rs b/codex-rs/core/tests/suite/user_shell_cmd.rs index 536ee7b495..3e2263705a 100644 --- a/codex-rs/core/tests/suite/user_shell_cmd.rs +++ b/codex-rs/core/tests/suite/user_shell_cmd.rs @@ -182,6 +182,7 @@ async fn user_shell_command_does_not_replace_active_turn() -> anyhow::Result<()> environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(cwd), approval_policy: Some(AskForApproval::Never), diff --git a/codex-rs/core/tests/suite/view_image.rs b/codex-rs/core/tests/suite/view_image.rs index 867ba10858..e6320ba1a7 100644 --- a/codex-rs/core/tests/suite/view_image.rs +++ b/codex-rs/core/tests/suite/view_image.rs @@ -77,6 +77,7 @@ fn disabled_user_turn(test: &TestCodex, items: Vec, model: String) -> environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(test.config.cwd.to_path_buf()), approval_policy: Some(AskForApproval::Never), diff --git a/codex-rs/core/tests/suite/websocket_fallback.rs b/codex-rs/core/tests/suite/websocket_fallback.rs index 32a6e30caf..be33467d6a 100644 --- a/codex-rs/core/tests/suite/websocket_fallback.rs +++ b/codex-rs/core/tests/suite/websocket_fallback.rs @@ -160,6 +160,7 @@ async fn websocket_fallback_hides_first_websocket_retry_stream_error() -> Result environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: codex_protocol::protocol::ThreadSettingsOverrides { cwd: Some(cwd.path().to_path_buf()), approval_policy: Some(AskForApproval::Never), diff --git a/codex-rs/core/tests/suite/window_headers.rs b/codex-rs/core/tests/suite/window_headers.rs index 76fcacf7a9..953b075f65 100644 --- a/codex-rs/core/tests/suite/window_headers.rs +++ b/codex-rs/core/tests/suite/window_headers.rs @@ -112,6 +112,7 @@ async fn submit_user_turn(codex: &Arc, text: &str) -> Result<()> { }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await?; diff --git a/codex-rs/exec/src/lib.rs b/codex-rs/exec/src/lib.rs index 9061227b6b..2df97b5bd7 100644 --- a/codex-rs/exec/src/lib.rs +++ b/codex-rs/exec/src/lib.rs @@ -781,6 +781,7 @@ async fn run_exec_session(args: ExecRunArgs) -> anyhow::Result<()> { thread_id: primary_thread_id_for_span.clone(), input: items.into_iter().map(Into::into).collect(), responsesapi_client_metadata: None, + additional_context: None, environments: None, cwd: Some(default_cwd), runtime_workspace_roots: None, diff --git a/codex-rs/ext/goal/src/steering.rs b/codex-rs/ext/goal/src/steering.rs index e08c47ae26..83f41ea538 100644 --- a/codex-rs/ext/goal/src/steering.rs +++ b/codex-rs/ext/goal/src/steering.rs @@ -1,3 +1,4 @@ +use codex_core::context::ContextualUserFragment; use codex_core::context::GoalContext; use codex_protocol::models::ResponseInputItem; use codex_protocol::protocol::ThreadGoal; diff --git a/codex-rs/mcp-server/src/codex_tool_runner.rs b/codex-rs/mcp-server/src/codex_tool_runner.rs index 167d56da5b..78714d1eaa 100644 --- a/codex-rs/mcp-server/src/codex_tool_runner.rs +++ b/codex-rs/mcp-server/src/codex_tool_runner.rs @@ -116,6 +116,7 @@ pub async fn run_codex_tool_session( }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }, trace: None, @@ -166,6 +167,7 @@ pub async fn run_codex_tool_session_reply( }], final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await diff --git a/codex-rs/memories/write/src/runtime.rs b/codex-rs/memories/write/src/runtime.rs index b1ffb2d215..7bf2fe159d 100644 --- a/codex-rs/memories/write/src/runtime.rs +++ b/codex-rs/memories/write/src/runtime.rs @@ -261,6 +261,7 @@ impl MemoryStartupContext { environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index d3b8a9e820..a675362b62 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -3,6 +3,7 @@ //! Uses a SQ (Submission Queue) / EQ (Event Queue) pattern to asynchronously communicate //! between user and agent. +use std::collections::BTreeMap; use std::collections::HashMap; use std::fmt; use std::ops::Mul; @@ -471,6 +472,21 @@ pub struct ThreadSettingsOverrides { pub personality: Option, } +/// Source classification for client-supplied context. +#[derive(Debug, Clone, Copy, Deserialize, Serialize, PartialEq, Eq, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum AdditionalContextKind { + Untrusted, + Application, +} + +/// Client-supplied context keyed by an opaque source identifier. +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq, JsonSchema)] +pub struct AdditionalContextEntry { + pub value: String, + pub kind: AdditionalContextKind, +} + /// Submission operation #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, JsonSchema)] #[serde(tag = "type", rename_all = "snake_case")] @@ -513,6 +529,9 @@ pub enum Op { /// Optional turn-scoped Responses API `client_metadata`. #[serde(default, skip_serializing_if = "Option::is_none")] responsesapi_client_metadata: Option>, + /// Client-supplied context fragments keyed by an opaque source identifier. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + additional_context: BTreeMap, /// Persistent thread-settings overrides to apply before the input. #[serde(default, flatten)] @@ -655,6 +674,7 @@ impl From> for Op { items: value, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: ThreadSettingsOverrides::default(), } } @@ -4954,6 +4974,7 @@ mod tests { items: Vec::new(), final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }; @@ -4974,6 +4995,7 @@ mod tests { items: Vec::new(), final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), } ); @@ -4996,6 +5018,7 @@ mod tests { items: Vec::new(), final_output_json_schema: Some(schema.clone()), responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }; @@ -5022,6 +5045,7 @@ mod tests { "fiber_run_id".to_string(), "fiber-123".to_string(), )])), + additional_context: Default::default(), thread_settings: Default::default(), }; diff --git a/codex-rs/thread-manager-sample/src/main.rs b/codex-rs/thread-manager-sample/src/main.rs index 56938f01e4..fb407de3e9 100644 --- a/codex-rs/thread-manager-sample/src/main.rs +++ b/codex-rs/thread-manager-sample/src/main.rs @@ -297,6 +297,7 @@ async fn run_turn(thread: &CodexThread, thread_id: &str, prompt: String) -> anyh environments: None, final_output_json_schema: None, responsesapi_client_metadata: None, + additional_context: Default::default(), thread_settings: Default::default(), }) .await diff --git a/codex-rs/tui/src/app_server_session.rs b/codex-rs/tui/src/app_server_session.rs index 91ed9176ff..aa2d5a645b 100644 --- a/codex-rs/tui/src/app_server_session.rs +++ b/codex-rs/tui/src/app_server_session.rs @@ -670,6 +670,7 @@ impl AppServerSession { thread_id: thread_id.to_string(), input: items, responsesapi_client_metadata: None, + additional_context: None, environments: None, cwd: Some(cwd), runtime_workspace_roots: Some( @@ -733,6 +734,7 @@ impl AppServerSession { thread_id: thread_id.to_string(), input: items, responsesapi_client_metadata: None, + additional_context: None, expected_turn_id: turn_id, }, }) From 3da89d4831251b704ac40d1357d26ad518abaf1d Mon Sep 17 00:00:00 2001 From: Anton Panasenko Date: Tue, 26 May 2026 13:17:58 -0700 Subject: [PATCH 10/16] fix(remote-control): surface websocket task stalls (#24473) ## Why When the app-server remote-control websocket path stalls during connection setup or teardown, the existing logs do not show where the task stopped, and several awaits can keep the task from returning promptly. That makes offline or stale-host incidents hard to distinguish from expected shutdown or disable flow. Issue: N/A (internal incident investigation) ## What Changed Added structured lifecycle and status logging around remote-control enable/disable requests, websocket task startup and exit, connection cycles, enrollment context, and status/environment transitions. Bound websocket connect, transport-event forwarding, and connection-worker shutdown waits. On timeout, the code logs the stalled operation and stops or aborts workers so the loop can reconnect or exit instead of waiting indefinitely. Ping sends now also observe shutdown cancellation. --- .../remote_control/client_tracker.rs | 412 +++++++++++++++++- .../src/transport/remote_control/mod.rs | 101 ++++- .../src/transport/remote_control/websocket.rs | 321 ++++++++++++-- 3 files changed, 764 insertions(+), 70 deletions(-) diff --git a/codex-rs/app-server-transport/src/transport/remote_control/client_tracker.rs b/codex-rs/app-server-transport/src/transport/remote_control/client_tracker.rs index 4639942b08..22e5e74eda 100644 --- a/codex-rs/app-server-transport/src/transport/remote_control/client_tracker.rs +++ b/codex-rs/app-server-transport/src/transport/remote_control/client_tracker.rs @@ -15,13 +15,21 @@ use codex_app_server_protocol::JSONRPCMessage; use std::collections::HashMap; use tokio::sync::mpsc; use tokio::sync::watch; +use tokio::task::JoinHandle; use tokio::task::JoinSet; use tokio::time::Duration; use tokio::time::Instant; +use tokio::time::timeout; use tokio_util::sync::CancellationToken; +use tracing::info; +use tracing::warn; const REMOTE_CONTROL_CLIENT_IDLE_TIMEOUT: Duration = Duration::from_secs(10 * 60); pub(crate) const REMOTE_CONTROL_IDLE_SWEEP_INTERVAL: Duration = Duration::from_secs(30); +#[cfg(not(test))] +const REMOTE_CONTROL_TRANSPORT_EVENT_SEND_TIMEOUT: Duration = Duration::from_secs(5); +#[cfg(test)] +const REMOTE_CONTROL_TRANSPORT_EVENT_SEND_TIMEOUT: Duration = Duration::from_millis(10); #[derive(Debug)] pub(crate) struct Stopped; @@ -138,9 +146,6 @@ impl ClientTracker { if let Some(connection_id) = self.clients.get_mut(&client_key).map(|client| { client.last_activity_at = Instant::now(); - if let Some(seq_id) = seq_id { - client.last_inbound_seq_id = Some(seq_id); - } client.connection_id }) { self.send_transport_event(TransportEvent::IncomingMessage { @@ -148,6 +153,7 @@ impl ClientTracker { message, }) .await?; + self.record_inbound_message_delivery(&client_key, seq_id); return Ok(()); } @@ -177,23 +183,37 @@ impl ClientTracker { disconnect_token.clone(), )); self.clients.insert( - client_key, + client_key.clone(), ClientState { connection_id, disconnect_token, last_activity_at: Instant::now(), - last_inbound_seq_id: if is_legacy_stream_id { None } else { seq_id }, + last_inbound_seq_id: None, status_tx, }, ); if is_legacy_stream_id { self.legacy_stream_ids.insert(client_id.clone(), stream_id); } - self.send_transport_event(TransportEvent::IncomingMessage { - connection_id, - message, - }) - .await + if let Err(err) = self + .send_transport_event(TransportEvent::IncomingMessage { + connection_id, + message, + }) + .await + { + if let Some(client) = self.remove_client(&client_key) { + client.disconnect_token.cancel(); + // The initialize send already timed out on this queue; preserve close + // delivery without blocking reconnect on the same backpressure. + drop(self.spawn_connection_closed(client.connection_id)); + } + return Err(err); + } + if !is_legacy_stream_id { + self.record_inbound_message_delivery(&client_key, seq_id); + } + Ok(()) } ClientEvent::ClientMessageChunk { .. } | ClientEvent::Ack { .. } => Ok(()), ClientEvent::Ping => { @@ -290,16 +310,9 @@ impl ClientTracker { &mut self, client_key: &(ClientId, StreamId), ) -> Result<(), Stopped> { - let Some(client) = self.clients.remove(client_key) else { + let Some(client) = self.remove_client(client_key) else { return Ok(()); }; - if self - .legacy_stream_ids - .get(&client_key.0) - .is_some_and(|stream_id| stream_id == &client_key.1) - { - self.legacy_stream_ids.remove(&client_key.0); - } client.disconnect_token.cancel(); self.send_transport_event(TransportEvent::ConnectionClosed { connection_id: client.connection_id, @@ -307,11 +320,109 @@ impl ClientTracker { .await } + fn remove_client(&mut self, client_key: &(ClientId, StreamId)) -> Option { + let client = self.clients.remove(client_key)?; + if self + .legacy_stream_ids + .get(&client_key.0) + .is_some_and(|stream_id| stream_id == &client_key.1) + { + self.legacy_stream_ids.remove(&client_key.0); + } + Some(client) + } + async fn send_transport_event(&self, event: TransportEvent) -> Result<(), Stopped> { - self.transport_event_tx - .send(event) - .await - .map_err(|_| Stopped) + let event = match event { + TransportEvent::ConnectionClosed { connection_id } => { + return self.send_connection_closed(connection_id).await; + } + event => event, + }; + + let event_name = transport_event_name(&event); + match timeout( + REMOTE_CONTROL_TRANSPORT_EVENT_SEND_TIMEOUT, + self.transport_event_tx.send(event), + ) + .await + { + Ok(Ok(())) => Ok(()), + Ok(Err(_)) => { + warn!( + transport_event = event_name, + "remote control transport event receiver dropped" + ); + Err(Stopped) + } + Err(_) => { + warn!( + transport_event = event_name, + timeout = ?REMOTE_CONTROL_TRANSPORT_EVENT_SEND_TIMEOUT, + "timed out forwarding remote control transport event" + ); + Err(Stopped) + } + } + } + + fn record_inbound_message_delivery( + &mut self, + client_key: &(ClientId, StreamId), + seq_id: Option, + ) { + // Timed forwarding can fail, so only dedupe retries after app-server receives it. + if let Some(seq_id) = seq_id + && let Some(client) = self.clients.get_mut(client_key) + { + client.last_inbound_seq_id = Some(seq_id); + } + } + + async fn send_connection_closed(&self, connection_id: ConnectionId) -> Result<(), Stopped> { + // Worker shutdown can abort the caller; detach the cleanup event before awaiting it. + match self.spawn_connection_closed(connection_id).await { + Ok(result) => result, + Err(err) => { + warn!( + transport_event = "connection_closed", + ?err, + "remote control transport event forwarding task failed" + ); + Err(Stopped) + } + } + } + + fn spawn_connection_closed( + &self, + connection_id: ConnectionId, + ) -> JoinHandle> { + info!( + connection_id = ?connection_id, + "forwarding remote control connection closed transport event" + ); + let transport_event_tx = self.transport_event_tx.clone(); + tokio::spawn(async move { + transport_event_tx + .send(TransportEvent::ConnectionClosed { connection_id }) + .await + .map_err(|_| { + warn!( + transport_event = "connection_closed", + "remote control transport event receiver dropped" + ); + Stopped + }) + }) + } +} + +fn transport_event_name(event: &TransportEvent) -> &'static str { + match event { + TransportEvent::ConnectionOpened { .. } => "connection_opened", + TransportEvent::ConnectionClosed { .. } => "connection_closed", + TransportEvent::IncomingMessage { .. } => "incoming_message", } } @@ -370,6 +481,13 @@ mod tests { } } + fn initialized_notification() -> JSONRPCMessage { + JSONRPCMessage::Notification(codex_app_server_protocol::JSONRPCNotification { + method: "initialized".to_string(), + params: None, + }) + } + #[tokio::test] async fn cancelled_outbound_task_emits_connection_closed() { let (server_event_tx, _server_event_rx) = mpsc::channel(CHANNEL_CAPACITY); @@ -487,6 +605,256 @@ mod tests { .expect("shutdown should not hang on blocked server forwarding"); } + #[tokio::test] + async fn non_close_transport_event_send_times_out_when_queue_stays_full() { + let (server_event_tx, _server_event_rx) = mpsc::channel(CHANNEL_CAPACITY); + let (transport_event_tx, _transport_event_rx) = mpsc::channel(1); + let shutdown_token = CancellationToken::new(); + let client_tracker = + ClientTracker::new(server_event_tx, transport_event_tx.clone(), &shutdown_token); + + transport_event_tx + .send(TransportEvent::ConnectionClosed { + connection_id: next_connection_id(), + }) + .await + .expect("transport event queue should accept prefill"); + + let send_result = client_tracker + .send_transport_event(TransportEvent::IncomingMessage { + connection_id: next_connection_id(), + message: initialized_notification(), + }) + .await; + + assert!(send_result.is_err()); + } + + #[tokio::test] + async fn incoming_message_timeout_does_not_advance_seq_id() { + let (server_event_tx, _server_event_rx) = mpsc::channel(CHANNEL_CAPACITY); + let (transport_event_tx, mut transport_event_rx) = mpsc::channel(2); + let shutdown_token = CancellationToken::new(); + let mut client_tracker = + ClientTracker::new(server_event_tx, transport_event_tx.clone(), &shutdown_token); + + client_tracker + .handle_message(initialize_envelope_with_stream_id( + "client-1", + Some("stream-1"), + )) + .await + .expect("initialize should open client"); + let connection_id = match transport_event_rx.recv().await.expect("open event") { + TransportEvent::ConnectionOpened { connection_id, .. } => connection_id, + other => panic!("expected connection opened, got {other:?}"), + }; + let _ = transport_event_rx.recv().await.expect("initialize event"); + + for _ in 0..2 { + transport_event_tx + .send(TransportEvent::ConnectionClosed { + connection_id: next_connection_id(), + }) + .await + .expect("transport event queue should accept prefill"); + } + + let retry_envelope = ClientEnvelope { + event: ClientEvent::ClientMessage { + message: initialized_notification(), + }, + client_id: ClientId("client-1".to_string()), + stream_id: Some(StreamId("stream-1".to_string())), + seq_id: Some(1), + cursor: None, + }; + assert!( + client_tracker + .handle_message(retry_envelope.clone()) + .await + .is_err() + ); + for _ in 0..2 { + let _ = transport_event_rx.recv().await.expect("prefilled event"); + } + + client_tracker + .handle_message(retry_envelope) + .await + .expect("retry should forward after timeout"); + match transport_event_rx.recv().await.expect("retried event") { + TransportEvent::IncomingMessage { + connection_id: queued_connection_id, + .. + } => assert_eq!(queued_connection_id, connection_id), + other => panic!("expected incoming message, got {other:?}"), + } + } + + #[tokio::test] + async fn initialize_timeout_closes_open_connection() { + let (server_event_tx, _server_event_rx) = mpsc::channel(CHANNEL_CAPACITY); + let (transport_event_tx, mut transport_event_rx) = mpsc::channel(1); + let shutdown_token = CancellationToken::new(); + let client_tracker = + ClientTracker::new(server_event_tx, transport_event_tx, &shutdown_token); + let mut handle_message = tokio::spawn(async move { + let mut client_tracker = client_tracker; + client_tracker + .handle_message(initialize_envelope_with_stream_id( + "client-1", + Some("stream-1"), + )) + .await + }); + + assert!( + timeout(Duration::from_millis(50), &mut handle_message) + .await + .expect("initialize timeout rollback should not wait for close delivery") + .expect("handle message task should not panic") + .is_err() + ); + let connection_id = match transport_event_rx.recv().await.expect("open event") { + TransportEvent::ConnectionOpened { connection_id, .. } => connection_id, + other => panic!("expected connection opened, got {other:?}"), + }; + + match transport_event_rx.recv().await.expect("close event") { + TransportEvent::ConnectionClosed { + connection_id: closed_connection_id, + } => assert_eq!(closed_connection_id, connection_id), + other => panic!("expected connection closed, got {other:?}"), + } + } + + #[tokio::test] + async fn close_client_waits_for_transport_event_queue_capacity() { + let (server_event_tx, _server_event_rx) = mpsc::channel(CHANNEL_CAPACITY); + let (transport_event_tx, mut transport_event_rx) = mpsc::channel(2); + let shutdown_token = CancellationToken::new(); + let mut client_tracker = + ClientTracker::new(server_event_tx, transport_event_tx, &shutdown_token); + + client_tracker + .handle_message(initialize_envelope_with_stream_id( + "client-1", + Some("stream-1"), + )) + .await + .expect("initialize should open client"); + let connection_id = match transport_event_rx.recv().await.expect("open event") { + TransportEvent::ConnectionOpened { connection_id, .. } => connection_id, + other => panic!("expected connection opened, got {other:?}"), + }; + let _ = transport_event_rx.recv().await.expect("initialize event"); + + for _ in 0..2 { + client_tracker + .transport_event_tx + .send(TransportEvent::IncomingMessage { + connection_id, + message: initialized_notification(), + }) + .await + .expect("transport event queue should accept prefill"); + } + + let client_key = ( + ClientId("client-1".to_string()), + StreamId("stream-1".to_string()), + ); + let close_client = client_tracker.close_client(&client_key); + tokio::pin!(close_client); + assert!( + timeout(Duration::from_millis(20), &mut close_client) + .await + .is_err() + ); + + for _ in 0..2 { + match transport_event_rx.recv().await.expect("prefilled event") { + TransportEvent::IncomingMessage { + connection_id: queued_connection_id, + .. + } => assert_eq!(queued_connection_id, connection_id), + other => panic!("expected incoming message, got {other:?}"), + } + } + + close_client + .await + .expect("close should forward after queue drains"); + match transport_event_rx.recv().await.expect("close event") { + TransportEvent::ConnectionClosed { + connection_id: closed_connection_id, + } => assert_eq!(closed_connection_id, connection_id), + other => panic!("expected connection closed, got {other:?}"), + } + } + + #[tokio::test] + async fn close_client_keeps_forwarding_after_caller_is_aborted() { + let (server_event_tx, _server_event_rx) = mpsc::channel(CHANNEL_CAPACITY); + let (transport_event_tx, mut transport_event_rx) = mpsc::channel(2); + let shutdown_token = CancellationToken::new(); + let mut client_tracker = + ClientTracker::new(server_event_tx, transport_event_tx, &shutdown_token); + + client_tracker + .handle_message(initialize_envelope_with_stream_id( + "client-1", + Some("stream-1"), + )) + .await + .expect("initialize should open client"); + let connection_id = match transport_event_rx.recv().await.expect("open event") { + TransportEvent::ConnectionOpened { connection_id, .. } => connection_id, + other => panic!("expected connection opened, got {other:?}"), + }; + let _ = transport_event_rx.recv().await.expect("initialize event"); + + for _ in 0..2 { + client_tracker + .transport_event_tx + .send(TransportEvent::IncomingMessage { + connection_id, + message: initialized_notification(), + }) + .await + .expect("transport event queue should accept prefill"); + } + + let client_key = ( + ClientId("client-1".to_string()), + StreamId("stream-1".to_string()), + ); + let mut close_client = + tokio::spawn(async move { client_tracker.close_client(&client_key).await }); + assert!( + timeout(Duration::from_millis(20), &mut close_client) + .await + .is_err() + ); + close_client.abort(); + let _ = close_client.await; + + for _ in 0..2 { + let _ = transport_event_rx.recv().await.expect("prefilled event"); + } + match timeout(Duration::from_secs(1), transport_event_rx.recv()) + .await + .expect("close should be delivered") + .expect("close event") + { + TransportEvent::ConnectionClosed { + connection_id: closed_connection_id, + } => assert_eq!(closed_connection_id, connection_id), + other => panic!("expected connection closed, got {other:?}"), + } + } + #[tokio::test] async fn initialize_with_new_stream_id_opens_new_connection_for_same_client() { let (server_event_tx, _server_event_rx) = mpsc::channel(CHANNEL_CAPACITY); diff --git a/codex-rs/app-server-transport/src/transport/remote_control/mod.rs b/codex-rs/app-server-transport/src/transport/remote_control/mod.rs index ae6d79462c..357a98a30d 100644 --- a/codex-rs/app-server-transport/src/transport/remote_control/mod.rs +++ b/codex-rs/app-server-transport/src/transport/remote_control/mod.rs @@ -19,16 +19,20 @@ use codex_app_server_protocol::RemoteControlConnectionStatus; use codex_app_server_protocol::RemoteControlStatusChangedNotification; use codex_login::AuthManager; use codex_state::StateRuntime; +use futures::FutureExt; use gethostname::gethostname; use std::error::Error; use std::fmt; use std::io; +use std::panic::AssertUnwindSafe; use std::sync::Arc; use tokio::sync::mpsc; use tokio::sync::oneshot; use tokio::sync::watch; use tokio::task::JoinHandle; use tokio_util::sync::CancellationToken; +use tracing::error; +use tracing::info; use tracing::warn; pub struct RemoteControlStartConfig { @@ -73,13 +77,21 @@ impl RemoteControlHandle { return Err(RemoteControlUnavailable); } - self.enabled_tx.send_if_modified(|state| { + let enabled_changed = self.enabled_tx.send_if_modified(|state| { let changed = !*state; *state = true; changed }); let status = self.status(); + info!( + enabled_changed, + current_status = ?status.status, + environment_id = ?status.environment_id, + installation_id = %status.installation_id, + server_name = %status.server_name, + "remote control enable requested" + ); if matches!( status.status, RemoteControlConnectionStatus::Connected | RemoteControlConnectionStatus::Connecting @@ -91,12 +103,21 @@ impl RemoteControlHandle { } pub fn disable(&self) -> RemoteControlStatusChangedNotification { - self.enabled_tx.send_if_modified(|state| { + let enabled_changed = self.enabled_tx.send_if_modified(|state| { let changed = *state; *state = false; changed }); + let status = self.status(); + info!( + enabled_changed, + current_status = ?status.status, + environment_id = ?status.environment_id, + installation_id = %status.installation_id, + server_name = %status.server_name, + "remote control disable requested" + ); self.publish_status(RemoteControlConnectionStatus::Disabled) } @@ -112,6 +133,7 @@ impl RemoteControlHandle { &self, connection_status: RemoteControlConnectionStatus, ) -> RemoteControlStatusChangedNotification { + let mut status_change = None; self.status_tx.send_if_modified(|status| { let next_status = remote_control_status_with_connection_status(status, connection_status); @@ -119,9 +141,21 @@ impl RemoteControlHandle { return false; } + status_change = Some((status.clone(), next_status.clone())); *status = next_status; true }); + if let Some((previous_status, next_status)) = status_change { + info!( + previous_status = ?previous_status.status, + next_status = ?next_status.status, + previous_environment_id = ?previous_status.environment_id, + next_environment_id = ?next_status.environment_id, + installation_id = %next_status.installation_id, + server_name = %next_status.server_name, + "remote control handle status changed" + ); + } self.status() } } @@ -165,6 +199,8 @@ pub async fn start_remote_control( let (enabled_tx, enabled_rx) = watch::channel(initial_enabled); let server_name = gethostname().to_string_lossy().trim().to_string(); + let remote_control_url = config.remote_control_url; + let installation_id = config.installation_id; let initial_status = RemoteControlStatusChangedNotification { status: if initial_enabled { RemoteControlConnectionStatus::Connecting @@ -172,16 +208,35 @@ pub async fn start_remote_control( RemoteControlConnectionStatus::Disabled }, server_name: server_name.clone(), - installation_id: config.installation_id.clone(), + installation_id: installation_id.clone(), environment_id: None, }; let (status_tx, _status_rx) = watch::channel(initial_status); let status_publisher = RemoteControlStatusPublisher::new(status_tx.clone()); + info!( + remote_control_url = %remote_control_url, + installation_id = %installation_id, + server_name = %server_name, + state_db_available, + initial_enabled, + "starting app-server remote control websocket task" + ); + let remote_control_url_for_log = remote_control_url.clone(); + let installation_id_for_log = installation_id.clone(); + let server_name_for_log = server_name.clone(); + let shutdown_token_for_log = shutdown_token.clone(); let join_handle = tokio::spawn(async move { - RemoteControlWebsocket::new( + info!( + remote_control_url = %remote_control_url_for_log, + installation_id = %installation_id_for_log, + server_name = %server_name_for_log, + initial_enabled, + "app-server remote control websocket task started" + ); + let websocket_task = RemoteControlWebsocket::new( websocket::RemoteControlWebsocketConfig { - remote_control_url: config.remote_control_url, - installation_id: config.installation_id, + remote_control_url, + installation_id, remote_control_target, server_name, }, @@ -194,8 +249,38 @@ pub async fn start_remote_control( shutdown_token, enabled_rx, ) - .run(app_server_client_name_rx) - .await; + .run(app_server_client_name_rx); + match AssertUnwindSafe(websocket_task).catch_unwind().await { + Ok(()) => { + let shutdown_requested = shutdown_token_for_log.is_cancelled(); + if shutdown_requested { + info!( + remote_control_url = %remote_control_url_for_log, + installation_id = %installation_id_for_log, + server_name = %server_name_for_log, + shutdown_requested, + "app-server remote control websocket task exited" + ); + } else { + warn!( + remote_control_url = %remote_control_url_for_log, + installation_id = %installation_id_for_log, + server_name = %server_name_for_log, + shutdown_requested, + "app-server remote control websocket task exited without shutdown" + ); + } + } + Err(panic) => { + error!( + remote_control_url = %remote_control_url_for_log, + installation_id = %installation_id_for_log, + server_name = %server_name_for_log, + "app-server remote control websocket task panicked" + ); + std::panic::resume_unwind(panic); + } + } }); Ok(( diff --git a/codex-rs/app-server-transport/src/transport/remote_control/websocket.rs b/codex-rs/app-server-transport/src/transport/remote_control/websocket.rs index 1e696c2e46..6358d592e4 100644 --- a/codex-rs/app-server-transport/src/transport/remote_control/websocket.rs +++ b/codex-rs/app-server-transport/src/transport/remote_control/websocket.rs @@ -66,6 +66,10 @@ const REMOTE_CONTROL_ACCOUNT_ID_RETRY_INTERVAL: std::time::Duration = std::time::Duration::from_secs(1); const REMOTE_CONTROL_RECONNECT_BACKOFF_CAP: std::time::Duration = std::time::Duration::from_secs(30); +const REMOTE_CONTROL_WEBSOCKET_CONNECT_TIMEOUT: std::time::Duration = + std::time::Duration::from_secs(30); +const REMOTE_CONTROL_CONNECTION_SHUTDOWN_TIMEOUT: std::time::Duration = + std::time::Duration::from_secs(5); struct BoundedOutboundBuffer { buffer_by_stream: HashMap<(ClientId, StreamId), VecDeque>, @@ -179,14 +183,32 @@ impl WebsocketState { return ClientSegmentObservation::Dropped; } - let observation = self.client_segment_reassembler.observe(client_envelope); - if matches!(observation, ClientSegmentObservation::Forward(_)) - && let Some((key, seq_id)) = client_message_key - { + self.client_segment_reassembler.observe(client_envelope) + } + + fn record_client_message_delivery( + &mut self, + client_envelope: &ClientEnvelope, + client_message_key: Option<((ClientId, Option), u64)>, + ) { + if let Some(cursor) = client_envelope.cursor.as_deref() { + self.subscribe_cursor = Some(cursor.to_string()); + } + if let Some((key, seq_id)) = client_message_key { self.last_completed_client_chunk_seq_id_by_stream .insert(key, seq_id); } - observation + if let ClientEvent::Ack { segment_id } = &client_envelope.event + && let Some(acked_seq_id) = client_envelope.seq_id + && let Some(stream_id) = client_envelope.stream_id.as_ref() + { + self.outbound_buffer.ack( + &client_envelope.client_id, + stream_id, + acked_seq_id, + *segment_id, + ); + } } fn invalidate_client_message_stream(&mut self, client_id: &ClientId, stream_id: &StreamId) { @@ -255,6 +277,14 @@ enum ConnectOutcome { Shutdown, } +#[derive(Debug, Clone, Copy)] +enum ConnectionEndReason { + Shutdown, + Disabled, + EnabledWatchClosed, + ConnectionWorkerStopped, +} + pub(super) struct RemoteControlChannels { pub(super) transport_event_tx: mpsc::Sender, pub(super) status_publisher: RemoteControlStatusPublisher, @@ -270,7 +300,12 @@ impl RemoteControlStatusPublisher { Self { tx } } + fn status(&self) -> RemoteControlStatusChangedNotification { + self.tx.borrow().clone() + } + fn publish_status(&self, connection_status: RemoteControlConnectionStatus) { + let mut status_change = None; self.tx.send_if_modified(|status| { let next_status = remote_control_status_with_connection_status(status, connection_status); @@ -278,12 +313,25 @@ impl RemoteControlStatusPublisher { return false; } + status_change = Some((status.clone(), next_status.clone())); *status = next_status; true }); + if let Some((previous_status, next_status)) = status_change { + info!( + previous_status = ?previous_status.status, + next_status = ?next_status.status, + previous_environment_id = ?previous_status.environment_id, + next_environment_id = ?next_status.environment_id, + installation_id = %next_status.installation_id, + server_name = %next_status.server_name, + "remote control websocket status changed" + ); + } } fn publish_environment_id(&self, environment_id: Option) { + let mut status_change = None; self.tx.send_if_modified(|status| { if status.status == RemoteControlConnectionStatus::Disabled { return false; @@ -298,9 +346,20 @@ impl RemoteControlStatusPublisher { return false; } + status_change = Some((status.clone(), next_status.clone())); *status = next_status; true }); + if let Some((previous_status, next_status)) = status_change { + info!( + status = ?next_status.status, + previous_environment_id = ?previous_status.environment_id, + next_environment_id = ?next_status.environment_id, + installation_id = %next_status.installation_id, + server_name = %next_status.server_name, + "remote control websocket environment changed" + ); + } } } @@ -367,12 +426,26 @@ impl RemoteControlWebsocket { mut self, app_server_client_name_rx: Option>, ) { + info!( + remote_control_url = %self.remote_control_url, + installation_id = %self.installation_id, + server_name = %self.server_name, + initial_enabled = *self.enabled_rx.borrow(), + "app-server remote control websocket loop started" + ); let app_server_client_name = match self .wait_for_app_server_client_name(app_server_client_name_rx) .await { Ok(app_server_client_name) => app_server_client_name, Err(_) => { + warn!( + remote_control_url = %self.remote_control_url, + installation_id = %self.installation_id, + server_name = %self.server_name, + shutdown_requested = self.shutdown_token.is_cancelled(), + "app-server remote control websocket loop stopped before client name was ready" + ); self.client_tracker.lock().await.shutdown().await; return; } @@ -380,9 +453,27 @@ impl RemoteControlWebsocket { loop { if !self.wait_until_enabled().await { + info!( + remote_control_url = %self.remote_control_url, + installation_id = %self.installation_id, + server_name = %self.server_name, + shutdown_requested = self.shutdown_token.is_cancelled(), + current_status = ?self.status_publisher.status().status, + "app-server remote control websocket loop exiting while waiting for enablement" + ); break; } + let status = self.status_publisher.status(); + info!( + remote_control_url = %self.remote_control_url, + installation_id = %self.installation_id, + server_name = %self.server_name, + reconnect_attempt = self.reconnect_attempt.saturating_add(1), + current_status = ?status.status, + environment_id = ?status.environment_id, + "starting app-server remote control websocket connection cycle" + ); let shutdown_token = self.shutdown_token.child_token(); let websocket_connection = match self .connect(&shutdown_token, app_server_client_name.as_deref()) @@ -397,11 +488,30 @@ impl RemoteControlWebsocket { ConnectOutcome::Shutdown => break, }; - self.run_connection(websocket_connection, shutdown_token) + let connection_end_reason = self + .run_connection(websocket_connection, shutdown_token) .await; + let status = self.status_publisher.status(); + info!( + remote_control_url = %self.remote_control_url, + installation_id = %self.installation_id, + server_name = %self.server_name, + connection_end_reason = ?connection_end_reason, + current_status = ?status.status, + environment_id = ?status.environment_id, + enabled = *self.enabled_rx.borrow(), + "app-server remote control websocket connection cycle ended" + ); } self.client_tracker.lock().await.shutdown().await; + info!( + remote_control_url = %self.remote_control_url, + installation_id = %self.installation_id, + server_name = %self.server_name, + shutdown_requested = self.shutdown_token.is_cancelled(), + "app-server remote control websocket loop exited" + ); } async fn wait_for_app_server_client_name( @@ -462,6 +572,19 @@ impl RemoteControlWebsocket { loop { let subscribe_cursor = self.state.lock().await.subscribe_cursor.clone(); + let enrollment = self.enrollment.as_ref(); + info!( + websocket_url = %remote_control_target.websocket_url, + installation_id = %self.installation_id, + server_name = %self.server_name, + reconnect_attempt = self.reconnect_attempt.saturating_add(1), + has_enrollment = enrollment.is_some(), + server_id = ?enrollment.map(|enrollment| enrollment.server_id.as_str()), + environment_id = ?enrollment.map(|enrollment| enrollment.environment_id.as_str()), + subscribe_cursor_present = subscribe_cursor.is_some(), + app_server_client_name = ?app_server_client_name, + "connecting to app-server remote control websocket" + ); let connect_options = RemoteControlConnectOptions { installation_id: &self.installation_id, server_name: &self.server_name, @@ -500,10 +623,16 @@ impl RemoteControlWebsocket { self.auth_recovery = self.auth_manager.unauthorized_recovery(); self.status_publisher .publish_status(RemoteControlConnectionStatus::Connected); + let enrollment = self.enrollment.as_ref(); info!( - "connected to app-server remote control websocket: {}, {}", - remote_control_target.websocket_url, - format_headers(response.headers()) + websocket_url = %remote_control_target.websocket_url, + installation_id = %self.installation_id, + server_name = %self.server_name, + server_id = ?enrollment.map(|enrollment| enrollment.server_id.as_str()), + environment_id = ?enrollment.map(|enrollment| enrollment.environment_id.as_str()), + subscribe_cursor_present = subscribe_cursor.is_some(), + response_headers = %format_headers(response.headers()), + "connected to app-server remote control websocket" ); return ConnectOutcome::Connected(Box::new(websocket_connection)); } @@ -519,12 +648,20 @@ impl RemoteControlWebsocket { let reconnect_attempt = self.reconnect_attempt.saturating_add(1); let (reconnect_delay, reconnect_backoff_reset) = next_reconnect_delay(&mut self.reconnect_attempt); + let enrollment = self.enrollment.as_ref(); warn!( websocket_url = %remote_control_target.websocket_url, + installation_id = %self.installation_id, + server_name = %self.server_name, error = %err, + error_kind = ?err.kind(), reconnect_attempt, reconnect_delay = ?reconnect_delay, reconnect_backoff_reset, + has_enrollment = enrollment.is_some(), + server_id = ?enrollment.map(|enrollment| enrollment.server_id.as_str()), + environment_id = ?enrollment.map(|enrollment| enrollment.environment_id.as_str()), + subscribe_cursor_present = subscribe_cursor.is_some(), "failed to connect to app-server remote control websocket" ); if reconnect_backoff_reset { @@ -562,7 +699,7 @@ impl RemoteControlWebsocket { &self, websocket_connection: WebSocketStream>, shutdown_token: CancellationToken, - ) { + ) -> ConnectionEndReason { let (websocket_writer, websocket_reader) = websocket_connection.split(); let mut join_set = tokio::task::JoinSet::new(); @@ -583,19 +720,48 @@ impl RemoteControlWebsocket { )); let mut enabled_rx = self.enabled_rx.clone(); - tokio::select! { - _ = shutdown_token.cancelled() => {} + let connection_end_reason = tokio::select! { + _ = shutdown_token.cancelled() => ConnectionEndReason::Shutdown, changed = enabled_rx.wait_for(|enabled| !*enabled) => { if changed.is_ok() { self.status_publisher .publish_status(RemoteControlConnectionStatus::Disabled); + ConnectionEndReason::Disabled + } else { + ConnectionEndReason::EnabledWatchClosed } } - _ = join_set.join_next() => {} + _ = join_set.join_next() => ConnectionEndReason::ConnectionWorkerStopped, }; shutdown_token.cancel(); - join_set.join_all().await; + Self::join_connection_workers(&mut join_set, REMOTE_CONTROL_CONNECTION_SHUTDOWN_TIMEOUT) + .await; + connection_end_reason + } + + async fn join_connection_workers( + join_set: &mut tokio::task::JoinSet<()>, + shutdown_timeout: std::time::Duration, + ) { + if tokio::time::timeout(shutdown_timeout, Self::drain_join_set(join_set)) + .await + .is_ok() + { + return; + } + + warn!( + shutdown_timeout = ?shutdown_timeout, + remaining_workers = join_set.len(), + "timed out waiting for remote control connection workers to stop; aborting" + ); + join_set.abort_all(); + Self::drain_join_set(join_set).await; + } + + async fn drain_join_set(join_set: &mut tokio::task::JoinSet<()>) { + while join_set.join_next().await.is_some() {} } async fn run_server_writer( @@ -675,12 +841,14 @@ impl RemoteControlWebsocket { let queued_server_envelope = tokio::select! { _ = shutdown_token.cancelled() => return Ok(()), _ = ping_interval.tick() => { - if let Err(err) = websocket_writer - .send(tungstenite::Message::Ping(Vec::new().into())) - .await - { - return Err(io::Error::other(err)); - } + tokio::select! { + _ = shutdown_token.cancelled() => return Ok(()), + send_result = websocket_writer.send(tungstenite::Message::Ping(Vec::new().into())) => { + if let Err(err) = send_result { + return Err(io::Error::other(err)); + } + } + }; continue; } wait_result = used_rx.changed(), if !outbound_has_capacity => @@ -886,6 +1054,7 @@ impl RemoteControlWebsocket { } }; + let client_message_key = WebsocketState::client_message_key(&client_envelope); let observation = { let mut websocket_state = state.lock().await; websocket_state.observe_client_message(client_envelope, wire_size_bytes) @@ -895,24 +1064,6 @@ impl RemoteControlWebsocket { ClientSegmentObservation::Pending | ClientSegmentObservation::Dropped => continue, }; - { - let mut websocket_state = state.lock().await; - if let Some(cursor) = client_envelope.cursor.as_deref() { - websocket_state.subscribe_cursor = Some(cursor.to_string()); - } - if let ClientEvent::Ack { segment_id } = &client_envelope.event - && let Some(acked_seq_id) = client_envelope.seq_id - && let Some(stream_id) = client_envelope.stream_id.as_ref() - { - websocket_state.outbound_buffer.ack( - &client_envelope.client_id, - stream_id, - acked_seq_id, - *segment_id, - ); - } - } - let closed_client = matches!(&client_envelope.event, ClientEvent::ClientClosed).then(|| { ( @@ -920,6 +1071,7 @@ impl RemoteControlWebsocket { client_envelope.stream_id.clone(), ) }); + let delivered_client_envelope = client_envelope.clone(); if client_tracker .handle_message(client_envelope) .await @@ -927,6 +1079,10 @@ impl RemoteControlWebsocket { { return Ok(()); } + state + .lock() + .await + .record_client_message_delivery(&delivered_client_envelope, client_message_key); if let Some((client_id, stream_id)) = closed_client { let mut websocket_state = state.lock().await; if let Some(stream_id) = stream_id { @@ -1190,7 +1346,22 @@ pub(super) async fn connect_remote_control_websocket( connect_options.subscribe_cursor, )?; - match connect_async(request).await { + let websocket_connect_result = tokio::time::timeout( + REMOTE_CONTROL_WEBSOCKET_CONNECT_TIMEOUT, + connect_async(request), + ) + .await + .map_err(|_| { + io::Error::new( + ErrorKind::TimedOut, + format!( + "timed out connecting to remote control websocket at `{}` after {:?}", + remote_control_target.websocket_url, REMOTE_CONTROL_WEBSOCKET_CONNECT_TIMEOUT + ), + ) + })?; + + match websocket_connect_result { Ok((websocket_stream, response)) => Ok((websocket_stream, response.map(|_| ()))), Err(err) => { match &err { @@ -2013,6 +2184,17 @@ mod tests { .expect("writer should stop cleanly"); } + #[tokio::test] + async fn join_connection_workers_aborts_stuck_worker_after_timeout() { + let mut join_set = tokio::task::JoinSet::new(); + join_set.spawn(futures::future::pending::<()>()); + + RemoteControlWebsocket::join_connection_workers(&mut join_set, Duration::from_millis(10)) + .await; + + assert!(join_set.is_empty()); + } + #[tokio::test] async fn run_server_writer_inner_assigns_contiguous_seq_ids_per_stream() { let (client_stream, mut server_stream) = connected_websocket_pair().await; @@ -2350,6 +2532,65 @@ mod tests { &raw[split..], ); + assert!(matches!( + observe_client_message(&mut state, first_chunk.clone()), + ClientSegmentObservation::Pending + )); + let completed_envelope = match observe_client_message(&mut state, second_chunk) { + ClientSegmentObservation::Forward(client_envelope) => *client_envelope, + _ => panic!("expected completed client message"), + }; + state.record_client_message_delivery( + &completed_envelope, + Some(( + ( + ClientId("client-1".to_string()), + Some(StreamId("stream-1".to_string())), + ), + 4, + )), + ); + assert!(matches!( + observe_client_message(&mut state, first_chunk), + ClientSegmentObservation::Dropped + )); + } + + #[test] + fn websocket_state_allows_replay_before_completed_chunk_delivery() { + let (outbound_buffer, _used_rx) = BoundedOutboundBuffer::new(); + let mut state = WebsocketState { + outbound_buffer, + subscribe_cursor: None, + next_seq_id_by_stream: HashMap::new(), + last_completed_client_chunk_seq_id_by_stream: HashMap::new(), + client_segment_reassembler: ClientSegmentReassembler::default(), + }; + let message = JSONRPCMessage::Notification(JSONRPCNotification { + method: "initialized".to_string(), + params: None, + }); + let raw = serde_json::to_vec(&message).expect("message should serialize"); + let split = raw.len() / 2; + let first_chunk = client_chunk_envelope( + "client-1", + "stream-1", + /*seq_id*/ 4, + /*segment_id*/ 0, + /*segment_count*/ 2, + raw.len(), + &raw[..split], + ); + let second_chunk = client_chunk_envelope( + "client-1", + "stream-1", + /*seq_id*/ 4, + /*segment_id*/ 1, + /*segment_count*/ 2, + raw.len(), + &raw[split..], + ); + assert!(matches!( observe_client_message(&mut state, first_chunk.clone()), ClientSegmentObservation::Pending @@ -2360,7 +2601,7 @@ mod tests { )); assert!(matches!( observe_client_message(&mut state, first_chunk), - ClientSegmentObservation::Dropped + ClientSegmentObservation::Pending )); } From 5cd9b8086a0b1d46121019506634abb6910a7617 Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Tue, 26 May 2026 13:50:44 -0700 Subject: [PATCH 11/16] Respect resume cwd overrides for idle cached threads (#24528) Fixes #24186. ## Why When the TUI resumes a thread through the local app-server daemon with a selected workspace, `thread/resume` can hit an already-loaded but idle cached thread. That path previously rejoined the cached `CodexThread`, so cwd/config overrides in `ThreadResumeParams` were ignored and the resumed session kept using the old cwd. ## What changed App-server now treats a loaded-but-idle thread with no subscribers as a cache entry when resume overrides differ: it unloads that cached thread and lets the normal resume path rebuild it with the requested cwd/config. Threads that still have subscribers, or active runtime work, continue to rejoin the existing loaded thread so in-flight state remains observable. The existing thread teardown helper was generalized from archive-specific cleanup to shared unload cleanup for this path. --- .../request_processors/thread_processor.rs | 70 ++++++++++++++----- .../tests/suite/v2/thread_resume.rs | 35 +++++++++- .../tests/suite/v2/thread_unsubscribe.rs | 1 + 3 files changed, 87 insertions(+), 19 deletions(-) diff --git a/codex-rs/app-server/src/request_processors/thread_processor.rs b/codex-rs/app-server/src/request_processors/thread_processor.rs index c9a42d8359..5a2e1a4de8 100644 --- a/codex-rs/app-server/src/request_processors/thread_processor.rs +++ b/codex-rs/app-server/src/request_processors/thread_processor.rs @@ -2787,15 +2787,20 @@ impl ThreadRequestProcessor { /*include_history*/ true, ) .await?; - if source_thread.thread_id != existing_thread_id { - return Err(invalid_request(format!( - "cannot resume running thread {existing_thread_id} from source thread {}", - source_thread.thread_id - ))); - } Some((existing_thread_id, existing_thread, source_thread)) } else { - None + let source_thread = self + .read_stored_thread_for_resume( + ¶ms.thread_id, + params.path.as_ref(), + /*include_history*/ true, + ) + .await?; + let existing_thread_id = source_thread.thread_id; + match self.thread_manager.get_thread(existing_thread_id).await { + Ok(existing_thread) => Some((existing_thread_id, existing_thread, source_thread)), + Err(_) => None, + } }; if let Some((existing_thread_id, existing_thread, source_thread)) = running_thread { @@ -2812,6 +2817,48 @@ impl ThreadRequestProcessor { active_path.display() ))); } + let config_snapshot = existing_thread.config_snapshot().await; + let mismatch_details = collect_resume_override_mismatches(params, &config_snapshot); + if !mismatch_details.is_empty() { + let has_subscribers = !self + .thread_state_manager + .subscribed_connection_ids(existing_thread_id) + .await + .is_empty(); + let loaded_status = self + .thread_watch_manager + .loaded_status_for_thread(&existing_thread_id.to_string()) + .await; + let is_running = + matches!(existing_thread.agent_status().await, AgentStatus::Running); + + if !has_subscribers && matches!(loaded_status, ThreadStatus::Idle) && !is_running { + // A loaded idle thread is only a cache entry. Shut it down + // before removing it so cold resume cannot duplicate a + // thread that timed out during shutdown. + match wait_for_thread_shutdown(&existing_thread).await { + ThreadShutdownResult::Complete => { + self.thread_manager.remove_thread(&existing_thread_id).await; + self.finalize_thread_teardown(existing_thread_id).await; + return Ok(false); + } + ThreadShutdownResult::SubmitFailed => { + warn!("failed to submit Shutdown to thread {existing_thread_id}"); + } + ThreadShutdownResult::TimedOut => { + warn!("thread {existing_thread_id} shutdown timed out"); + } + } + } + + // Preserve rejoin semantics when another client can still observe + // the loaded thread or shutdown did not complete. + tracing::warn!( + "thread/resume overrides ignored for loaded thread {}: {}", + existing_thread_id, + mismatch_details.join("; ") + ); + } let redact_resume_payloads = should_redact_thread_resume_payloads(app_server_client_name.as_deref()); let history_items = source_thread @@ -2841,15 +2888,6 @@ impl ThreadRequestProcessor { ) .await?; - let config_snapshot = existing_thread.config_snapshot().await; - let mismatch_details = collect_resume_override_mismatches(params, &config_snapshot); - if !mismatch_details.is_empty() { - tracing::warn!( - "thread/resume overrides ignored for running thread {}: {}", - existing_thread_id, - mismatch_details.join("; ") - ); - } let mut summary_source_thread = source_thread; summary_source_thread.history = None; let mut thread_summary = self.stored_thread_to_api_thread( diff --git a/codex-rs/app-server/tests/suite/v2/thread_resume.rs b/codex-rs/app-server/tests/suite/v2/thread_resume.rs index 6416499663..8e60c90a01 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_resume.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_resume.rs @@ -43,6 +43,7 @@ use codex_app_server_protocol::ThreadSource; use codex_app_server_protocol::ThreadStartParams; use codex_app_server_protocol::ThreadStartResponse; use codex_app_server_protocol::ThreadStatus; +use codex_app_server_protocol::ThreadUnsubscribeParams; use codex_app_server_protocol::TurnItemsView; use codex_app_server_protocol::TurnStartParams; use codex_app_server_protocol::TurnStartResponse; @@ -1929,7 +1930,7 @@ async fn thread_resume_and_read_interrupt_incomplete_rollout_turn_when_thread_is } #[tokio::test] -async fn thread_resume_without_overrides_does_not_change_updated_at_or_mtime() -> Result<()> { +async fn thread_resume_defers_updated_at_until_turn_start() -> Result<()> { let server = create_mock_responses_server_repeating_assistant("Done").await; let codex_home = TempDir::new()?; let rollout = setup_rollout_fixture(codex_home.path(), &server.uri())?; @@ -1973,6 +1974,33 @@ async fn thread_resume_without_overrides_does_not_change_updated_at_or_mtime() - let after_modified = std::fs::metadata(&rollout.rollout_file_path)?.modified()?; assert_eq!(after_modified, rollout.before_modified); + let unsubscribe_id = mcp + .send_thread_unsubscribe_request(ThreadUnsubscribeParams { + thread_id: thread_id.clone(), + }) + .await?; + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(unsubscribe_id)), + ) + .await??; + + let resume_id = mcp + .send_thread_resume_request(ThreadResumeParams { + thread_id: "not-a-valid-thread-id".to_string(), + path: Some(normalized_existing_path(&rollout.rollout_file_path)?), + cwd: Some(codex_home.path().to_string_lossy().to_string()), + ..Default::default() + }) + .await?; + let resume_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(resume_id)), + ) + .await??; + let ThreadResumeResponse { cwd, .. } = to_response::(resume_resp)?; + assert_eq!(cwd, AbsolutePathBuf::from_absolute_path(codex_home.path())?); + let turn_id = mcp .send_turn_start_request(TurnStartParams { thread_id, @@ -2291,9 +2319,10 @@ async fn thread_resume_rejects_mismatched_path_for_running_thread_id() -> Result ) .await??; - let stale_path = rollout_path(codex_home.path(), "2025-01-01T00-00-00", &thread_id); + let stale_thread_id = Uuid::new_v4().to_string(); + let stale_path = rollout_path(codex_home.path(), "2025-01-01T00-00-00", &stale_thread_id); std::fs::create_dir_all(stale_path.parent().expect("stale path parent"))?; - let thread_uuid = Uuid::parse_str(&thread_id)?; + let thread_uuid = Uuid::parse_str(&stale_thread_id)?; let mut stale_file = std::fs::File::create(&stale_path)?; let stale_meta = json!({ "timestamp": "2025-01-01T00:00:00Z", diff --git a/codex-rs/app-server/tests/suite/v2/thread_unsubscribe.rs b/codex-rs/app-server/tests/suite/v2/thread_unsubscribe.rs index c0188add8c..45437b7ee1 100644 --- a/codex-rs/app-server/tests/suite/v2/thread_unsubscribe.rs +++ b/codex-rs/app-server/tests/suite/v2/thread_unsubscribe.rs @@ -317,6 +317,7 @@ async fn thread_unsubscribe_preserves_cached_status_before_idle_unload() -> Resu let resume_id = mcp .send_thread_resume_request(ThreadResumeParams { thread_id, + cwd: Some(codex_home.path().to_string_lossy().to_string()), ..Default::default() }) .await?; From 1911021c0eae972e622c8445d136ae04545a530f Mon Sep 17 00:00:00 2001 From: Owen Lin Date: Tue, 26 May 2026 14:05:28 -0700 Subject: [PATCH 12/16] Add forked_from_thread_id turn metadata (#24160) ## Why When Codex calls responsesapi, we currently send `session_id`, `thread_id`, and `turn_id` among other things as `client_metadata["x-codex-turn-metadata"]`. This PR adds `forked_from_thread_id` which helps explain the "lineage" of a forked thread. ## What's changed - Track the immediate history source copied into a forked thread through thread/session creation, including subagent and review turn metadata paths. - Include `forked_from_thread_id` in Codex turn metadata while preventing turn-scoped Responses API client metadata from overwriting Codex-owned lineage fields. - Add coverage for fork lineage in turn metadata and the app-server Responses API request path. --- .../tests/suite/v2/client_metadata.rs | 204 ++++++++++++++++++ codex-rs/core/src/agent/control.rs | 3 + codex-rs/core/src/codex_delegate.rs | 1 + codex-rs/core/src/session/mod.rs | 3 + codex-rs/core/src/session/review.rs | 5 + codex-rs/core/src/session/session.rs | 7 +- codex-rs/core/src/session/tests.rs | 8 + .../core/src/session/tests/guardian_tests.rs | 1 + codex-rs/core/src/session/turn_context.rs | 1 + codex-rs/core/src/thread_manager.rs | 34 ++- codex-rs/core/src/turn_metadata.rs | 101 ++++----- codex-rs/core/src/turn_metadata_tests.rs | 63 +++++- .../suite/responses_api_proxy_headers.rs | 9 + codex-rs/core/tests/suite/review.rs | 33 ++- 14 files changed, 404 insertions(+), 69 deletions(-) diff --git a/codex-rs/app-server/tests/suite/v2/client_metadata.rs b/codex-rs/app-server/tests/suite/v2/client_metadata.rs index 80f41d7b11..3a00a433d1 100644 --- a/codex-rs/app-server/tests/suite/v2/client_metadata.rs +++ b/codex-rs/app-server/tests/suite/v2/client_metadata.rs @@ -1,8 +1,16 @@ use anyhow::Result; use app_test_support::McpProcess; +use app_test_support::create_fake_rollout; use app_test_support::to_response; use codex_app_server_protocol::JSONRPCResponse; use codex_app_server_protocol::RequestId; +use codex_app_server_protocol::ReviewDelivery; +use codex_app_server_protocol::ReviewStartParams; +use codex_app_server_protocol::ReviewStartResponse; +use codex_app_server_protocol::ReviewTarget; +use codex_app_server_protocol::ThreadForkParams; +use codex_app_server_protocol::ThreadForkResponse; +use codex_app_server_protocol::ThreadSource; use codex_app_server_protocol::ThreadStartParams; use codex_app_server_protocol::ThreadStartResponse; use codex_app_server_protocol::TurnStartParams; @@ -60,6 +68,7 @@ async fn turn_start_forwards_client_metadata_to_responses_request_v2() -> Result let client_metadata = HashMap::from([ ("fiber_run_id".to_string(), "fiber-start-123".to_string()), ("origin".to_string(), "gaas".to_string()), + ("thread_source".to_string(), "client-supplied".to_string()), ]); let turn_req = mcp .send_turn_start_request(TurnStartParams { @@ -93,12 +102,188 @@ async fn turn_start_forwards_client_metadata_to_responses_request_v2() -> Result .unwrap_or_else(|| panic!("missing x-codex-turn-metadata header")); assert_eq!(metadata["fiber_run_id"].as_str(), Some("fiber-start-123")); assert_eq!(metadata["origin"].as_str(), Some("gaas")); + assert_eq!(metadata["thread_source"].as_str(), Some("client-supplied")); assert_eq!(metadata["turn_id"].as_str(), Some(turn.id.as_str())); assert!(metadata.get("session_id").is_some()); Ok(()) } +#[tokio::test] +async fn turn_start_sends_fork_lineage_in_turn_metadata_for_thread_fork_v2() -> Result<()> { + skip_if_no_network!(Ok(())); + + let server = responses::start_mock_server().await; + let response_mock = responses::mount_sse_once( + &server, + responses::sse(vec![ + responses::ev_response_created("resp-1"), + responses::ev_assistant_message("msg-1", "Done"), + responses::ev_completed("resp-1"), + ]), + ) + .await; + + let codex_home = TempDir::new()?; + create_config_toml( + codex_home.path(), + &server.uri(), + /*supports_websockets*/ false, + )?; + + let source_thread_id = create_fake_rollout( + codex_home.path(), + "2025-01-05T12-00-00", + "2025-01-05T12:00:00Z", + "Saved user message", + Some("mock_provider"), + /*git_info*/ None, + )?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let ThreadForkResponse { thread, .. } = + fork_fake_rollout_thread(&mut mcp, source_thread_id.clone()).await?; + + let turn_req = mcp + .send_turn_start_request(TurnStartParams { + thread_id: thread.id.clone(), + input: vec![V2UserInput::Text { + text: "Continue".to_string(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + let turn_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(turn_req)), + ) + .await??; + let TurnStartResponse { turn } = to_response::(turn_resp)?; + + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("turn/completed"), + ) + .await??; + + let request = response_mock.single_request(); + let metadata = request + .header("x-codex-turn-metadata") + .as_deref() + .map(parse_json_header) + .unwrap_or_else(|| panic!("missing x-codex-turn-metadata header")); + assert_eq!( + metadata["forked_from_thread_id"].as_str(), + Some(source_thread_id.as_str()) + ); + assert_eq!(metadata["thread_id"].as_str(), Some(thread.id.as_str())); + assert_eq!(metadata["turn_id"].as_str(), Some(turn.id.as_str())); + + Ok(()) +} + +#[tokio::test] +async fn review_start_sends_fork_lineage_in_turn_metadata_for_thread_fork_v2() -> Result<()> { + skip_if_no_network!(Ok(())); + + let review_payload = serde_json::json!({ + "findings": [], + "overall_correctness": "good", + "overall_explanation": "Done", + "overall_confidence_score": 0.5 + }) + .to_string(); + let server = responses::start_mock_server().await; + let response_mock = responses::mount_sse_once( + &server, + responses::sse(vec![ + responses::ev_response_created("resp-1"), + responses::ev_assistant_message("msg-1", &review_payload), + responses::ev_completed("resp-1"), + ]), + ) + .await; + + let codex_home = TempDir::new()?; + create_config_toml( + codex_home.path(), + &server.uri(), + /*supports_websockets*/ false, + )?; + + let source_thread_id = create_fake_rollout( + codex_home.path(), + "2025-01-05T12-00-00", + "2025-01-05T12:00:00Z", + "Saved user message", + Some("mock_provider"), + /*git_info*/ None, + )?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let ThreadForkResponse { thread, .. } = + fork_fake_rollout_thread(&mut mcp, source_thread_id.clone()).await?; + + let review_req = mcp + .send_review_start_request(ReviewStartParams { + thread_id: thread.id.clone(), + delivery: Some(ReviewDelivery::Inline), + target: ReviewTarget::Custom { + instructions: "Review the fork".to_string(), + }, + }) + .await?; + let review_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(review_req)), + ) + .await??; + let ReviewStartResponse { + review_thread_id, .. + } = to_response::(review_resp)?; + assert_eq!(review_thread_id, thread.id); + + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("turn/completed"), + ) + .await??; + + let request = response_mock.single_request(); + let metadata = request + .header("x-codex-turn-metadata") + .as_deref() + .map(parse_json_header) + .unwrap_or_else(|| panic!("missing x-codex-turn-metadata header")); + assert_eq!( + request.header("x-openai-subagent").as_deref(), + Some("review") + ); + assert_eq!( + metadata["forked_from_thread_id"].as_str(), + Some(review_thread_id.as_str()) + ); + let review_request_thread_id = metadata["thread_id"] + .as_str() + .unwrap_or_else(|| panic!("missing review request thread_id")); + assert!(review_request_thread_id != review_thread_id.as_str()); + assert_eq!( + request + .header("x-codex-window-id") + .as_deref() + .and_then(|window_id| window_id.split_once(':').map(|(thread_id, _)| thread_id)), + Some(review_request_thread_id) + ); + assert!(metadata["turn_id"].as_str().is_some()); + + Ok(()) +} + #[tokio::test] async fn turn_steer_updates_client_metadata_on_follow_up_responses_request_v2() -> Result<()> { skip_if_no_network!(Ok(())); @@ -345,6 +530,25 @@ supports_websockets = {supports_websockets} ) } +async fn fork_fake_rollout_thread( + mcp: &mut McpProcess, + source_thread_id: String, +) -> Result { + let fork_req = mcp + .send_thread_fork_request(ThreadForkParams { + thread_id: source_thread_id, + thread_source: Some(ThreadSource::User), + ..Default::default() + }) + .await?; + let fork_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(fork_req)), + ) + .await??; + to_response::(fork_resp) +} + fn parse_json_header(value: &str) -> serde_json::Value { match serde_json::from_str(value) { Ok(value) => value, diff --git a/codex-rs/core/src/agent/control.rs b/codex-rs/core/src/agent/control.rs index a9a90be54a..4706a455ff 100644 --- a/codex-rs/core/src/agent/control.rs +++ b/codex-rs/core/src/agent/control.rs @@ -262,10 +262,12 @@ impl AgentControl { .await? } (Some(session_source), None) => { + let forked_from_thread_id = thread_spawn_parent_thread_id(&session_source); Box::pin(state.spawn_new_thread_with_source( config.clone(), self.clone(), session_source, + forked_from_thread_id, /*thread_source*/ Some(ThreadSource::Subagent), /*persist_extended_history*/ false, /*metrics_service_name*/ None, @@ -487,6 +489,7 @@ impl AgentControl { self.clone(), session_source, /*thread_source*/ Some(ThreadSource::Subagent), + /*forked_from_thread_id*/ Some(parent_thread_id), /*persist_extended_history*/ false, inherited_shell_snapshot, inherited_exec_policy, diff --git a/codex-rs/core/src/codex_delegate.rs b/codex-rs/core/src/codex_delegate.rs index b27cf745c9..dd88e9238f 100644 --- a/codex-rs/core/src/codex_delegate.rs +++ b/codex-rs/core/src/codex_delegate.rs @@ -86,6 +86,7 @@ pub(crate) async fn run_codex_thread_interactive( extensions: Arc::clone(&parent_session.services.extensions), conversation_history: initial_history.unwrap_or(InitialHistory::New), session_source: SessionSource::SubAgent(subagent_source.clone()), + forked_from_thread_id: Some(parent_session.conversation_id), thread_source: Some(ThreadSource::Subagent), agent_control: parent_session.services.agent_control.clone(), dynamic_tools: Vec::new(), diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs index 1876cbf980..5ea30e5bb5 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -400,6 +400,7 @@ pub(crate) struct CodexSpawnArgs { pub(crate) extensions: Arc>, pub(crate) conversation_history: InitialHistory, pub(crate) session_source: SessionSource, + pub(crate) forked_from_thread_id: Option, pub(crate) thread_source: Option, pub(crate) agent_control: AgentControl, pub(crate) dynamic_tools: Vec, @@ -464,6 +465,7 @@ impl Codex { extensions, conversation_history, session_source, + forked_from_thread_id, thread_source, agent_control, dynamic_tools, @@ -619,6 +621,7 @@ impl Codex { app_server_client_name: None, app_server_client_version: None, session_source, + forked_from_thread_id, thread_source, dynamic_tools, persist_extended_history, diff --git a/codex-rs/core/src/session/review.rs b/codex-rs/core/src/session/review.rs index 32122873cb..762ebc1458 100644 --- a/codex-rs/core/src/session/review.rs +++ b/codex-rs/core/src/session/review.rs @@ -69,12 +69,17 @@ pub(super) async fn spawn_review_thread( .model_reasoning_summary .unwrap_or(model_info.default_reasoning_summary); let session_source = parent_turn_context.session_source.clone(); + let forked_from_thread_id = { + let state = sess.state.lock().await; + state.session_configuration.forked_from_thread_id + }; let per_turn_config = Arc::new(per_turn_config); let review_turn_id = sub_id.to_string(); let turn_metadata_state = Arc::new(TurnMetadataState::new( sess.session_id().to_string(), sess.thread_id().to_string(), + forked_from_thread_id, parent_turn_context.thread_source, review_turn_id.clone(), #[allow(deprecated)] diff --git a/codex-rs/core/src/session/session.rs b/codex-rs/core/src/session/session.rs index 75eb7d0653..0823e93666 100644 --- a/codex-rs/core/src/session/session.rs +++ b/codex-rs/core/src/session/session.rs @@ -95,6 +95,8 @@ pub(crate) struct SessionConfiguration { pub(super) app_server_client_version: Option, /// Source of the session (cli, vscode, exec, mcp, ...) pub(super) session_source: SessionSource, + /// Immediate history source copied into this thread, when this thread was forked. + pub(super) forked_from_thread_id: Option, /// Optional analytics source classification for this thread. pub(super) thread_source: Option, pub(super) dynamic_tools: Vec, @@ -505,7 +507,10 @@ impl Session { session_configuration.collaboration_mode.model(), session_configuration.provider ); - let forked_from_id = initial_history.forked_from_id(); + let forked_from_id = session_configuration + .forked_from_thread_id + .or_else(|| initial_history.forked_from_id()); + session_configuration.forked_from_thread_id = forked_from_id; let event_persistence_mode = if session_configuration.persist_extended_history { ThreadEventPersistenceMode::Extended diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs index 0638168422..5cb8f438ab 100644 --- a/codex-rs/core/src/session/tests.rs +++ b/codex-rs/core/src/session/tests.rs @@ -3001,6 +3001,7 @@ async fn set_rate_limits_retains_previous_credits() { app_server_client_name: None, app_server_client_version: None, session_source: SessionSource::Exec, + forked_from_thread_id: None, thread_source: None, dynamic_tools: Vec::new(), persist_extended_history: false, @@ -3105,6 +3106,7 @@ async fn set_rate_limits_updates_plan_type_when_present() { app_server_client_name: None, app_server_client_version: None, session_source: SessionSource::Exec, + forked_from_thread_id: None, thread_source: None, dynamic_tools: Vec::new(), persist_extended_history: false, @@ -3632,6 +3634,7 @@ pub(crate) async fn make_session_configuration_for_tests() -> SessionConfigurati app_server_client_name: None, app_server_client_version: None, session_source: SessionSource::Exec, + forked_from_thread_id: None, thread_source: None, dynamic_tools: Vec::new(), persist_extended_history: false, @@ -4375,6 +4378,7 @@ async fn session_new_fails_when_zsh_fork_enabled_without_packaged_zsh() { app_server_client_name: None, app_server_client_version: None, session_source: SessionSource::Exec, + forked_from_thread_id: None, thread_source: None, dynamic_tools: Vec::new(), persist_extended_history: false, @@ -4484,6 +4488,7 @@ pub(crate) async fn make_session_and_context() -> (Session, TurnContext) { app_server_client_name: None, app_server_client_version: None, session_source: SessionSource::Exec, + forked_from_thread_id: None, thread_source: None, dynamic_tools: Vec::new(), persist_extended_history: false, @@ -4718,6 +4723,7 @@ async fn make_session_with_config_and_rx( app_server_client_name: None, app_server_client_version: None, session_source: SessionSource::Exec, + forked_from_thread_id: None, thread_source: None, dynamic_tools: Vec::new(), persist_extended_history: false, @@ -4821,6 +4827,7 @@ async fn make_session_with_history_source_and_agent_control_and_rx( app_server_client_name: None, app_server_client_version: None, session_source: session_source.clone(), + forked_from_thread_id: None, thread_source: None, dynamic_tools: Vec::new(), persist_extended_history: false, @@ -6316,6 +6323,7 @@ where app_server_client_name: None, app_server_client_version: None, session_source: SessionSource::Exec, + forked_from_thread_id: None, thread_source: None, dynamic_tools, persist_extended_history: false, diff --git a/codex-rs/core/src/session/tests/guardian_tests.rs b/codex-rs/core/src/session/tests/guardian_tests.rs index 5db4aab985..dabf9c360e 100644 --- a/codex-rs/core/src/session/tests/guardian_tests.rs +++ b/codex-rs/core/src/session/tests/guardian_tests.rs @@ -692,6 +692,7 @@ async fn guardian_subagent_does_not_inherit_parent_exec_policy_rules() { session_source: SessionSource::SubAgent(SubAgentSource::Other( GUARDIAN_REVIEWER_NAME.to_string(), )), + forked_from_thread_id: None, thread_source: None, agent_control: AgentControl::default(), dynamic_tools: Vec::new(), diff --git a/codex-rs/core/src/session/turn_context.rs b/codex-rs/core/src/session/turn_context.rs index de38083135..df1502274b 100644 --- a/codex-rs/core/src/session/turn_context.rs +++ b/codex-rs/core/src/session/turn_context.rs @@ -482,6 +482,7 @@ impl Session { let turn_metadata_state = Arc::new(TurnMetadataState::new( session_id.to_string(), thread_id.to_string(), + session_configuration.forked_from_thread_id, session_configuration.thread_source, sub_id.clone(), cwd.clone(), diff --git a/codex-rs/core/src/thread_manager.rs b/codex-rs/core/src/thread_manager.rs index 232f12ca71..d209026b62 100644 --- a/codex-rs/core/src/thread_manager.rs +++ b/codex-rs/core/src/thread_manager.rs @@ -581,6 +581,15 @@ impl ThreadManager { pub async fn start_thread_with_options( &self, options: StartThreadOptions, + ) -> CodexResult { + self.start_thread_with_options_and_fork_source(options, /*forked_from_thread_id*/ None) + .await + } + + async fn start_thread_with_options_and_fork_source( + &self, + options: StartThreadOptions, + forked_from_thread_id: Option, ) -> CodexResult { let session_source = options .session_source @@ -594,6 +603,7 @@ impl ThreadManager { Arc::clone(&self.state.auth_manager), self.agent_control(), session_source, + forked_from_thread_id, thread_source, options.dynamic_tools, options.persist_extended_history, @@ -634,7 +644,8 @@ impl ThreadManager { history, InterruptedTurnHistoryMarker::from_config(&options.config), ); - self.start_thread_with_options(options).await + self.start_thread_with_options_and_fork_source(options, Some(forked_from_thread_id)) + .await } pub async fn resume_thread_from_rollout( @@ -673,6 +684,7 @@ impl ThreadManager { initial_history, auth_manager, self.agent_control(), + /*forked_from_thread_id*/ None, thread_source, Vec::new(), persist_extended_history, @@ -698,6 +710,7 @@ impl ThreadManager { InitialHistory::New, Arc::clone(&self.state.auth_manager), self.agent_control(), + /*forked_from_thread_id*/ None, /*thread_source*/ None, Vec::new(), /*persist_extended_history*/ false, @@ -727,6 +740,7 @@ impl ThreadManager { initial_history, auth_manager, self.agent_control(), + /*forked_from_thread_id*/ None, thread_source, Vec::new(), /*persist_extended_history*/ false, @@ -876,6 +890,13 @@ impl ThreadManager { persist_extended_history: bool, parent_trace: Option, ) -> CodexResult { + // `forked_from_id()` describes this history's existing lineage. When + // forking a resumed thread, the child copies the resumed thread itself. + let forked_from_thread_id = match &history { + InitialHistory::Resumed(resumed) => Some(resumed.conversation_id), + InitialHistory::Forked(_) => history.forked_from_id(), + InitialHistory::New | InitialHistory::Cleared => None, + }; let interrupted_marker = InterruptedTurnHistoryMarker::from_config(&config); let history = fork_history_from_snapshot(snapshot, history, interrupted_marker); let environments = default_thread_environment_selections( @@ -887,6 +908,7 @@ impl ThreadManager { history, Arc::clone(&self.state.auth_manager), self.agent_control(), + forked_from_thread_id, thread_source, Vec::new(), persist_extended_history, @@ -1020,6 +1042,7 @@ impl ThreadManagerState { config, agent_control, self.session_source.clone(), + /*forked_from_thread_id*/ None, /*thread_source*/ None, /*persist_extended_history*/ false, /*metrics_service_name*/ None, @@ -1036,6 +1059,7 @@ impl ThreadManagerState { config: Config, agent_control: AgentControl, session_source: SessionSource, + forked_from_thread_id: Option, thread_source: Option, persist_extended_history: bool, metrics_service_name: Option, @@ -1052,6 +1076,7 @@ impl ThreadManagerState { Arc::clone(&self.auth_manager), agent_control, session_source, + forked_from_thread_id, thread_source, Vec::new(), persist_extended_history, @@ -1086,6 +1111,7 @@ impl ThreadManagerState { Arc::clone(&self.auth_manager), agent_control, session_source, + /*forked_from_thread_id*/ None, thread_source, Vec::new(), /*persist_extended_history*/ false, @@ -1107,6 +1133,7 @@ impl ThreadManagerState { agent_control: AgentControl, session_source: SessionSource, thread_source: Option, + forked_from_thread_id: Option, persist_extended_history: bool, inherited_shell_snapshot: Option>, inherited_exec_policy: Option>, @@ -1121,6 +1148,7 @@ impl ThreadManagerState { Arc::clone(&self.auth_manager), agent_control, session_source, + forked_from_thread_id, thread_source, Vec::new(), persist_extended_history, @@ -1142,6 +1170,7 @@ impl ThreadManagerState { initial_history: InitialHistory, auth_manager: Arc, agent_control: AgentControl, + forked_from_thread_id: Option, thread_source: Option, dynamic_tools: Vec, persist_extended_history: bool, @@ -1156,6 +1185,7 @@ impl ThreadManagerState { auth_manager, agent_control, self.session_source.clone(), + forked_from_thread_id, thread_source, dynamic_tools, persist_extended_history, @@ -1177,6 +1207,7 @@ impl ThreadManagerState { auth_manager: Arc, agent_control: AgentControl, session_source: SessionSource, + forked_from_thread_id: Option, thread_source: Option, dynamic_tools: Vec, persist_extended_history: bool, @@ -1229,6 +1260,7 @@ impl ThreadManagerState { extensions: Arc::clone(&self.extensions), conversation_history: initial_history, session_source, + forked_from_thread_id, thread_source, agent_control, dynamic_tools, diff --git a/codex-rs/core/src/turn_metadata.rs b/codex-rs/core/src/turn_metadata.rs index 1b2c6b4b12..52a714960a 100644 --- a/codex-rs/core/src/turn_metadata.rs +++ b/codex-rs/core/src/turn_metadata.rs @@ -16,6 +16,7 @@ use codex_git_utils::get_git_remote_urls_assume_git_repo; use codex_git_utils::get_git_repo_root; use codex_git_utils::get_has_changes; use codex_git_utils::get_head_commit_hash; +use codex_protocol::ThreadId; use codex_protocol::config_types::WindowsSandboxLevel; use codex_protocol::models::PermissionProfile; use codex_protocol::openai_models::ReasoningEffort as ReasoningEffortConfig; @@ -74,6 +75,8 @@ pub(crate) struct TurnMetadataBag { #[serde(default, skip_serializing_if = "Option::is_none")] thread_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + forked_from_thread_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] thread_source: Option, #[serde(default, skip_serializing_if = "Option::is_none")] turn_id: Option, @@ -107,7 +110,14 @@ fn merge_turn_metadata( } if let Some(responsesapi_client_metadata) = responsesapi_client_metadata { for (key, value) in responsesapi_client_metadata { - if key == TURN_STARTED_AT_UNIX_MS_KEY { + if matches!( + key.as_str(), + "session_id" + | "thread_id" + | "turn_id" + | TURN_STARTED_AT_UNIX_MS_KEY + | "forked_from_thread_id" + ) { continue; } metadata @@ -118,32 +128,6 @@ fn merge_turn_metadata( to_ascii_json_string(&metadata).ok() } -fn build_turn_metadata_bag( - session_id: Option, - thread_id: Option, - thread_source: Option, - turn_id: Option, - sandbox: Option, - repo_root: Option, - workspace_git_metadata: Option, -) -> TurnMetadataBag { - let mut workspaces = BTreeMap::new(); - if let (Some(repo_root), Some(workspace_git_metadata)) = (repo_root, workspace_git_metadata) - && !workspace_git_metadata.is_empty() - { - workspaces.insert(repo_root, workspace_git_metadata.into()); - } - - TurnMetadataBag { - session_id, - thread_id, - thread_source, - turn_id, - workspaces, - sandbox, - } -} - pub async fn build_turn_metadata_header( cwd: &AbsolutePathBuf, sandbox: Option<&str>, @@ -164,20 +148,24 @@ pub async fn build_turn_metadata_header( return None; } - build_turn_metadata_bag( - /*session_id*/ None, - /*thread_id*/ None, - /*thread_source*/ None, - /*turn_id*/ None, - sandbox.map(ToString::to_string), - repo_root, - Some(WorkspaceGitMetadata { - associated_remote_urls, - latest_git_commit_hash, - has_changes, - }), - ) - .to_header_value() + let workspace_git_metadata = WorkspaceGitMetadata { + associated_remote_urls, + latest_git_commit_hash, + has_changes, + }; + let mut metadata = TurnMetadataBag { + sandbox: sandbox.map(ToString::to_string), + ..Default::default() + }; + if let Some(repo_root) = repo_root + && !workspace_git_metadata.is_empty() + { + metadata + .workspaces + .insert(repo_root, workspace_git_metadata.into()); + } + + metadata.to_header_value() } #[derive(Clone, Debug)] @@ -198,6 +186,7 @@ impl TurnMetadataState { pub(crate) fn new( session_id: String, thread_id: String, + forked_from_thread_id: Option, thread_source: Option, turn_id: String, cwd: AbsolutePathBuf, @@ -214,15 +203,15 @@ impl TurnMetadataState { ) .to_string(), ); - let base_metadata = build_turn_metadata_bag( - Some(session_id), - Some(thread_id), + let base_metadata = TurnMetadataBag { + session_id: Some(session_id), + thread_id: Some(thread_id), + forked_from_thread_id, thread_source, - Some(turn_id), + turn_id: Some(turn_id), sandbox, - /*repo_root*/ None, - /*workspace_git_metadata*/ None, - ); + ..Default::default() + }; let base_header = base_metadata .to_header_value() .unwrap_or_else(|| "{}".to_string()); @@ -346,19 +335,13 @@ impl TurnMetadataState { let Some(repo_root) = state.repo_root.clone() else { return; }; - - let enriched_metadata = build_turn_metadata_bag( - state.base_metadata.session_id.clone(), - state.base_metadata.thread_id.clone(), - state.base_metadata.thread_source, - state.base_metadata.turn_id.clone(), - state.base_metadata.sandbox.clone(), - Some(repo_root), - Some(workspace_git_metadata), - ); - if enriched_metadata.workspaces.is_empty() { + if workspace_git_metadata.is_empty() { return; } + let mut enriched_metadata = state.base_metadata.clone(); + enriched_metadata + .workspaces + .insert(repo_root, workspace_git_metadata.into()); if let Some(header_value) = enriched_metadata.to_header_value() { *state diff --git a/codex-rs/core/src/turn_metadata_tests.rs b/codex-rs/core/src/turn_metadata_tests.rs index 2f79927103..a874b0db9b 100644 --- a/codex-rs/core/src/turn_metadata_tests.rs +++ b/codex-rs/core/src/turn_metadata_tests.rs @@ -93,6 +93,7 @@ fn turn_metadata_state_uses_platform_sandbox_tag() { let state = TurnMetadataState::new( "session-a".to_string(), "thread-a".to_string(), + /*forked_from_thread_id*/ None, Some(ThreadSource::User), "turn-a".to_string(), cwd, @@ -128,6 +129,7 @@ fn turn_metadata_state_uses_explicit_subagent_thread_source() { let state = TurnMetadataState::new( "session-a".to_string(), "thread-a".to_string(), + /*forked_from_thread_id*/ None, Some(ThreadSource::Subagent), "turn-a".to_string(), cwd, @@ -143,6 +145,35 @@ fn turn_metadata_state_uses_explicit_subagent_thread_source() { assert!(json.get("session_source").is_none()); } +#[test] +fn turn_metadata_state_includes_root_fork_lineage() { + let temp_dir = TempDir::new().expect("temp dir"); + let cwd = temp_dir.path().abs(); + let permission_profile = PermissionProfile::read_only(); + let source_thread_id = + ThreadId::from_string("11111111-1111-4111-8111-111111111111").expect("thread id"); + + let state = TurnMetadataState::new( + "session-a".to_string(), + "thread-a".to_string(), + Some(source_thread_id), + Some(ThreadSource::User), + "turn-a".to_string(), + cwd, + &permission_profile, + WindowsSandboxLevel::Disabled, + /*enforce_managed_network*/ false, + ); + + let header = state.current_header_value().expect("header"); + let json: Value = serde_json::from_str(&header).expect("json"); + + assert_eq!( + json["forked_from_thread_id"].as_str(), + Some("11111111-1111-4111-8111-111111111111") + ); +} + #[test] fn turn_metadata_state_includes_turn_started_at_unix_ms_after_start() { let temp_dir = TempDir::new().expect("temp dir"); @@ -152,6 +183,7 @@ fn turn_metadata_state_includes_turn_started_at_unix_ms_after_start() { let state = TurnMetadataState::new( "session-a".to_string(), "thread-a".to_string(), + /*forked_from_thread_id*/ None, Some(ThreadSource::User), "turn-a".to_string(), cwd, @@ -179,6 +211,7 @@ fn turn_metadata_state_includes_model_and_reasoning_effort_only_in_request_meta( let state = TurnMetadataState::new( "session-a".to_string(), "thread-a".to_string(), + /*forked_from_thread_id*/ None, /*thread_source*/ None, "turn-a".to_string(), cwd, @@ -224,6 +257,7 @@ fn turn_metadata_state_marks_user_input_requested_during_turn_only_for_mcp_reque let state = TurnMetadataState::new( "session-a".to_string(), "thread-a".to_string(), + /*forked_from_thread_id*/ None, /*thread_source*/ None, "turn-a".to_string(), cwd, @@ -266,7 +300,7 @@ fn turn_metadata_state_marks_user_input_requested_during_turn_only_for_mcp_reque } #[test] -fn turn_metadata_state_ignores_client_turn_started_at_unix_ms_before_start() { +fn turn_metadata_state_ignores_client_reserved_metadata_before_start() { let temp_dir = TempDir::new().expect("temp dir"); let cwd = temp_dir.path().abs(); let permission_profile = PermissionProfile::read_only(); @@ -274,6 +308,7 @@ fn turn_metadata_state_ignores_client_turn_started_at_unix_ms_before_start() { let state = TurnMetadataState::new( "session-a".to_string(), "thread-a".to_string(), + /*forked_from_thread_id*/ None, Some(ThreadSource::User), "turn-a".to_string(), cwd, @@ -281,15 +316,22 @@ fn turn_metadata_state_ignores_client_turn_started_at_unix_ms_before_start() { WindowsSandboxLevel::Disabled, /*enforce_managed_network*/ false, ); - state.set_responsesapi_client_metadata(HashMap::from([( - "turn_started_at_unix_ms".to_string(), - "client-supplied".to_string(), - )])); + state.set_responsesapi_client_metadata(HashMap::from([ + ( + "turn_started_at_unix_ms".to_string(), + "client-supplied".to_string(), + ), + ( + "forked_from_thread_id".to_string(), + "client-supplied".to_string(), + ), + ])); let header = state.current_header_value().expect("header"); let json: Value = serde_json::from_str(&header).expect("json"); assert!(json.get("turn_started_at_unix_ms").is_none()); + assert!(json.get("forked_from_thread_id").is_none()); } #[test] @@ -297,10 +339,13 @@ fn turn_metadata_state_merges_client_metadata_without_replacing_reserved_fields( let temp_dir = TempDir::new().expect("temp dir"); let cwd = temp_dir.path().abs(); let permission_profile = PermissionProfile::read_only(); + let source_thread_id = + ThreadId::from_string("44444444-4444-4444-8444-444444444444").expect("thread id"); let state = TurnMetadataState::new( "session-a".to_string(), "thread-a".to_string(), + Some(source_thread_id), Some(ThreadSource::User), "turn-a".to_string(), cwd, @@ -318,6 +363,10 @@ fn turn_metadata_state_merges_client_metadata_without_replacing_reserved_fields( ), ("session_id".to_string(), "client-supplied".to_string()), ("thread_id".to_string(), "client-supplied".to_string()), + ( + "forked_from_thread_id".to_string(), + "client-supplied".to_string(), + ), ("thread_source".to_string(), "client-supplied".to_string()), ( "turn_started_at_unix_ms".to_string(), @@ -337,6 +386,10 @@ fn turn_metadata_state_merges_client_metadata_without_replacing_reserved_fields( assert_eq!(json["reasoning_effort"].as_str(), Some("client-supplied")); assert_eq!(json["session_id"].as_str(), Some("session-a")); assert_eq!(json["thread_id"].as_str(), Some("thread-a")); + assert_eq!( + json["forked_from_thread_id"].as_str(), + Some("44444444-4444-4444-8444-444444444444") + ); assert_eq!(json["thread_source"].as_str(), Some("user")); assert_eq!(json["turn_id"].as_str(), Some("turn-a")); assert_eq!( diff --git a/codex-rs/core/tests/suite/responses_api_proxy_headers.rs b/codex-rs/core/tests/suite/responses_api_proxy_headers.rs index 809a8bb701..0c401d05fc 100644 --- a/codex-rs/core/tests/suite/responses_api_proxy_headers.rs +++ b/codex-rs/core/tests/suite/responses_api_proxy_headers.rs @@ -125,6 +125,15 @@ async fn responses_api_parent_and_subagent_requests_include_identity_headers() - child.header("x-codex-parent-thread-id").as_deref(), Some(parent_thread_id) ); + let child_turn_metadata: serde_json::Value = serde_json::from_str( + &child + .header("x-codex-turn-metadata") + .ok_or_else(|| anyhow!("child request missing x-codex-turn-metadata"))?, + )?; + assert_eq!( + child_turn_metadata["forked_from_thread_id"].as_str(), + Some(parent_thread_id) + ); Ok(()) } diff --git a/codex-rs/core/tests/suite/review.rs b/codex-rs/core/tests/suite/review.rs index 802035d7dc..9efaf6c2d1 100644 --- a/codex-rs/core/tests/suite/review.rs +++ b/codex-rs/core/tests/suite/review.rs @@ -61,7 +61,7 @@ async fn review_op_emits_lifecycle_and_review_output() { "overall_confidence_score": 0.8 }) .to_string(); - let (server, _request_log) = start_responses_server_with_sse( + let (server, request_log) = start_responses_server_with_sse( assistant_message_sse(&review_json), /*expected_requests*/ 1, ) @@ -111,11 +111,38 @@ async fn review_op_emits_lifecycle_and_review_output() { assert_eq!(expected, review); let _complete = wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; - // Also verify that a user message with the header and a formatted finding - // was recorded back in the parent session's rollout. let path = codex.rollout_path().expect("rollout path"); let text = std::fs::read_to_string(&path).expect("read rollout file"); + let parent_thread_id = text + .lines() + .filter(|line| !line.trim().is_empty()) + .find_map(|line| { + let rollout_line: RolloutLine = serde_json::from_str(line).expect("rollout line"); + match rollout_line.item { + RolloutItem::SessionMeta(session_meta) => Some(session_meta.meta.id.to_string()), + _ => None, + } + }) + .expect("parent session meta"); + let request = request_log.single_request(); + assert_eq!( + request.header("x-openai-subagent").as_deref(), + Some("review") + ); + let turn_metadata: serde_json::Value = serde_json::from_str( + &request + .header("x-codex-turn-metadata") + .expect("review request turn metadata"), + ) + .expect("review request turn metadata json"); + assert_eq!( + turn_metadata["forked_from_thread_id"].as_str(), + Some(parent_thread_id.as_str()) + ); + + // Also verify that a user message with the header and a formatted finding + // was recorded back in the parent session's rollout. let mut saw_header = false; let mut saw_finding_line = false; let expected_assistant_text = render_review_output_text(&expected); From 66ff8b0f5467ea4b77e30b5b4f186d9145fef636 Mon Sep 17 00:00:00 2001 From: sayan-oai Date: Tue, 26 May 2026 14:05:40 -0700 Subject: [PATCH 13/16] make direct only allowed caller for standalone websearch (#24646) only allow `Direct` callers of the standalone websearch tool because its not supported in codemode --- codex-rs/app-server/tests/suite/v2/web_search.rs | 4 ++++ codex-rs/ext/web-search/src/extension.rs | 2 ++ 2 files changed, 6 insertions(+) diff --git a/codex-rs/app-server/tests/suite/v2/web_search.rs b/codex-rs/app-server/tests/suite/v2/web_search.rs index 52c327a93f..58b2ff91a8 100644 --- a/codex-rs/app-server/tests/suite/v2/web_search.rs +++ b/codex-rs/app-server/tests/suite/v2/web_search.rs @@ -128,6 +128,10 @@ async fn standalone_web_search_round_trips_encrypted_output() -> Result<()> { "search_query": [{"q": "standalone web search"}], }) ); + assert_eq!( + search_body["settings"]["allowed_callers"], + json!(["direct"]) + ); assert_eq!( search_body["input"] .as_array() diff --git a/codex-rs/ext/web-search/src/extension.rs b/codex-rs/ext/web-search/src/extension.rs index 228d77897b..b7b6c83951 100644 --- a/codex-rs/ext/web-search/src/extension.rs +++ b/codex-rs/ext/web-search/src/extension.rs @@ -1,5 +1,6 @@ use std::sync::Arc; +use codex_api::AllowedCaller; use codex_api::ApproximateLocation; use codex_api::LocationType; use codex_api::SearchContextSize; @@ -71,6 +72,7 @@ fn search_settings(config: &Config, web_search_mode: WebSearchMode) -> SearchSet allowed_domains: filters.allowed_domains.clone(), blocked_domains: None, }), + allowed_callers: Some(vec![AllowedCaller::Direct]), external_web_access: Some(match web_search_mode { WebSearchMode::Live => true, WebSearchMode::Cached | WebSearchMode::Disabled => false, From 675cb1afbda5de4585345371019756b107a5812e Mon Sep 17 00:00:00 2001 From: Curtis 'Fjord' Hawthorne Date: Tue, 26 May 2026 14:17:43 -0700 Subject: [PATCH 14/16] Clarify view_image tool description (#23949) --- codex-rs/core/src/tools/handlers/view_image_spec.rs | 2 +- codex-rs/core/tests/suite/code_mode.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/codex-rs/core/src/tools/handlers/view_image_spec.rs b/codex-rs/core/src/tools/handlers/view_image_spec.rs index 662cd639a6..ed0b1021a5 100644 --- a/codex-rs/core/src/tools/handlers/view_image_spec.rs +++ b/codex-rs/core/src/tools/handlers/view_image_spec.rs @@ -40,7 +40,7 @@ pub fn create_view_image_tool(options: ViewImageToolOptions) -> ToolSpec { ToolSpec::Function(ResponsesApiTool { name: VIEW_IMAGE_TOOL_NAME.to_string(), - description: "View a local image from the filesystem (only use if given a full filepath by the user, and the image isn't already attached to the thread context within tags)." + description: "View a local image file from the filesystem when visual inspection is needed. Use this for images already available on disk." .to_string(), strict: false, defer_loading: None, diff --git a/codex-rs/core/tests/suite/code_mode.rs b/codex-rs/core/tests/suite/code_mode.rs index fb9e7fb420..90daf21963 100644 --- a/codex-rs/core/tests/suite/code_mode.rs +++ b/codex-rs/core/tests/suite/code_mode.rs @@ -2861,7 +2861,7 @@ text(JSON.stringify(tool)); parsed, serde_json::json!({ "name": "view_image", - "description": "View a local image from the filesystem (only use if given a full filepath by the user, and the image isn't already attached to the thread context within tags).\n\nexec tool declaration:\n```ts\ndeclare const tools: { view_image(args: {\n // Local filesystem path to an image file\n path: string;\n}): Promise<{\n // Image detail hint returned by view_image. Returns `high` for default resized behavior or `original` when original resolution is preserved.\n detail: \"high\" | \"original\";\n // Data URL for the loaded image.\n image_url: string;\n}>; };\n```", + "description": "View a local image file from the filesystem when visual inspection is needed. Use this for images already available on disk.\n\nexec tool declaration:\n```ts\ndeclare const tools: { view_image(args: {\n // Local filesystem path to an image file\n path: string;\n}): Promise<{\n // Image detail hint returned by view_image. Returns `high` for default resized behavior or `original` when original resolution is preserved.\n detail: \"high\" | \"original\";\n // Data URL for the loaded image.\n image_url: string;\n}>; };\n```", }) ); From 22e45014a2820a8c5a7037d15dab903ccd3eb994 Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Tue, 26 May 2026 14:34:02 -0700 Subject: [PATCH 15/16] TUI config cleanup: plugin mentions (#24266) ## Summary TUI plugin mention refresh still joined app-server plugin inventory with client-local plugin config, which can diverge once plugin state is owned by the app server. This changes the TUI to mirror the GUI client: `plugin/list` is the autocomplete source, and mention candidates are plugin-level entries filtered to installed, enabled, and not disabled by admin. The TUI no longer reads local plugin config or calls `plugin/read` while refreshing plugin mention candidates. ## API shape and limitations The current app-server API does not expose effective per-session plugin capability summaries for mention autocomplete. As in the GUI, autocomplete now trusts `plugin/list` metadata rather than proving which plugin capabilities are loaded in the active session. That avoids stale client-local reads and the cwd/remote detail gaps in `plugin/read`, but intentionally accepts the same list-level tradeoff as the app: if `plugin/list` reports a remote plugin before its local bundle is materialized, the plugin can still appear as a mention candidate. --- codex-rs/tui/src/app/background_requests.rs | 2 +- codex-rs/tui/src/app/plugin_mentions.rs | 192 ++++++++++---------- 2 files changed, 97 insertions(+), 97 deletions(-) diff --git a/codex-rs/tui/src/app/background_requests.rs b/codex-rs/tui/src/app/background_requests.rs index f81f8b071e..4b8b5f8920 100644 --- a/codex-rs/tui/src/app/background_requests.rs +++ b/codex-rs/tui/src/app/background_requests.rs @@ -409,7 +409,7 @@ impl App { } tokio::spawn(async move { - match fetch_plugin_mentions(request_handle, config).await { + match fetch_plugin_mentions(request_handle, config.cwd.to_path_buf()).await { Ok(plugins) => { app_event_tx.send(AppEvent::PluginMentionsLoaded { plugins: Some(plugins), diff --git a/codex-rs/tui/src/app/plugin_mentions.rs b/codex-rs/tui/src/app/plugin_mentions.rs index ef300896bc..5d94f51c08 100644 --- a/codex-rs/tui/src/app/plugin_mentions.rs +++ b/codex-rs/tui/src/app/plugin_mentions.rs @@ -1,125 +1,62 @@ //! Plugin mention capability enrichment for the TUI. //! -//! Mention inventory comes from app-server `plugin/list`, while mention eligibility still reuses -//! the older local bulk capability summaries. That keeps the feature app-server-shaped without -//! paying for a `plugin/read` per plugin. +//! Mention inventory comes from app-server `plugin/list`, matching the GUI +//! client. The current API exposes plugin-level mention metadata there, but not +//! effective per-session capability summaries. use super::background_requests::request_plugin_list; use super::*; +use codex_app_server_protocol::PluginAvailability; use codex_app_server_protocol::PluginListResponse; -use codex_app_server_protocol::PluginMarketplaceEntry; use codex_app_server_protocol::PluginSummary; -use codex_core_plugins::PluginsManager; use codex_plugin::PluginCapabilitySummary; -use std::collections::HashMap; - -#[derive(Debug, Clone)] -struct PluginMentionEntry { - config_name: String, - display_name: String, - description: Option, -} - -impl PluginMentionEntry { - fn capability_summary( - self, - capabilities_by_config_name: &HashMap, - ) -> Option { - let capabilities = capabilities_by_config_name.get(&self.config_name)?; - Some(PluginCapabilitySummary { - config_name: self.config_name, - display_name: self.display_name, - description: self.description, - has_skills: capabilities.has_skills, - mcp_server_names: capabilities.mcp_server_names.clone(), - app_connector_ids: capabilities.app_connector_ids.clone(), - }) - } -} pub(super) async fn fetch_plugin_mentions( request_handle: AppServerRequestHandle, - config: crate::legacy_core::config::Config, + cwd: PathBuf, ) -> Result> { - let response = request_plugin_list(request_handle, config.cwd.to_path_buf()).await?; - let mention_entries = plugin_mention_entries_from_list_response(response); - let capabilities_by_config_name = load_plugin_mention_capabilities(&config).await; - - Ok(mention_entries - .into_iter() - .filter_map(|entry| entry.capability_summary(&capabilities_by_config_name)) - .collect()) + let response = request_plugin_list(request_handle, cwd).await?; + Ok(plugin_mentions_from_list_response(response)) } -async fn load_plugin_mention_capabilities( - config: &crate::legacy_core::config::Config, -) -> HashMap { - let plugins_input = config.plugins_config_input(); - PluginsManager::new(config.codex_home.to_path_buf()) - .plugins_for_config(&plugins_input) - .await - .capability_summaries() - .iter() - .cloned() - .map(|summary| (summary.config_name.clone(), summary)) - .collect() -} - -fn plugin_mention_entries_from_list_response( +fn plugin_mentions_from_list_response( response: PluginListResponse, -) -> Vec { +) -> Vec { response .marketplaces .into_iter() - .flat_map(plugin_mention_entries_from_marketplace) + .flat_map(|marketplace| { + let marketplace_name = marketplace.name; + marketplace + .plugins + .into_iter() + .filter_map(move |plugin| plugin_mention_from_summary(&marketplace_name, plugin)) + }) .collect() } -fn plugin_mention_entries_from_marketplace( - marketplace: PluginMarketplaceEntry, -) -> Vec { - let marketplace_name = marketplace.name; - marketplace - .plugins - .into_iter() - .filter_map(|plugin| plugin_mention_entry(&marketplace_name, plugin)) - .collect() +fn plugin_is_eligible_for_mentions(plugin: &PluginSummary) -> bool { + plugin.installed && plugin.enabled && plugin.availability != PluginAvailability::DisabledByAdmin } -fn plugin_mention_entry( +fn plugin_mention_from_summary( marketplace_name: &str, plugin: PluginSummary, -) -> Option { +) -> Option { if !plugin_is_eligible_for_mentions(&plugin) { return None; } - let config_name = plugin_mention_config_name(marketplace_name, &plugin)?; - Some(PluginMentionEntry { - config_name, + Some(PluginCapabilitySummary { + config_name: plugin.id.clone(), display_name: plugin_mention_display_name(&plugin), - description: plugin_mention_description(&plugin), + description: plugin_mention_description(marketplace_name, &plugin), + has_skills: false, + mcp_server_names: Vec::new(), + app_connector_ids: Vec::new(), }) } -fn plugin_is_eligible_for_mentions(plugin: &PluginSummary) -> bool { - plugin.installed && plugin.enabled -} - -fn plugin_mention_config_name(marketplace_name: &str, plugin: &PluginSummary) -> Option { - codex_plugin::PluginId::new(plugin.name.clone(), marketplace_name.to_string()) - .map(|plugin_id| plugin_id.as_key()) - .map_err(|err| { - tracing::warn!( - plugin_name = plugin.name, - marketplace_name, - error = %err, - "skipping plugin mention with invalid identity" - ); - }) - .ok() -} - fn plugin_mention_display_name(plugin: &PluginSummary) -> String { plugin .interface @@ -131,17 +68,80 @@ fn plugin_mention_display_name(plugin: &PluginSummary) -> String { .unwrap_or_else(|| plugin.name.clone()) } -fn plugin_mention_description(plugin: &PluginSummary) -> Option { +fn plugin_mention_description(marketplace_name: &str, plugin: &PluginSummary) -> Option { plugin .interface .as_ref() - .and_then(|interface| { - interface - .short_description - .as_deref() - .or(interface.long_description.as_deref()) - }) + .and_then(|interface| interface.short_description.as_deref()) .map(str::trim) .filter(|description| !description.is_empty()) .map(str::to_string) + .or_else(|| { + let marketplace_name = marketplace_name.trim(); + (!marketplace_name.is_empty()).then(|| marketplace_name.to_string()) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use codex_app_server_protocol::PluginAuthPolicy; + use codex_app_server_protocol::PluginAvailability; + use codex_app_server_protocol::PluginInstallPolicy; + use codex_app_server_protocol::PluginListResponse; + use codex_app_server_protocol::PluginMarketplaceEntry; + use codex_app_server_protocol::PluginSource; + use pretty_assertions::assert_eq; + + #[test] + fn plugin_mentions_use_plugin_list_summaries_and_gui_eligibility() { + let active = plugin_summary("active"); + let mut disabled_by_admin = plugin_summary("disabled-by-admin"); + disabled_by_admin.availability = PluginAvailability::DisabledByAdmin; + let mut disabled = plugin_summary("disabled"); + disabled.enabled = false; + let mut uninstalled = plugin_summary("uninstalled"); + uninstalled.installed = false; + + let response = PluginListResponse { + marketplaces: vec![PluginMarketplaceEntry { + name: "server-marketplace".to_string(), + path: None, + interface: None, + plugins: vec![active, disabled_by_admin, disabled, uninstalled], + }], + marketplace_load_errors: Vec::new(), + featured_plugin_ids: Vec::new(), + }; + + assert_eq!( + plugin_mentions_from_list_response(response), + vec![PluginCapabilitySummary { + config_name: "active@server-marketplace".to_string(), + display_name: "active".to_string(), + description: Some("server-marketplace".to_string()), + has_skills: false, + mcp_server_names: Vec::new(), + app_connector_ids: Vec::new(), + }] + ); + } + + fn plugin_summary(name: &str) -> PluginSummary { + PluginSummary { + id: format!("{name}@server-marketplace"), + remote_plugin_id: Some(format!("plugins~{name}")), + local_version: None, + name: name.to_string(), + share_context: None, + source: PluginSource::Remote, + installed: true, + enabled: true, + install_policy: PluginInstallPolicy::Available, + auth_policy: PluginAuthPolicy::OnInstall, + availability: PluginAvailability::Available, + interface: None, + keywords: Vec::new(), + } + } } From 414561294c3955ed6e2020873758cb3262287fb9 Mon Sep 17 00:00:00 2001 From: Eric Traut Date: Tue, 26 May 2026 14:40:06 -0700 Subject: [PATCH 16/16] Avoid repeated marketplace upgrades for alternate layouts (#24320) Fixes #24249. ## Why Codex already supports discovering marketplaces under both `.agents/plugins/marketplace.json` and `.claude-plugin/marketplace.json`. The Git marketplace auto-upgrade no-op check only looked for the `.agents` layout. That meant an installed `.claude-plugin` marketplace with matching revision metadata still looked absent, so plugin list/startup upgrade work could stage and re-activate the same marketplace again. That matches the failure shape in #24249: the report called out repeated marketplace sync/cache refresh logs and a large recently-touched `.tmp/marketplaces/.staging` directory. This change makes the auto-upgrade path recognize the installed `.claude-plugin` marketplace as already current, which should remove that staging/activation feedback loop. ## What changed `codex-rs/core-plugins/src/marketplace_upgrade.rs` now uses the existing supported marketplace manifest discovery helper when deciding whether an installed Git marketplace is already current. Existing local plugin source validation is unchanged; `source: "./"` still remains invalid. ## Confidence Confidence is high that this fixes the repeated marketplace upgrade path: the old hardcoded layout check was definitely wrong for installed `.claude-plugin` marketplaces, and the reported staging churn points directly at that path. Confidence is not 100% because we do not have a CPU profile or a fully re-run reporter repro. A malformed marketplace entry can still be logged as invalid if another caller repeatedly lists plugins; this PR fixes the staging/upgrade feedback loop that likely made the failure pathological, not every possible source of repeated marketplace resolution. --- codex-rs/core-plugins/src/marketplace_upgrade.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/codex-rs/core-plugins/src/marketplace_upgrade.rs b/codex-rs/core-plugins/src/marketplace_upgrade.rs index 9dda8a9a3e..010694c8b3 100644 --- a/codex-rs/core-plugins/src/marketplace_upgrade.rs +++ b/codex-rs/core-plugins/src/marketplace_upgrade.rs @@ -6,6 +6,7 @@ use self::activation::installed_marketplace_metadata_matches; use self::activation::write_installed_marketplace_metadata; use self::git::clone_git_source; use self::git::git_remote_revision; +use crate::marketplace::find_marketplace_manifest_path; use crate::marketplace::validate_marketplace_root; use codex_config::CONFIG_TOML_FILE; use codex_config::ConfigLayerStack; @@ -176,9 +177,7 @@ fn upgrade_configured_git_marketplace( MARKETPLACE_UPGRADE_GIT_TIMEOUT, )?; let destination = install_root.join(&marketplace.name); - if destination - .join(".agents/plugins/marketplace.json") - .is_file() + if find_marketplace_manifest_path(&destination).is_some() && marketplace.last_revision.as_deref() == Some(remote_revision.as_str()) && installed_marketplace_metadata_matches(&destination, marketplace, &remote_revision) {